def __init__(self, family: ComposeFamily):
"""
Initialize network settings for the family ServiceConfig
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
"""
self.family = family
self._network_mode = "awsvpc"
if family.service_compute.launch_type == "EXTERNAL":
LOG.warning(
f"{family.name} - External mode cannot use awsvpc mode. Falling back to bridge"
)
self.network_mode = "bridge"
self.ports = []
self.networks = {}
self.merge_services_ports()
self.merge_networks()
self.definition = merge_family_services_networking(family)
self.ingress_from_self = False
if any([svc.expose_ports for svc in family.services]):
self.ingress_from_self = True
LOG.info(
f"{family.name} - services have export ports, allowing internal ingress"
)
self._security_group = None
self.extra_security_groups = []
self._subnets = Ref(APP_SUBNETS)
self.cloudmap_config = (merge_cloudmap_settings(family, self.ports)
if self.ports else {})
self.ingress = Ingress(self.definition[Ingress.master_key], self.ports)
self.ingress_from_self = keyisset(self.self_key, self.definition)
def set_services_scaling_list(self, settings):
"""
Method to map services and families targets of the services defined.
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
if not self.services or isinstance(self.services, dict):
return
for service in self.services:
name_key = get_setting_key("name", service)
scaling_key = get_setting_key("scaling", service)
if not keyisset(scaling_key, service):
LOG.debug(
f"{self.module.res_key}.{self.name} - No Scaling set for {service[name_key]}"
)
continue
service_name = service[name_key]
if service_name in settings.families and service_name not in [
f[0].name for f in self.families_scaling
]:
self.families_scaling.append(
(settings.families[service_name], service[scaling_key]))
elif service_name in settings.families and service_name in [
f[0].name for f in self.families_scaling
]:
LOG.debug(
f"{self.module.res_key}.{self.name} - Family {service_name} has already been added. Skipping"
)
elif service_name in [s.name for s in settings.services]:
self.handle_family_scaling_expansion(service, settings)
def set_update_log_configuration(self, **kwargs):
if kwargs and keyisset("LogDriver", kwargs) and keyisset("Options", kwargs):
self.log_configuration = LogConfiguration(**kwargs)
return
if self.log_driver == "awslogs":
self.log_configuration = handle_awslogs_options(
self.service, self.log_config
)
if self.replace_cw_with_firelens:
self.log_configuration = replace_awslogs_with_firelens_configuration(
self.service, self.log_configuration
)
elif self.log_driver == "awsfirelens":
self.log_configuration = handle_firelens_options(
self.service, self.log_config
)
def set_sid_name(access_definition, access_subkey: str) -> str:
"""
Defines the name of the SID to use for the policy. Defines access_type
:param dict,str access_definition:
:param str access_subkey:
:return: access_type
:rtype: str
"""
if isinstance(access_definition, dict) and keyisset(
access_subkey, access_definition
):
if isinstance(access_definition[access_subkey], bool):
access_type = access_subkey
else:
access_type = f"{access_subkey}{access_definition[access_subkey]}"
elif isinstance(access_definition, str):
access_type = access_definition
else:
raise ValueError(
"The access_definition is not valid",
access_definition,
type(access_definition),
"subkey is",
access_subkey,
)
return access_type
def handle_families_targets_expansion_dict(self, service_name, service,
settings) -> None:
"""
Method to list all families and services that are targets of the resource.
Allows to implement family and service level association to resource
:param str service_name:
:param dict service: Service definition in compose file
:param ecs_composex.common.settings.ComposeXSettings settings: Execution settings
"""
for svc in settings.services:
if svc.name == service_name:
the_service = svc
break
else:
raise KeyError(
f"Service {service_name} not found in ",
[_svc.name for _svc in settings.services],
)
for family_name in the_service.families:
family_name = NONALPHANUM.sub("", family_name)
if family_name not in [f[0].name for f in self.families_targets]:
self.families_targets.append((
settings.families[family_name],
False,
[the_service],
service["Access"] if keyisset("Access", service) else {},
service,
))
def get_bucket_config(bucket: Bucket, resource_id: str) -> dict:
"""
:param ecs_composex.s3.s3_bucket.Bucket bucket:
:param str resource_id:
"""
bucket_config = {
S3_BUCKET_NAME: resource_id,
S3_BUCKET_ARN: bucket.arn,
}
client = bucket.lookup_session.client("s3")
try:
encryption_r = client.get_bucket_encryption(Bucket=resource_id)
encryption_attributes = attributes_to_mapping(
encryption_r, CONTROL_CLOUD_ATTR_MAPPING)
if keyisset(
CONTROL_CLOUD_ATTR_MAPPING[S3_BUCKET_KMS_KEY],
encryption_attributes,
):
bucket_config[S3_BUCKET_KMS_KEY] = encryption_attributes[
S3_BUCKET_KMS_KEY]
except ClientError as error:
if (not error.response["Error"]["Code"]
== "ServerSideEncryptionConfigurationNotFoundError"):
raise
LOG.warning(error.response["Error"]["Message"])
return bucket_config
def set_service_update_config(family) -> dict:
"""
Method to determine the update_config for the service. When a family has multiple containers, this applies
to all tasks.
"""
deployment_config = {}
min_percents = [
int(service.definition["x-aws-min_percent"])
for service in family.services
if keypresent("x-aws-min_percent", service.definition)
]
max_percents = [
int(service.definition["x-aws-max_percent"])
for service in family.services
if keypresent("x-aws-max_percent", service.definition)
]
family_min_percent = define_family_deploy_percents(min_percents, 100)
family_max_percent = define_family_deploy_percents(max_percents, 200)
rollback = True
actions = [
service.update_config["failure_action"] != "rollback"
for service in family.services if service.update_config
and keyisset("failure_action", service.update_config)
]
if any(actions):
rollback = False
deployment_config.update({
"MinimumHealthyPercent": family_min_percent,
"MaximumPercent": family_max_percent,
"RollBack": rollback,
})
return deployment_config
def get_setting_key(name: str, settings_dict: dict) -> str:
"""
Allows for flexibility in the syntax, i.e. to make access/Access both valid
"""
if keyisset(name.title(), settings_dict):
return name.title()
return name
def set_resources(settings: ComposeXSettings, resource_class,
module: XResourceModule):
"""
Method to define the ComposeXResource for each service.
First updates the resources dict
:param ecs_composex.common.settings.ComposeXSettings settings:
:param ecs_composex.common.compose_resources.XResource resource_class:
:param XResourceModule module:
"""
if not keyisset(module.res_key, settings.compose_content):
return
resources_ordered_dict = OrderedDict(
sorted(
settings.compose_content[module.res_key].items(),
key=lambda item: item[0],
))
del settings.compose_content[module.res_key]
settings.compose_content[module.res_key] = resources_ordered_dict
for resource_name, resource_definition in resources_ordered_dict.items():
new_definition = resource_class(
name=resource_name,
definition=resource_definition,
module=module,
settings=settings,
)
LOG.debug(type(new_definition))
LOG.debug(new_definition.__dict__)
settings.compose_content[
module.res_key][resource_name] = new_definition
def create_root_template(new_resources: list, module_res_key: str) -> Template:
"""
Function to create the root stack template for profiles
:param list[CodeProfiler] new_resources:
:param str module_res_key:
:return: The template wit the profiles
:rtype: troposphere.Template
"""
root_tpl = build_template(f"Root stack to manage {module_res_key}")
for res in new_resources:
try:
props = import_record_properties(res.properties,
ProfilingGroup,
ignore_missing_required=False)
if res.parameters and keyisset("AppendStackId", res.parameters):
props["ProfilingGroupName"] = Sub(
f"{res.properties['ProfilingGroupName']}-${{StackId}}",
StackId=STACK_ID_SHORT,
)
except KeyError:
props = import_record_properties(res.properties,
ProfilingGroup,
ignore_missing_required=True)
props["ProfilingGroupName"] = Sub(
f"{res.logical_name}-${{StackId}}", StackId=STACK_ID_SHORT)
res.cfn_resource = ProfilingGroup(res.logical_name, **props)
res.init_outputs()
res.generate_outputs()
add_outputs(root_tpl, res.outputs)
root_tpl.add_resource(res.cfn_resource)
return root_tpl
def determine_resource_type(db_name, properties):
"""
Function to determine if the properties are the ones of a DB Cluster or DB Instance.
By default it will assume Cluster if cannot conclude that it is a DB Instance
:param str db_name:
:param dict properties:
:return:
"""
if (
keyisset(DB_ENGINE_NAME.title, properties)
and properties[DB_ENGINE_NAME.title].startswith("aurora")
or all(
property_name in DBCluster.props.keys()
for property_name in properties.keys()
)
):
LOG.info(f"Identified {db_name} to be a RDS Aurora Cluster")
return DBCluster
elif all(
property_name in DBInstance.props.keys() for property_name in properties.keys()
):
LOG.info(f"Identified {db_name} to be a RDS Instance")
return DBInstance
LOG.error(
"From the properties defined, we cannot determine whether this is a RDS Cluster or RDS Instance."
" Setting to Cluster"
)
return None
def __init__(self, title, settings: ComposeXSettings,
module: XResourceModule, **kwargs):
"""
Init method
:param str title:
:param ecs_composex.common.settings.ComposeXSettings settings:
:param kwargs:
"""
set_resources(settings, CodeProfiler, module)
x_resources = settings.compose_content[module.res_key].values()
lookup_resources = set_lookup_resources(x_resources)
if lookup_resources:
if not keyisset(module.mapping_key, settings.mappings):
settings.mappings[module.mapping_key] = {}
define_lookup_profile_mappings(
settings.mappings[module.mapping_key], lookup_resources,
settings)
new_resources = set_new_resources(x_resources, False)
if new_resources:
stack_template = create_root_template(new_resources,
module.res_key)
super().__init__(title, stack_template, **kwargs)
else:
self.is_void = True
for resource in settings.compose_content[module.res_key].values():
resource.stack = self
def resolve_lookup(
lookup_resources: list[UserPool],
settings: ComposeXSettings,
module: XResourceModule,
):
"""
Iterates over the lookup resources and performs the lookup to create the resource mapping used in the template.
:param list[UserPool] lookup_resources: the lookup resources to process
:param ecs_composex.common.settings.ComposeXSettings settings: the ComposeX Execution settings.
:param module:
"""
if not keyisset(module.mapping_key, settings.mappings):
settings.mappings[module.mapping_key] = {}
for resource in lookup_resources:
resource.lookup_resource(
USER_POOL_RE,
get_userpool_config,
CfnUserPool.resource_type,
"cognito-idp",
)
resource.init_outputs()
resource.generate_cfn_mappings_from_lookup_properties()
resource.generate_outputs()
settings.mappings[module.mapping_key].update(
{resource.logical_name: resource.mappings})
LOG.info(
f"{resource.module.res_key}.{resource.name} Found in AWS Account")
def __init__(self, title, settings: ComposeXSettings,
module: XResourceModule, **kwargs):
set_resources(settings, NeptuneDBCluster, module)
x_resources = settings.compose_content[module.res_key].values()
new_resources = set_new_resources(x_resources, True)
lookup_resources = set_lookup_resources(x_resources)
if new_resources:
stack_template = build_template(
"Root template for Neptune by ComposeX",
[VPC_ID, STORAGE_SUBNETS])
super().__init__(title, stack_template, **kwargs)
create_neptune_template(stack_template, new_resources, settings,
self)
else:
self.is_void = True
if lookup_resources:
if not keyisset(module.mapping_key, settings.mappings):
settings.mappings[module.mapping_key] = {}
for resource in lookup_resources:
resource.lookup_resource(
NEPTUNE_DB_CLUSTER_ARN_RE,
get_db_cluster_config,
CfnDBCluster.resource_type,
"rds:cluster",
)
resource.generate_cfn_mappings_from_lookup_properties()
resource.generate_outputs()
settings.mappings[module.mapping_key].update(
{resource.logical_name: resource.mappings})
for resource in settings.compose_content[module.res_key].values():
resource.stack = self
def set_content(self, kwargs, content=None, fully_load=True):
"""
Method to initialize the compose content
:param dict kwargs:
:param dict content:
:param bool fully_load:
"""
files = (
[]
if not keyisset(self.input_file_arg, kwargs)
else kwargs[self.input_file_arg]
)
content_def = ComposeDefinition(files, content)
self.compose_content = content_def.definition
source = pkg_files("ecs_composex").joinpath("specs/compose-spec.json")
LOG.info(f"Validating against input schema {source}")
resolver = jsonschema.RefResolver(
f"file://{path.abspath(path.dirname(source))}/", None
)
jsonschema.validate(
content_def.definition,
loads(source.read_text()),
resolver=resolver,
)
if fully_load:
self.set_secrets()
self.set_volumes()
self.set_services()
self.set_families()
self.set_efs()
def handle_firelens_options(service: ComposeService,
logging_def: dict,
set_cw_default: bool = False) -> LogConfiguration:
default_cloudwatch_options = {
"region": Region,
"auto_create_group": True,
"log_group_name": service.logical_name,
"log_stream_prefix": service.service_name,
}
if set_cw_default:
options = set_else_none("options",
logging_def,
alt_value=default_cloudwatch_options)
else:
options = set_else_none("options", logging_def, alt_value=NoValue)
config_name_map = {
"delivery_stream": "kinesis_firehose",
"log_group_name": "cloudwatch",
"stream": "kinesis_streams",
"bucket": "s3",
}
for key, value in config_name_map.items():
if keyisset(key, options):
options.update({"Name": value})
break
return LogConfiguration(LogDriver="awsfirelens", Options=options)
def set_output_settings(self, kwargs):
"""
Method to set the output settings based on kwargs
"""
self.format = self.default_format
if (
keyisset(self.format_arg, kwargs)
and kwargs[self.format_arg] in self.allowed_formats
):
self.format = kwargs[self.format_arg]
self.output_dir = (
kwargs[self.output_dir_arg]
if keyisset(self.output_dir_arg, kwargs)
else self.default_output_dir
)
def replicas(self):
if self.family.stack and keyisset(
ecs_params.SERVICE_COUNT.title, self.family.stack.Parameters
):
return self.family.stack.Parameters[ecs_params.SERVICE_COUNT.title]
else:
return max(service.replicas for service in self.family.services)
def add_all_tags(root_template, settings, params=None, xtags=None):
"""
Function to go through all stacks of a given template and update the template
It will recursively render sub stacks defined.
If there are no substacks, it will go over the resources of the template add the tags.
:param troposphere.Template root_template: the root template to iterate over the resources.
:param ecs_composex.common.settings.ComposeXSettings settings: Execution settings
:param list params: Parameters to add to template if any
:param troposphere.Tags xtags: List of Tags to add to the resources.
"""
if not params or not xtags:
if not keyisset("x-tags", settings.compose_content):
xtags = default_tags()
params = None
else:
tags = settings.compose_content["x-tags"]
params = generate_tags_parameters(tags)
xtags = define_extended_tags(tags)
xtags += default_tags()
resources = root_template.resources if root_template else []
for resource_name in resources:
resource = resources[resource_name]
apply_tags_to_resources(settings, resource, params, xtags)
def define_kms_key(self):
"""
Method to set the KMS Key
"""
if not self.properties:
props = {
"Description":
Sub(
f"{self.name} created in ${{STACK_NAME}}",
STACK_NAME=define_stack_name(),
),
"Enabled":
True,
"EnableKeyRotation":
True,
"KeyUsage":
"ENCRYPT_DECRYPT",
"PendingWindowInDays":
7,
}
else:
props = import_record_properties(self.properties, Key)
if not keyisset("KeyPolicy", props):
props.update({"KeyPolicy": define_default_key_policy()})
props.update({"Metadata": metadata})
LOG.debug(props)
self.cfn_resource = Key(self.logical_name, **props)
def define_deployment_options(family, props: dict) -> None:
"""
Function to define the DeploymentConfiguration
Default is to have Rollback and CircuitBreaker on.
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
:param dict props: the troposphere.ecs.Service properties definition to update with deployment config.
"""
default = DeploymentConfiguration(
DeploymentCircuitBreaker=DeploymentCircuitBreaker(Enable=True,
Rollback=True), )
deployment_config = set_service_update_config(family)
if deployment_config:
deploy_config = DeploymentConfiguration(
MaximumPercent=int(deployment_config["MaximumPercent"]),
MinimumHealthyPercent=int(
deployment_config["MinimumHealthyPercent"]),
DeploymentCircuitBreaker=DeploymentCircuitBreaker(
Enable=True,
Rollback=keyisset("RollBack", deployment_config),
),
)
props.update({"DeploymentConfiguration": deploy_config})
else:
props.update({"DeploymentConfiguration": default})
def handle_key_settings(self, template):
"""
Method to add to the template for additional KMS key related resources.
:param troposphere.Template template:
"""
if self.parameters and keyisset("Alias", self.parameters):
alias_name = self.parameters["Alias"]
if not (alias_name.startswith("alias/")
or alias_name.startswith("aws")):
alias_name = Sub(
f"alias/${{STACK_NAME}}/{alias_name}",
STACK_NAME=define_stack_name(template),
)
elif alias_name.startswith("alias/aws") or alias_name.startswith(
"aws"):
raise ValueError(
f"Alias {alias_name} cannot start with alias/aws.")
Alias(
f"{self.logical_name}Alias",
template=template,
AliasName=alias_name,
TargetKeyId=Ref(self.cfn_resource),
Metadata=metadata,
)
def map_resource_env_vars_to_family_services(
target,
resource,
) -> None:
"""
Function to deal with the env vars to add to the family stack based on the resource
Services definition
:param tuple target:
:param ecs_composex.compose.x_resources.XResource resource:
"""
map_resource_env_vars_to_family_service_environment(target, resource)
return_values = (
{} if not keyisset("ReturnValues", target[-1]) else target[-1]["ReturnValues"]
)
for svc in target[2]:
if svc in target[0].managed_sidecars:
continue
if return_values:
extend_container_envvars(
svc.container_definition,
resource.generate_resource_service_env_vars(target, return_values),
)
else:
extend_container_envvars(
svc.container_definition, resource.generate_ref_env_var(target)
)
def init_outputs(self):
spacer = ""
if (self.properties and keyisset("Name", self.properties) and
not self.properties["Name"].startswith(r"/")) or not keyisset(
"Name", self.properties):
spacer = "/"
self.output_properties = {
SSM_PARAM_NAME: (self.logical_name, self.cfn_resource, Ref, None),
SSM_PARAM_ARN: (
f"{self.logical_name}{SSM_PARAM_ARN.title}",
self.cfn_resource,
Sub,
f"arn:${{{AWS_PARTITION}}}:ssm:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}:parameter{spacer}"
f"${{{self.cfn_resource.title}}}",
),
}
def set_services_targets_from_dict(self, settings):
"""
Deals with services set as a dict
:param settings:
:return:
"""
for service_name, service_def in self.services.items():
if service_name in settings.families and service_name not in [
f[0].name for f in self.families_targets
]:
self.families_targets.append((
settings.families[service_name],
True,
settings.families[service_name].services,
service_def["Access"]
if keyisset("Access", service_def) else {},
service_def,
))
elif service_name in settings.families and service_name in [
f[0].name for f in self.families_targets
]:
LOG.debug(
f"{self.module.res_key}.{self.name} - Family {service_name} has already been added. Skipping"
)
elif service_name in [s.name for s in settings.services]:
self.handle_families_targets_expansion_dict(
service_name, service_def, settings)
def handle_user_defined_policies(bucket: Bucket, param_key: str,
user_policies_key: str, statement: list):
"""
Function to add user defined policies
:param bucket:
:param str param_key:
:param str user_policies_key:
:param list statement:
"""
policies = bucket.parameters[param_key][user_policies_key]
policy_docs = {}
for count, policy_doc in enumerate(policies):
if keyisset("Sid", policy_doc):
name = policy_doc["Sid"]
else:
name = f"UserDefined{count}"
policy_docs[name] = policy_doc
generated_policies = generate_resource_permissions(
bucket.logical_name,
policy_docs,
Sub(f"arn:${{{AWS_PARTITION}}}:s3:::${{{bucket.cfn_resource.title}}}"),
True,
)
for policy in generated_policies.values():
statement += policy.PolicyDocument["Statement"]
def set_services_targets_scaling_from_dict(self, settings) -> None:
"""
Deals with services set as a dict
:param settings:
"""
for service_name, service_def in self.services.items():
if not keyisset("Scaling", service_def):
LOG.debug(
f"{self.module.res_key}.{self.name} - No Scaling set for {service_name}"
)
continue
if service_name in settings.families and service_name not in [
f[0].name for f in self.families_scaling
]:
self.families_scaling.append((
settings.families[service_name],
service_def["Scaling"],
))
elif service_name in settings.families and service_name in [
f[0].name for f in self.families_scaling
]:
LOG.debug(
f"{self.module.res_key}.{self.name} - Family {service_name} has already been added. Skipping"
)
elif service_name in [s.name for s in settings.services]:
self.handle_families_scaling_expansion_dict(
service_name, service_def, settings)
def generate_outputs(self):
for (
attribute_parameter,
output_definition,
) in self.output_properties.items():
output_name = f"{self.title}{attribute_parameter.title}"
value = self.set_new_resource_outputs(output_definition)
self.attributes_outputs[attribute_parameter] = {
"Name":
output_name,
"Output":
Output(output_name, Value=value),
"ImportParameter":
Parameter(
output_name,
return_value=attribute_parameter.return_value,
Type=attribute_parameter.Type,
),
"ImportValue":
GetAtt(
self.stack,
f"Outputs.{output_name}",
),
"Original":
attribute_parameter,
}
for attr in self.attributes_outputs.values():
if keyisset("Output", attr):
self.outputs.append(attr["Output"])
def get_topic_config(topic: Topic, account_id: str, resource_id: str) -> dict | None:
"""
Function to create the mapping definition for SNS topics
"""
topic_config = {TOPIC_NAME: resource_id}
client = topic.lookup_session.client("sns")
attributes_mapping = {
TOPIC_ARN: "Attributes::TopicArn",
TOPIC_KMS_KEY: "Attributes::KmsMasterKeyId",
}
try:
topic_r = client.get_topic_attributes(TopicArn=topic.arn)
attributes = attributes_to_mapping(topic_r, attributes_mapping)
if keyisset(TOPIC_KMS_KEY, attributes) and not attributes[
TOPIC_KMS_KEY
].startswith("arn:aws"):
if attributes[TOPIC_KMS_KEY].startswith("alias/aws"):
LOG.warning(
f"{topic.module.res_key}.{topic.name} - Topic uses the default AWS CMK."
)
else:
LOG.warning(
f"{topic.module.res_key}.{topic.name} - KMS Key provided is not a valid ARN."
)
del attributes[TOPIC_KMS_KEY]
topic_config.update(attributes)
return topic_config
except client.exceptions.QueueDoesNotExist:
return None
except ClientError as error:
LOG.error(error)
raise
def set_services_mount_points(family):
"""
Method to set the mount points to the Container Definition of the defined service
if the volume["volume"] is none, this is not a shared volume, which then works only
when not using Fargate (i.e. EC2 host/ ECS Anywhere)
"""
for service in family.services:
mount_points = []
if not hasattr(service.container_definition, "MountPoints"):
setattr(service.container_definition, "MountPoints", mount_points)
else:
mount_points = getattr(service.container_definition, "MountPoints")
for volume in service.volumes:
if keyisset("volume", volume):
mnt_point = MountPoint(
ContainerPath=volume["target"],
ReadOnly=volume["read_only"],
SourceVolume=volume["volume"].volume_name,
)
else:
mnt_point = If(
USE_FARGATE_CON_T,
NoValue,
MountPoint(
ContainerPath=volume["target"],
ReadOnly=volume["read_only"],
SourceVolume=NONALPHANUM.sub("", volume["target"]),
),
)
if not mount_point_exists(mount_points, mnt_point, family,
service):
mount_points.append(mnt_point)
