def logging_from_defined_region(family: ComposeFamily,
service: ComposeService) -> None:
"""
If the region for a log group is given, we just take the shortcut to granting all access
to logs in all regions, for any log group / stream.
:param family:
:param service:
:return:
"""
LOG.warning(
f"{family.name}.logging.awslogs driver "
"- When defining awslogs-region, Compose-X does not create the CW Log Group"
)
family.iam_manager.exec_role.cfn_resource.Policies.append(
Policy(
PolicyName=f"CloudWatchAccessFor{family.logical_name}",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Sid":
"AllowCloudWatchLoggingToSpecificLogGroup",
"Effect":
"Allow",
"Action":
LOGGING_ACTIONS,
"Resource":
Sub("arn:${AWS::Partition}:logs:*:${AWS::AccountId}:log-group:*"
),
}],
},
))
service.logging.log_options.update({"awslogs-create-group": True})
def set_ext_sources_ingress(self, destination_tile, security_group):
"""
Method to add ingress rules from external sources to a given Security Group (ie. ALB Security Group).
If a list of IPs is found in the config['ext_sources'] part of the network section of configs for the service,
then it will use that. If no IPv4 source is indicated, it will by default allow traffic from 0.0.0.0/0
:param str destination_tile: The name of the destination for description
:param security_group: security group (object or title string) to add the rules to
:type security_group: str or troposphere.ec2.SecurityGroup or troposphere.Ref or Troposphere.GetAtt
"""
if not self.ext_sources:
LOG.debug("No external rules defined. Skipping.")
return
for allowed_source in self.ext_sources:
if not keyisset(self.ipv4_key, allowed_source) and not keyisset(
self.ipv6_key, allowed_source):
LOG.warning(
f"No {self.ipv4_key} or {self.ipv6_key} set. Skipping")
continue
props = generate_security_group_props(allowed_source)
if props:
LOG.debug(f"Adding {allowed_source} for ingress")
self.create_ext_sources_ingress_rule(destination_tile,
allowed_source,
security_group, **props)
def deploy(settings, root_stack):
"""
Function to deploy (create or update) the stack to CFN.
:param ComposeXSettings settings:
:param ComposeXStack root_stack:
:return:
"""
validate_stack_availability(settings, root_stack)
client = settings.session.client("cloudformation")
if assert_can_create_stack(client, settings.name):
res = client.create_stack(
StackName=settings.name,
Capabilities=["CAPABILITY_IAM", "CAPABILITY_AUTO_EXPAND"],
Parameters=root_stack.render_parameters_list_cfn(),
TemplateURL=root_stack.TemplateURL,
)
LOG.info(f"Stack {settings.name} successfully deployed.")
LOG.info(res["StackId"])
return res["StackId"]
elif assert_can_update_stack(client, settings.name):
LOG.warning(f"Stack {settings.name} already exists. Updating.")
res = client.update_stack(
StackName=settings.name,
Capabilities=["CAPABILITY_IAM", "CAPABILITY_AUTO_EXPAND"],
Parameters=root_stack.render_parameters_list_cfn(),
TemplateURL=root_stack.TemplateURL,
)
LOG.info(f"Stack {settings.name} successfully updating.")
LOG.info(res["StackId"])
return res["StackId"]
return None
def get_bucket_config(bucket: Bucket, resource_id: str) -> dict:
"""
:param ecs_composex.s3.s3_bucket.Bucket bucket:
:param str resource_id:
"""
bucket_config = {
S3_BUCKET_NAME: resource_id,
S3_BUCKET_ARN: bucket.arn,
}
client = bucket.lookup_session.client("s3")
try:
encryption_r = client.get_bucket_encryption(Bucket=resource_id)
encryption_attributes = attributes_to_mapping(
encryption_r, CONTROL_CLOUD_ATTR_MAPPING)
if keyisset(
CONTROL_CLOUD_ATTR_MAPPING[S3_BUCKET_KMS_KEY],
encryption_attributes,
):
bucket_config[S3_BUCKET_KMS_KEY] = encryption_attributes[
S3_BUCKET_KMS_KEY]
except ClientError as error:
if (not error.response["Error"]["Code"]
== "ServerSideEncryptionConfigurationNotFoundError"):
raise
LOG.warning(error.response["Error"]["Message"])
return bucket_config
def process_stacks(root_stack, settings, is_root=True):
"""
Function to go through all stacks of a given template and update the template
It will recursively render sub stacks defined.
:param root_stack: the root template to iterate over the resources.
:type root_stack: ecs_composex.common.stacks.ComposeXStack
:param settings: The settings for execution
:type settings: ecs_composex.common.settings.ComposeXSettings
:param bool is_root: Allows to know whether the stack is parent stack
"""
for resource_name, resource in root_stack.stack_template.resources.items():
if isinstance(resource, ComposeXStack) or issubclass(
type(resource), ComposeXStack):
LOG.debug(resource)
LOG.debug(resource.title)
process_stacks(resource, settings, is_root=False)
if is_root:
resource.Parameters.update(
{ROOT_STACK_NAME_T: Ref(AWS_STACK_NAME)})
else:
resource.Parameters.update(
cfn_conditions.pass_root_stack_name())
elif isinstance(resource, Stack):
LOG.warning(resource_name)
LOG.warning(resource)
root_stack.render(settings)
def add_parameter_to_family_stack(
self, family, settings: ComposeXSettings,
parameter: Union[str, Parameter]) -> dict:
if self.stack == family.stack:
LOG.warning(
"Cannot add parameter to resource",
f"{self.name}",
"because it is in the same stack as family",
"{family.name}",
)
return self
if (isinstance(parameter, str)
and parameter in self.property_to_parameter_mapping.keys()):
the_parameter = self.property_to_parameter_mapping[parameter]
elif (isinstance(parameter, Parameter)
and parameter in self.property_to_parameter_mapping.values()):
the_parameter = parameter
else:
raise TypeError("parameter must be one of", str, Parameter, "got",
type(parameter))
if self.mappings and self.lookup:
add_update_mapping(
family.template,
self.module.mapping_key,
settings.mappings[self.module.mapping_key],
)
return self.attributes_outputs[the_parameter]
param_id = self.attributes_outputs[the_parameter]
add_parameters(family.template, [param_id["ImportParameter"]])
family.stack.Parameters.update(
{param_id["ImportParameter"].title: param_id["ImportValue"]})
return param_id
def handle_service_scaling(alarm, alarms_stack):
"""
Function to create the scaling steps for defined services
:param ecs_composex.alarms.alarms_stack.Alarm alarm:
:param ecs_composex.common.stacks.ComposeXStack alarms_stack:
"""
for target in alarm.families_scaling:
if SERVICE_SCALING_TARGET not in target[0].template.resources:
LOG.warning(
f"No Scalable target defined for {target[0].name}."
" You need to define `scaling.scaling_range` in x-configs first. No scaling applied"
)
return
scaling_out_policy = generate_alarm_scaling_out_policy(
target[0].logical_name,
target[0].template,
target[1],
scaling_source=alarm.logical_name,
)
scaling_in_policy = reset_to_zero_policy(
target[0].logical_name,
target[0].template,
target[1],
scaling_source=alarm.logical_name,
)
add_service_actions(alarm, alarms_stack, target, scaling_in_policy,
scaling_out_policy)
def set_listeners(self, template):
"""
Method to define the listeners
:return:
"""
if not keyisset("Listeners", self.definition):
raise KeyError(
f"You must define at least one listener for LB {self.name}")
ports = [listener["Port"] for listener in self.definition["Listeners"]]
validate_listeners_duplicates(self.name, ports)
for listener_def in self.definition["Listeners"]:
if keyisset("Targets", listener_def):
for target in listener_def["Targets"]:
if target["name"] not in [
svc["name"] for svc in self.services
]:
listener_def["Targets"].remove(target)
if keyisset("Targets", listener_def) or keyisset(
"DefaultActions", listener_def):
new_listener = template.add_resource(
ComposeListener(self, listener_def))
self.listeners.append(new_listener)
else:
LOG.warning(
f"{self.module.res_key}.{self.name} - "
f"Listener {listener_def['Port']} has no action or service. Not used."
)
def create_bucket(bucket_name, session):
"""
Function that checks if the S3 bucket exists and if not attempts to create it.
:param bucket_name: name of the s3 bucket
:type bucket_name: str
:param session: boto3 session to use if wanted to override settings.
:type session: boto3.session.Session
:returns: True/False, Returns whether the bucket exists or not for upload
:rtype: bool
"""
client = session.client("s3")
region = session.region_name
location = {"LocationConstraint": region}
try:
client.create_bucket(
ACL="private",
Bucket=bucket_name,
ObjectLockEnabledForBucket=True,
CreateBucketConfiguration=location,
)
LOG.info(f"Bucket {bucket_name} successfully created.")
except client.exceptions.BucketAlreadyExists:
LOG.warning(f"Bucket {bucket_name} already exists.")
except client.exceptions.BucketAlreadyOwnedByYou:
LOG.info(f"You already own the bucket {bucket_name}")
except ClientError as error:
LOG.error("Error whilst creating the bucket")
LOG.error(error)
raise
def define_default_actions(self, template):
"""
If DefaultTarget is set it will set it if not a service, otherwise at the service level.
If not defined, and there is more than one service, it will fail.
If not defined and there is only one service defined, it will skip
"""
if not self.default_actions and not self.services:
warnings.warn(
f"{self.name} - There are no actions defined or services for listener {self.title}. Skipping"
)
return
if self.default_actions:
handle_default_actions(self)
elif not self.default_actions and self.services and len(
self.services) == 1:
LOG.info(
f"{self.title} has no defined DefaultActions and only 1 service. Default all to service."
)
self.DefaultActions = define_actions(self, self.services[0])
elif not self.default_actions and self.services and len(
self.services) > 1:
LOG.warning(
f"{self.title} - "
"No default actions defined and more than one service defined. "
"If one of the access path is / it will be used as default")
rules = handle_non_default_services(self, self.services)
for rule in rules:
template.add_resource(rule)
else:
raise ValueError(
f"Failed to determine any default action for {self.title}")
def sort_alb_ingress(self, settings, stack_template):
"""
Method to handle Ingress to ALB
"""
if (not self.parameters or
(self.parameters and not keyisset("Ingress", self.parameters))
or self.is_nlb()):
LOG.warning(
"You defined ingress rules for a NLB. This is invalid. Define ingress rules at the service level."
)
return
elif not self.parameters or (self.parameters and
not keyisset("Ingress", self.parameters)):
LOG.warning(
f"You did not define any Ingress rules for ALB {self.name}.")
return
ports = [listener["Port"] for listener in self.definition["Listeners"]]
ports = set_service_ports(ports)
self.ingress = Ingress(self.parameters["Ingress"], ports)
if self.ingress and self.is_alb():
self.ingress.set_aws_sources_ingress(settings, self.logical_name,
GetAtt(self.lb_sg, "GroupId"))
self.ingress.set_ext_sources_ingress(self.logical_name,
GetAtt(self.lb_sg, "GroupId"))
self.ingress.associate_aws_ingress_rules(stack_template)
self.ingress.associate_ext_ingress_rules(stack_template)
def handle_certificates(self, settings, listener_stack):
"""
Method to handle certificates
:param ecs_composex.common.settings.ComposeXSettings settings:
:param listener_stack: The stack that has the listener as resource
:return:
"""
if not keyisset("Certificates", self.definition):
LOG.warning(f"No certificates defined for Listener {self.name}")
return
valid_sources = [
("x-acm", str, import_new_acm_certs),
("Arn", str, add_acm_certs_arn),
("CertificateArn", str, add_acm_certs_arn),
]
for cert_def in self.definition["Certificates"]:
if isinstance(cert_def, dict):
cert_source = list(cert_def.keys())[0]
source_value = cert_def[cert_source]
for src_type in valid_sources:
if (src_type[0] == cert_source
and isinstance(cert_source, src_type[1])
and src_type[2]):
src_type[2](self, source_value, settings,
listener_stack)
def define_names_from_lookup(self, session):
"""
Method to Lookup the secret based on its tags.
"""
lookup_info = self.definition[self.x_key]["Lookup"]
if keyisset("Name", self.definition[self.x_key]):
lookup_info["Name"] = self.definition[self.x_key]["Name"]
secret_config = lookup_secret_config(self.logical_name, lookup_info, session)
self.aws_name = get_secret_name_from_arn(secret_config[self.logical_name])
self.arn = secret_config[self.logical_name]
self.iam_arn = secret_config[self.logical_name]
if keyisset("KmsKeyId", secret_config) and not secret_config[
"KmsKeyId"
].startswith("alias"):
self.kms_key = secret_config["KmsKeyId"]
elif keyisset("KmsKeyId", secret_config) and secret_config[
"KmsKeyId"
].startswith("alias"):
LOG.warning(
f"secrets.{self.name} - The KMS Key retrieved is a KMS Key Alias, not importing."
)
self.mapping = {
self.map_arn_name: secret_config[self.logical_name],
self.map_name_name: secret_config[self.map_name_name],
}
if self.kms_key:
self.mapping[self.map_kms_name] = self.kms_key
self.kms_key_arn = FindInMap(
self.map_name, self.logical_name, self.map_kms_name
)
self.arn = FindInMap(self.map_name, self.logical_name, self.map_arn_name)
self.ecs_secret = [EcsSecret(Name=self.name, ValueFrom=self.arn)]
def add_json_keys(self):
"""
Add secrets definitions based on JSON secret keys
"""
if not keyisset(self.json_keys_key, self.definition[self.x_key]):
return
unfiltered_secrets = self.definition[self.x_key][self.json_keys_key]
filtered_secrets = [
dict(y) for y in {tuple(x.items()) for x in unfiltered_secrets}
]
old_secrets = deepcopy(self.ecs_secret)
secrets_to_map = {}
self.ecs_secret = []
for secret_json_key in filtered_secrets:
secret_key = secret_json_key["SecretKey"]
secret_name = define_env_var_name(secret_json_key)
if secret_name not in secrets_to_map:
secrets_to_map[secret_name] = self.define_secret(
secret_name, secret_key
)
else:
LOG.warning(
f"secrets.{self.name} - Secret VarName {secret_name} already defined. Overriding value"
)
secrets_to_map[secret_name] = self.define_secret(
secret_name, secret_key
)
self.ecs_secret = [
_defined_secret for _defined_secret in secrets_to_map.values()
]
if not self.ecs_secret:
self.ecs_secret = old_secrets
def merge_family_network_setting(family, key: str, definition: dict,
network: dict, network_config: dict) -> None:
"""
Merges a network setting (key) and its definition (definition) with new definition (network) into network_config
If the key is x-cloudmap, and is unset, set to value. If another service of the family comes in, comes second.
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
:param str key:
:param dict definition:
:param dict network:
:param dict network_config:
:return:
"""
if keyisset(key, network) and key == Ingress.master_key:
handle_ingress_rules(network[key], network_config[key])
elif keyisset(key, network) and key == CLOUDMAP_KEY:
if definition:
LOG.warning(
family.name,
f"x-network.{CLOUDMAP_KEY}",
"is already set to",
definition,
)
else:
network_config[CLOUDMAP_KEY] = network[CLOUDMAP_KEY]
def __init__(self, family: ComposeFamily):
"""
Initialize network settings for the family ServiceConfig
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
"""
self.family = family
self._network_mode = "awsvpc"
if family.service_compute.launch_type == "EXTERNAL":
LOG.warning(
f"{family.name} - External mode cannot use awsvpc mode. Falling back to bridge"
)
self.network_mode = "bridge"
self.ports = []
self.networks = {}
self.merge_services_ports()
self.merge_networks()
self.definition = merge_family_services_networking(family)
self.ingress_from_self = False
if any([svc.expose_ports for svc in family.services]):
self.ingress_from_self = True
LOG.info(
f"{family.name} - services have export ports, allowing internal ingress"
)
self._security_group = None
self.extra_security_groups = []
self._subnets = Ref(APP_SUBNETS)
self.cloudmap_config = (merge_cloudmap_settings(family, self.ports)
if self.ports else {})
self.ingress = Ingress(self.definition[Ingress.master_key], self.ports)
self.ingress_from_self = keyisset(self.self_key, self.definition)
def __init__(self, name, definition):
self.name = name
self.volume_name = name
self.autogenerated = False
self.definition = deepcopy(definition)
self.is_shared = False
self.services = []
self.parameters = {}
self.device = None
self.cfn_volume = None
self.efs_definition = {}
self.use = {}
self.lookup = {}
self.type = "volume"
self.driver = "local"
self.external = False
self.efs_definition = evaluate_plugin_efs_properties(
self.definition, self.driver_opts_key)
if self.efs_definition:
LOG.info(
f"volumes.{self.name} - Identified properties as defined by Docker Plugin"
)
self.type = "bind"
self.driver = "nfs"
elif (keyisset("external", self.definition)
and keyisset("name", self.definition)
and FS_REGEXP.match(self.definition["name"])):
LOG.warning(f"volumes.{self.name} - Identified a EFS to use")
self.efs_definition = {"Use": self.definition["name"]}
self.use = self.definition["name"]
else:
self.import_volume_from_definition()
def set_compute_resources(service, deployment):
"""
Function to analyze the Docker Compose deploy attribute and set settings accordingly.
deployment keys: replicas, mode, resources
:param dict deployment: definition['deploy']
"""
if not keyisset("resources", deployment):
return
resources = deployment["resources"]
cpu_alloc = 0
cpu_resa = 0
cpus = "cpus"
memory = "memory"
resa = "reservations"
alloc = "limits"
if keyisset(alloc, resources):
cpu_alloc = (int(float(resources[alloc][cpus]) *
1024) if keyisset(cpus, resources[alloc]) else 0)
service.mem_alloc = (set_memory_to_mb(resources[alloc][memory].strip())
if keyisset(memory, resources[alloc]) else 0)
if keyisset(resa, resources):
cpu_resa = (int(float(resources[resa][cpus]) *
1024) if keyisset(cpus, resources[resa]) else 0)
service.mem_resa = (set_memory_to_mb(resources[resa][memory].strip())
if keyisset(memory, resources[resa]) else 0)
service.cpu_amount = (max(cpu_resa, cpu_alloc) if
(cpu_resa or cpu_alloc) else NoValue)
if isinstance(service.cpu_amount, int) and service.cpu_amount > 4096:
LOG.warning(
f"{service.name} - Fargate does not support more than 4 vCPU.")
def handle_volume_str_config(service: ComposeService, config: str,
volumes: list):
"""
Function to return the volume configuration (long)
:param ComposeService service:
:param str config:
:param list volumes:
"""
volume_config = {"read_only": False}
path_finder = re.compile(
r"(?:(?P<source>[\S][^:]+):)?(?P<target>/[^:\n]+)(?::(?P<mode>ro|rw))?"
)
path_match = path_finder.match(config)
if not path_match or (path_match and not path_match.group("target")):
raise ValueError(
f"Volume syntax {config} is invalid. Must follow the pattern",
path_finder.pattern,
)
else:
volume_config["target"] = path_match.group("target")
if path_match.group("source"):
volume_config["source"] = path_match.group("source")
else:
LOG.warning(
f"No source defined with {config}. Creating docker volume")
new_volume = ComposeVolume(str(uuid4().hex)[:6], {})
new_volume.autogenerated = True
volumes.append(new_volume)
volume_config["source"] = new_volume.name
volume_config["volume"] = new_volume
if path_match.group("mode") and path_match.group("mode") == "ro":
volume_config["read_only"] = True
match_volumes_services_config(service, volume_config, volumes)
def merge_capacity_providers(service_compute):
"""
Merge capacity providers set on the services of the task service_compute.family if service is not sidecar
"""
task_config = {}
for svc in service_compute.family.ordered_services:
if not svc.capacity_provider_strategy or svc.is_aws_sidecar:
continue
for provider in svc.capacity_provider_strategy:
if provider["CapacityProvider"] not in task_config.keys():
name = provider["CapacityProvider"]
task_config[name] = {
"Base": [],
"Weight": [],
"CapacityProvider": name,
}
task_config[name]["Base"].append(
set_else_none("Base", provider, alt_value=0)
)
task_config[name]["Weight"].append(
set_else_none("Weight", provider, alt_value=0)
)
for count, provider in enumerate(task_config.values()):
if count == 0:
provider["Base"] = int(max(provider["Base"]))
elif count > 0 and keypresent("Base", provider):
del provider["Base"]
LOG.warning(
f"{service_compute.family.name}.x-ecs Only one capacity provider can have a base value. "
f"Deleting Base for {provider['CapacityProvider']}"
)
provider["Weight"] = int(max(provider["Weight"]))
service_compute.ecs_capacity_providers = [
CapacityProviderStrategyItem(**config) for config in task_config.values()
]
def fix_nlb_settings(props):
"""
Function to automatically adjust/correct settings for NLB to avoid users cringe on fails
:param dict props:
"""
network_modes = ["TCP", "UDP", "TCP_UDP"]
if (keyisset("HealthCheckProtocol", props)
and not props["HealthCheckProtocol"] in network_modes):
return
if keyisset("HealthCheckTimeoutSeconds", props):
LOG.warning("With NLB you cannot set intervals. Resetting")
props["HealthCheckTimeoutSeconds"] = Ref(AWS_NO_VALUE)
if (keyisset("HealthCheckIntervalSeconds", props)
and not (props["HealthCheckIntervalSeconds"] == 10
or props["HealthCheckIntervalSeconds"] == 30)
and not isinstance(props["HealthCheckIntervalSeconds"], Ref)):
right_value = min(
[10, 30],
key=lambda x: abs(x - props["HealthCheckIntervalSeconds"]))
LOG.warning(
f"Set to {props['HealthCheckIntervalSeconds']} - "
f"The only intervals value valid for NLB are 10 and 30. Closest value is {right_value}"
)
props["HealthCheckIntervalSeconds"] = right_value
validate_tcp_health_counts(props)
def main():
"""
Main entry point for CLI
:return: status code
"""
parser = main_parser()
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
args = parser.parse_args()
LOG.debug(args)
settings = ComposeXSettings(**vars(args))
settings.set_bucket_name_from_account_id()
settings.set_azs_from_api()
LOG.debug(settings)
if settings.deploy and not settings.upload:
LOG.warning(
"You must update the templates in order to deploy. We won't be deploying."
)
settings.deploy = False
root_stack = generate_full_template(settings)
process_stacks(root_stack, settings)
if settings.deploy:
deploy(settings, root_stack)
return 0
def set_azs_from_vpc_import(self, subnets, session=None):
"""
Function to get the list of AZs for a given set of subnets
:param dict subnets:
:param session: The Session used to find the EC2 subnets (useful for lookup).
:return:
"""
if session is None:
client = self.lookup_session.client("ec2")
else:
client = session.client("ec2")
for subnet_name, subnet_definition in subnets.items():
if not isinstance(subnet_definition, list):
continue
for subnet_param in self.subnets_parameters:
if subnet_param.title == subnet_name:
subnets_param = subnet_param
break
else:
raise KeyError(
f"x-vpc.set_azs_from_vpc_import - No parameter defined for {subnet_name}"
)
try:
subnets_r = client.describe_subnets(
SubnetIds=subnet_definition)["Subnets"]
azs = [subnet["AvailabilityZone"] for subnet in subnets_r]
self.mappings[subnet_name]["Azs"] = azs
self.azs[subnets_param] = azs
except ClientError:
LOG.warning(
"Could not define the AZs based on the imported subnets")
def define_cluster(root_stack, cluster_def):
"""
Function to create the cluster from provided properties.
:param dict cluster_def:
:param ecs_composex.common.stacks.ComposeXStack root_stack:
:return: cluster
:rtype: troposphere.ecs.Cluster
"""
cluster_params = {}
if not keyisset("Properties", cluster_def):
return get_default_cluster_config()
props = cluster_def["Properties"]
if keyisset("ClusterName", props):
root_stack.Parameters.update({CLUSTER_NAME_T: props["ClusterName"]})
if not keyisset("CapacityProviders", props):
LOG.warning("No capacity providers defined. Setting it to default.")
cluster_params["CapacityProviders"] = DEFAULT_PROVIDERS
else:
cluster_params["CapacityProviders"] = props["CapacityProviders"]
if not keyisset("DefaultCapacityProviderStrategy", props):
LOG.warning("No Default Strategy set. Setting to default.")
cluster_params["DefaultCapacityProviderStrategy"] = DEFAULT_STRATEGY
else:
cluster_params[
"DefaultCapacityProviderStrategy"] = import_capacity_strategy(
props["DefaultCapacityProviderStrategy"])
cluster_params["Metadata"] = metadata
cluster_params["ClusterName"] = If(GENERATED_CLUSTER_NAME_CON_T,
Ref(AWS_STACK_NAME),
Ref(CLUSTER_NAME_T))
cluster = Cluster(CLUSTER_T, **cluster_params)
return cluster
def __init__(self, definition, ports):
"""
Initialize network settings for the family ServiceConfig
"""
self.definition = deepcopy(definition)
self.aws_sources = (self.definition[self.aws_sources_key] if keyisset(
self.aws_sources_key, self.definition) else [])
self.ext_sources = (self.definition[self.ext_sources_key] if keyisset(
self.ext_sources_key, self.definition) else [])
self.ext_sources = []
if keyisset(self.ext_sources_key, self.definition):
cidrs = []
for ext_source in self.definition[self.ext_sources_key]:
source_cidr = set_else_none(
self.ipv4_key,
ext_source,
set_else_none(self.ipv6_key, ext_source, None),
)
if source_cidr and source_cidr not in cidrs:
self.ext_sources.append(ext_source)
else:
LOG.warning(
f"Ingress source {source_cidr} already defined in a previous Ingress rule."
)
self.services = (self.definition[self.services_key] if keyisset(
self.services_key, self.definition) else [])
self.ports = ports
self.aws_ingress_rules = []
self.ext_ingress_rules = []
self.to_self_rules = []
def lookup_ecs_cluster(session, cluster_lookup):
"""
Function to find the ECS Cluster.
:param boto3.session.Session session: Boto3 session to make API calls.
:param cluster_lookup: Cluster lookup definition.
:return:
"""
if not isinstance(cluster_lookup, str):
raise TypeError("The value for Lookup must be", str, "Got",
type(cluster_lookup))
client = session.client("ecs")
try:
cluster_r = client.describe_clusters(clusters=[cluster_lookup])
if not keyisset("clusters", cluster_r):
LOG.warning(
f"No cluster named {cluster_lookup} found. Creating one with default settings"
)
return get_default_cluster_config()
elif (keyisset("clusters", cluster_r)
and cluster_r["clusters"][0]["clusterName"] == cluster_lookup):
LOG.info(
f"Found ECS Cluster {cluster_lookup}. Setting {CLUSTER_NAME_T} accordingly."
)
return cluster_r["clusters"][0]["clusterName"]
except ClientError as error:
LOG.error(error)
raise
def add_vpc_to_root(root_stack, settings):
"""
Function to figure whether to create the VPC Stack and if not, set the parameters.
:param root_stack:
:param settings:
:return: vpc_stack
:rtype: VpcStack
"""
vpc_stack = None
vpc_xkey = f"{X_KEY}{RES_KEY}"
if keyisset(vpc_xkey, settings.compose_content):
if keyisset("Lookup", settings.compose_content[vpc_xkey]):
x_settings = lookup_x_vpc_settings(
settings.session, settings.compose_content[vpc_xkey]["Lookup"])
apply_vpc_settings(x_settings, root_stack)
elif keyisset("Use", settings.compose_content[vpc_xkey]):
x_settings = import_vpc_settings(
settings.compose_content[vpc_xkey]["Use"])
apply_vpc_settings(x_settings, root_stack)
else:
if keyisset("Create",
settings.compose_content[vpc_xkey]) and keyisset(
"Lookup", settings.compose_content[vpc_xkey]):
LOG.warning("We have both Create and Lookup set for x-vpc."
"Creating a new VPC")
vpc_stack = create_new_vpc(vpc_xkey, settings)
else:
LOG.info(f"No {vpc_xkey} detected. Creating a new VPC.")
vpc_stack = create_new_vpc(vpc_xkey, settings, default=True)
if isinstance(vpc_stack, VpcStack):
root_stack.stack_template.add_resource(vpc_stack)
return vpc_stack
def validate(self, settings):
"""
Method to validate the CloudFormation template, either via URL once uploaded to S3 or via TemplateBody
"""
try:
if not settings.no_upload and self.url:
validate_wrapper(settings.session, url=self.url)
elif settings.no_upload or not self.url:
if not self.file_path:
self.write(settings)
LOG.debug(f"No upload - Validating template body - {self.file_path}")
if len(self.body) >= 51200:
LOG.warning(
f"Template body for {self.file_name} is too big for local validation."
" No upload is True, so skipping."
)
else:
validate_wrapper(settings.session, body=self.body)
LOG.debug(f"Template {self.file_name} was validated successfully by CFN")
except ClientError as error:
LOG.error(error)
with open(f"/tmp/{settings.name}.{settings.format}", "w") as failed_file_fd:
failed_file_fd.write(self.body)
LOG.error(
f"Failed validation template written at /tmp/{settings.name}.{settings.format}"
)
raise
def get_topic_config(topic: Topic, account_id: str, resource_id: str) -> dict | None:
"""
Function to create the mapping definition for SNS topics
"""
topic_config = {TOPIC_NAME: resource_id}
client = topic.lookup_session.client("sns")
attributes_mapping = {
TOPIC_ARN: "Attributes::TopicArn",
TOPIC_KMS_KEY: "Attributes::KmsMasterKeyId",
}
try:
topic_r = client.get_topic_attributes(TopicArn=topic.arn)
attributes = attributes_to_mapping(topic_r, attributes_mapping)
if keyisset(TOPIC_KMS_KEY, attributes) and not attributes[
TOPIC_KMS_KEY
].startswith("arn:aws"):
if attributes[TOPIC_KMS_KEY].startswith("alias/aws"):
LOG.warning(
f"{topic.module.res_key}.{topic.name} - Topic uses the default AWS CMK."
)
else:
LOG.warning(
f"{topic.module.res_key}.{topic.name} - KMS Key provided is not a valid ARN."
)
del attributes[TOPIC_KMS_KEY]
topic_config.update(attributes)
return topic_config
except client.exceptions.QueueDoesNotExist:
return None
except ClientError as error:
LOG.error(error)
raise
def main():
"""
Main entry point for CLI
:return: status code
"""
parser = main_parser()
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
args = parser.parse_args()
LOG.debug(args)
settings = ComposeXSettings(**vars(args))
settings.set_bucket_name_from_account_id()
LOG.debug(settings)
if settings.deploy and not settings.upload:
LOG.warning(
"You must update the templates in order to deploy. We won't be deploying."
)
settings.deploy = False
evaluate_docker_configs(settings)
scan_results = evaluate_ecr_configs(settings)
if scan_results and not settings.ignore_ecr_findings:
warnings.warn("SCAN Images failed for instructed images. Failure")
return 1
root_stack = generate_full_template(settings)
process_stacks(root_stack, settings)
if settings.deploy:
deploy(settings, root_stack)
elif settings.plan:
plan(settings, root_stack)
return 0
