def handle_firelens_options(service: ComposeService, logging_def: dict, set_cw_default: bool = False) -> LogConfiguration: default_cloudwatch_options = { "region": Region, "auto_create_group": True, "log_group_name": service.logical_name, "log_stream_prefix": service.service_name, } if set_cw_default: options = set_else_none("options", logging_def, alt_value=default_cloudwatch_options) else: options = set_else_none("options", logging_def, alt_value=NoValue) config_name_map = { "delivery_stream": "kinesis_firehose", "log_group_name": "cloudwatch", "stream": "kinesis_streams", "bucket": "s3", } for key, value in config_name_map.items(): if keyisset(key, options): options.update({"Name": value}) break return LogConfiguration(LogDriver="awsfirelens", Options=options)
def __init__(self, definition, ports): """ Initialize network settings for the family ServiceConfig """ self.definition = deepcopy(definition) self.aws_sources = (self.definition[self.aws_sources_key] if keyisset( self.aws_sources_key, self.definition) else []) self.ext_sources = (self.definition[self.ext_sources_key] if keyisset( self.ext_sources_key, self.definition) else []) self.ext_sources = [] if keyisset(self.ext_sources_key, self.definition): cidrs = [] for ext_source in self.definition[self.ext_sources_key]: source_cidr = set_else_none( self.ipv4_key, ext_source, set_else_none(self.ipv6_key, ext_source, None), ) if source_cidr and source_cidr not in cidrs: self.ext_sources.append(ext_source) else: LOG.warning( f"Ingress source {source_cidr} already defined in a previous Ingress rule." ) self.services = (self.definition[self.services_key] if keyisset( self.services_key, self.definition) else []) self.ports = ports self.aws_ingress_rules = [] self.ext_ingress_rules = [] self.to_self_rules = []
def merge_capacity_providers(service_compute): """ Merge capacity providers set on the services of the task service_compute.family if service is not sidecar """ task_config = {} for svc in service_compute.family.ordered_services: if not svc.capacity_provider_strategy or svc.is_aws_sidecar: continue for provider in svc.capacity_provider_strategy: if provider["CapacityProvider"] not in task_config.keys(): name = provider["CapacityProvider"] task_config[name] = { "Base": [], "Weight": [], "CapacityProvider": name, } task_config[name]["Base"].append( set_else_none("Base", provider, alt_value=0) ) task_config[name]["Weight"].append( set_else_none("Weight", provider, alt_value=0) ) for count, provider in enumerate(task_config.values()): if count == 0: provider["Base"] = int(max(provider["Base"])) elif count > 0 and keypresent("Base", provider): del provider["Base"] LOG.warning( f"{service_compute.family.name}.x-ecs Only one capacity provider can have a base value. " f"Deleting Base for {provider['CapacityProvider']}" ) provider["Weight"] = int(max(provider["Weight"])) service_compute.ecs_capacity_providers = [ CapacityProviderStrategyItem(**config) for config in task_config.values() ]
def add_dependant_ingress_rules(dst_family: ComposeFamily, dst_family_sg_param: Parameter, src_family: ComposeFamily) -> None: for port in dst_family.service_networking.ports: target_port = set_else_none("published", port, alt_value=set_else_none( "target", port, None)) if target_port is None: raise ValueError( "Wrong port definition value for security group ingress", port) common_args = { "FromPort": target_port, "ToPort": target_port, "IpProtocol": port["protocol"], "SourceSecurityGroupOwnerId": Ref(AWS_ACCOUNT_ID), "Description": Sub(f"From ${{{SERVICE_NAME_T}}} to {dst_family.stack.title} on port {target_port}" ), } src_family.template.add_resource( SecurityGroupIngress( f"From{src_family.logical_name}To{dst_family.stack.title}On{target_port}", SourceSecurityGroupId=GetAtt( src_family.service_networking.security_group, "GroupId"), GroupId=Ref(dst_family_sg_param), **common_args, ))
def init_update_policies(self): for service in self.family.services: managed_policies = set_else_none("ManagedPolicyArns", service.x_iam, []) if managed_policies: self.add_new_managed_policies(managed_policies) permissions_boundary = set_else_none("PermissionsBoundary", service.x_iam, False) if permissions_boundary and not self.permissions_boundary: self.permissions_boundary = permissions_boundary add_role_boundaries(self.exec_role.cfn_resource, self.permissions_boundary) add_role_boundaries(self.task_role.cfn_resource, self.permissions_boundary) elif (permissions_boundary and self.permissions_boundary and permissions_boundary != self.permissions_boundary): print( f"{self.family} - Permissions boundary already set: {self.permissions_boundary}." f" Cannot add {permissions_boundary}" " as PermissionsBoundary is single string") policies = set_else_none("Policies", service.x_iam, []) if policies: add_policies_from_x_iam(self.policies, policies) setattr(self.task_role.cfn_resource, "Policies", self.policies)
def add_self_ingress(self) -> None: """ Method to allow communications internally to the group on set ports """ if (not self.family.template or not self.family.ecs_service or not self.ingress_from_self): return for port in self.ports: target_port = set_else_none("published", port, alt_value=set_else_none( "target", port, None)) if target_port is None: raise ValueError( "Wrong port definition value for security group ingress", port) self.ingress.to_self_rules.append( SecurityGroupIngress( f"AllowingInterCommunicationPort{target_port}{port['protocol']}", template=self.family.template, FromPort=target_port, ToPort=target_port, IpProtocol=port["protocol"], GroupId=GetAtt( self.family.service_networking.security_group, "GroupId"), SourceSecurityGroupId=GetAtt( self.family.service_networking.security_group, "GroupId"), SourceSecurityGroupOwnerId=Ref(AWS_ACCOUNT_ID), Description=Sub( f"Allowing traffic internally on port {target_port}"), ))
def cpu_amount(self) -> Union[int, Ref]: if not self._cpu_amount or self.container_start_condition in [ "SUCCESS", "COMPLETE", ]: return NoValue alloc = "limits" resa = "reservations" resource = "cpus" _set_limit = float( set_else_none( resource, set_else_none(alloc, self.resources, alt_value={}), alt_value=0, )) _set_resa = float( set_else_none( resource, set_else_none(resa, self.resources, alt_value={}), alt_value=0, )) to_set = float(max([_set_limit, _set_resa])) if to_set: return int(float(to_set * 1024)) return NoValue
def ephemeral_storage(self): storage_key = "ecs.ephemeral.storage" storage_value = set_else_none(storage_key, set_else_none("labels", self.deploy, alt_value={}), alt_value=0) if isinstance(storage_value, (int, float)): ephemeral_storage = int(storage_value) elif isinstance(storage_value, str): ephemeral_storage = int(set_memory_to_mb(storage_value) / 1024) else: raise TypeError( f"The value for {storage_key} is of type", type(storage_value), "Expected one of", [int, float, str], ) if ephemeral_storage <= 21: return 0 elif ephemeral_storage > 200: return 200 else: LOG.info(f"{self.name} - {storage_key} set to {ephemeral_storage}") return int(ephemeral_storage)
def launch_type(self) -> Union[str, None]: compute_key = "ecs.compute.platform" return set_else_none( compute_key, set_else_none("labels", self.deploy, alt_value={}), alt_value=None, )
def add_parameters_metadata(template, parameter): """ Simple function that will auto-add AWS::CloudFormation::Interface to the template if the parameter has a group and labels defined :param template: :param parameter: :return: """ if not hasattr(template, "metadata"): metadata = {} else: metadata = getattr(template, "metadata") interface_metadata = set_else_none("AWS::CloudFormation::Interface", metadata, {}) if not interface_metadata: metadata["AWS::CloudFormation::Interface"] = interface_metadata if parameter.group_label: add_parameter_to_group_label(interface_metadata, parameter) if parameter.label: labels = set_else_none("ParameterLabels", interface_metadata, {}, eval_bool=True) if not labels: interface_metadata["ParameterLabels"] = labels labels[parameter.title] = {"default": parameter.label} else: labels.update({parameter.title: {"default": parameter.label}})
def __init__(self, service, definition: dict, family, settings: ComposeXSettings): self.service = service self._definition = copy.deepcopy(definition) self.family = family self.source_file = set_else_none("SourceFile", self.definition) self._parser_files = set_else_none("ParserFiles", self.definition, alt_value=[]) self._env_vars = set_else_none("EnvironmentVariables", self.definition) self.managed_destinations = [] self.extra_env_vars = set_else_none( "EnvironmentVariables", self.definition, alt_value={} ) if keyisset("ComposeXManagedAwsDestinations", self.definition): for destination_definition in self.definition[ "ComposeXManagedAwsDestinations" ]: if keyisset("log_group_name", destination_definition): self.managed_destinations.append( FireLensCloudWatchManagedDestination( destination_definition, self, settings ) ) elif keyisset("delivery_stream", destination_definition): self.managed_destinations.append( FireLensFirehoseManagedDestination( destination_definition, self, settings ) ) else: LOG.error("Invalid definition for ComposeXManagedAwsDestinations") LOG.error(destination_definition)
def set_aws_sources_ingress(self, settings, destination_title, sg_ref) -> None: """ Method to define AWS Sources ingresses :param settings: :param destination_title: :param sg_ref: """ for source in self.aws_sources: for port in self.ports: if (keyisset("Ports", source) and port["published"] not in source["Ports"]): continue target_port = set_else_none("published", port, alt_value=set_else_none( "target", port, None)) if target_port is None: raise ValueError( "Wrong port definition value for security group ingress", port) common_args = { "FromPort": target_port, "ToPort": target_port, "IpProtocol": port["protocol"], "GroupId": sg_ref, } if source["Type"] == "SecurityGroup": if keyisset("Id", source): sg_id = source["Id"] elif keyisset("Lookup", source): sg_id = lookup_security_group(settings, source["Lookup"]) else: raise KeyError( "Information missing to identify the SecurityGroup. Requires either Id or Lookup" ) common_args.update({ "Description": Sub(f"From {sg_id} to {destination_title} on port {target_port}" ) }) self.aws_ingress_rules.append( SecurityGroupIngress( f"From{NONALPHANUM.sub('', sg_id)}ToServiceOn{target_port}", SourceSecurityGroupId=sg_id, SourceSecurityGroupOwnerId=Ref(AWS_ACCOUNT_ID), **common_args, )) elif source["Type"] == "PrefixList": self.aws_ingress_rules.append( SecurityGroupIngress( f"From{NONALPHANUM.sub('', source['Id'])}ToServiceOn{target_port}", SourcePrefixListId=source["Id"], **common_args, ))
def memory_limit(self): if not self._mem_alloc or self.container_start_condition in [ "SUCCESS", "COMPLETE", ]: return NoValue resource = "memory" str_value = set_else_none( resource, set_else_none("limits", self.resources, alt_value={})) if not str_value: return NoValue return set_memory_to_mb(str_value)
def env_files(self) -> list: """ Method to list all the env files and check the files are found and available. """ env_file_key = "env_file" _env_files = set_else_none(env_file_key, self.definition) if not _env_files: return [] if not isinstance(_env_files, (str, list)): raise TypeError( self.name, env_file_key, "must be one of", (str, list), "Got", _env_files, type(_env_files), ) env_files = [] if isinstance(self.definition[env_file_key], str): env_files = [_env_files] for file_path in _env_files: if not isinstance(file_path, str): raise TypeError( "Files in the env_file is supposed to be a list of paths to files (str). Got", type(file_path), ) if not path.exists(path.abspath(file_path)): raise FileNotFoundError("No file found at", path.abspath(file_path)) env_files.append(path.abspath(file_path)) return env_files
def set_drop_capacities(service, drop_key, valid, cap_adds, all_adds, all_drops, fargate): """ Set the drop kernel capacities :param str drop_key: :param list valid: :param list cap_adds: :param list all_adds: :param list all_drops: :param list fargate: """ to_drop = set_else_none(drop_key, service.definition, alt_value=[]) for capacity in to_drop: if capacity not in valid: raise ValueError( f"{service.name} - Linux kernel capacity {capacity} is not supported in ECS or simply not valid" ) if capacity in all_adds or capacity in cap_adds: raise KeyError( f"{service.name} - Capacity {capacity} already detected in cap_add. " "You cannot both add and remove the capacity") if capacity in fargate: cap_adds.append(capacity) else: all_drops.append(capacity)
def add_parameter_to_group_label(interface_metadata: dict, parameter: Parameter) -> None: """ Simply goes over the ParameterGroups of the metadata.AWS::CloudFormation::Interface and if already exists, adds to group, else, create group and adds first element :param dict interface_metadata: :param ecs_composex.common.cfn_params.Parameter parameter: """ groups = set_else_none("ParameterGroups", interface_metadata, [], eval_bool=True) if not groups: interface_metadata["ParameterGroups"] = groups groups.append({ "Label": { "default": parameter.group_label }, "Parameters": [parameter.title], }) else: for group in groups: if group["Label"]["default"] == parameter.group_label: if parameter.title not in group["Parameters"]: group["Parameters"].append(parameter.title) break else: groups.append({ "Label": { "default": parameter.group_label }, "Parameters": [parameter.title], })
def user_define_essential(self) -> Union[None, bool]: """ Allows user to override whether a container is essential or not. By default, in absence of the label, service is considered essential as it might be the only one in the family """ essential_key = "ecs.essential" _defined_essential = set_else_none(essential_key, self.deploy_labels, alt_value=None) if _defined_essential is None: return None positive_values = [True, "yes", "True"] negative_values = [False, "no", "False"] if (_defined_essential not in positive_values or _defined_essential not in negative_values): raise ValueError( "The values allowed for", essential_key, "are", positive_values, negative_values, "Got", _defined_essential, ) if _defined_essential in negative_values: return False return True
def define_service_image(service, settings): """ Function to parse and identify the image for the service in AWS ECR :param ecs_composex.common.compose_services.ComposeService service: :param ecs_composex.common.settings.ComposeXSettings settings: The settings for the execution :return: """ if not service.image.private_ecr: return image_sha = None image_tag = None parts = service.image.private_ecr tag = parts.group("tag") if tag.startswith(r":"): image_tag = tag.split(":")[-1] elif tag.startswith(r"@"): image_sha = tag.split("@")[-1] repo_name = parts.group("repo_name") account_id = parts.group("account_id") region = parts.group("region") session = define_ecr_session( account_id, repo_name, region, settings, role_arn=set_else_none("RoleArn", service.x_ecr), ) the_image = identify_service_image(service, repo_name, image_sha, image_tag, session) return the_image
def create_ext_sources_ingress_rule(self, destination_title, allowed_source, security_group, **props) -> None: """ Creates the Security Ingress rule for a CIDR based rule :param str destination_title: :param dict allowed_source: :param security_group: :param dict props: """ for port in self.ports: target_port = set_else_none("published", port, alt_value=set_else_none( "target", port, None)) if target_port is None: raise ValueError( "Wrong port definition value for security group ingress", port) if (keyisset("Ports", allowed_source) and target_port not in allowed_source["Ports"]): continue if keyisset("Name", allowed_source): name = NONALPHANUM.sub("", allowed_source["Name"]) title = f"From{name.title()}To{target_port}{port['protocol']}" description = Sub( f"From {name.title()} " f"To {target_port}{port['protocol']} for {destination_title}" ) else: title = (f"From{flatten_ip(allowed_source[self.ipv4_key])}" f"To{target_port}{port['protocol']}") description = Sub(f"Public {target_port}{port['protocol']}" f" for {destination_title}") self.ext_ingress_rules.append( SecurityGroupIngress( title, Description=description if not keyisset("Description", allowed_source) else allowed_source["Description"], GroupId=security_group, IpProtocol=port["protocol"], FromPort=target_port, ToPort=target_port, **props, ))
def ulimits(self) -> Union[list, Ref]: """ Set the ulimits """ _ulimits = set_else_none("ulimits", self.definition, alt_value={}) if not _ulimits: return NoValue rendered_limits = [] fargate_supported = ["nofile"] allowed = [ "core", "cpu", "data", "fsize", "locks", "memlock", "msgqueue", "nice", "nofile", "nproc", "rss", "rtprio", "rttime", "sigpending", "stack", ] for limit_name, limit_value in _ulimits.items(): if limit_name not in allowed: raise KeyError( f"{self.name} - ulimit property {limit_name} is not supported by ECS. Valid ones are", allowed, ) elif isinstance(limit_value, (str, int)): ulimit = Ulimit( SoftLimit=int(limit_value), HardLimit=int(limit_value), Name=limit_name, ) elif isinstance(limit_value, dict): if keyisset("soft", limit_value) and keyisset( "hard", limit_value): ulimit = Ulimit( SoftLimit=int(limit_value["soft"]), HardLimit=int(limit_value["hard"]), Name=limit_name, ) else: raise KeyError( f"Missing hard or soft properties for ulimit {limit_name}" ) else: raise TypeError( f"{self.name} - ulimit is not of the proper definition") if limit_name not in fargate_supported: rendered_limits.append(If(USE_FARGATE_CON_T, NoValue, ulimit)) else: rendered_limits.append(ulimit) return rendered_limits if rendered_limits else NoValue
def set_healthcheck_definition(props, target_definition): """ :param dict props: :param dict target_definition: :return: """ healthcheck_props = { "HealthCheckEnabled": Ref(AWS_NO_VALUE), "HealthCheckIntervalSeconds": Ref(AWS_NO_VALUE), "HealthCheckPath": Ref(AWS_NO_VALUE), "HealthCheckPort": Ref(AWS_NO_VALUE), "HealthCheckProtocol": Ref(AWS_NO_VALUE), "HealthCheckTimeoutSeconds": Ref(AWS_NO_VALUE), "HealthyThresholdCount": Ref(AWS_NO_VALUE), } required_mapping = ( "HealthCheckPort", "HealthCheckProtocol", ) required_rex = re.compile( r"^([\d]{2,5}):(HTTPS|HTTP|TCP_UDP|TCP|TLS|UDP)$") healthcheck_reg = re.compile( r"(^(?:[\d]{2,5}):(?:HTTPS|HTTP|TCP_UDP|TCP|TLS|UDP)):?" r"((?:[\d]{1}|10):(?:[\d]{1}|10):[\d]{1,3}:[\d]{1,3})?:" r"?((?:/[\S][^:]+.$)|(?:/[\S]+)(?::)(?:(?:[\d]{1,4},?){1,}.$)|(?:(?:[\d]{1,4},?){1,}.$))?" ) healthcheck_definition = set_else_none("healthcheck", target_definition) if isinstance(healthcheck_definition, str): groups = healthcheck_reg.search(healthcheck_definition).groups() if not groups[0]: raise ValueError( "You need to define at least the Protocol and port for healthcheck" ) for count, value in enumerate(required_rex.match(groups[0]).groups()): healthcheck_props[required_mapping[count]] = value if groups[1]: handle_ping_settings(healthcheck_props, groups[1]) if groups[2]: try: handle_path_settings(healthcheck_props, groups[2]) except ValueError: LOG.error(target_definition["name"], target_definition["healthcheck"]) raise elif isinstance(healthcheck_definition, dict): healthcheck_props.update(healthcheck_definition) if keyisset("Matcher", healthcheck_definition): healthcheck_props["Matcher"] = Matcher( **healthcheck_definition["Matcher"]) else: raise TypeError( healthcheck_definition, type(healthcheck_definition), "must be one of", (str, dict), ) props.update(healthcheck_props)
def handle_awslogs_options(service: ComposeService, logging_def: dict) -> LogConfiguration: options_def = set_else_none("options", logging_def) options = { "awslogs-group": set_else_none("awslogs-group", options_def, alt_value=service.logical_name), "awslogs-region": set_else_none("awslogs-region", options_def, alt_value=Region), "awslogs-stream-prefix": set_else_none("awslogs-stream-prefix", options_def, alt_value=service.name), "awslogs-endpoint": set_else_none("awslogs-endpoint", options_def, alt_value=NoValue), "awslogs-datetime-format": set_else_none( "awslogs-datetime-format", options_def, alt_value=NoValue, ), "awslogs-multiline-pattern": set_else_none( "awslogs-multiline-pattern", options_def, alt_value=NoValue, ), "mode": set_else_none("mode", options_def, alt_value=NoValue), "max-buffer-size": set_else_none("max-buffer-size", options_def, alt_value=NoValue), } if keypresent("awslogs-create-group", options_def) and isinstance( options_def["awslogs-create-group"], bool): options["awslogs-create-group"] = keyisset("awslogs-create-group", options_def) elif keypresent("awslogs-create-group", options_def) and isinstance( options_def["awslogs-create-group"], str): options["awslogs-create-group"] = options_def[ "awslogs-create-group"] in [ "yes", "true", "Yes", "True", ] return LogConfiguration( LogDriver="awslogs", Options=options, )
def families(self): ecs_task_family = "ecs.task.family" __families = set_else_none(ecs_task_family, self.deploy_labels) if __families is None: return [self.name] if not isinstance(__families, str): raise TypeError(ecs_task_family, "must be", str, "got", __families, type(__families)) return __families.split(r",")
def update_config(self): _config = set_else_none("update_config", self.deploy, alt_value={}) if not isinstance(_config, dict): raise TypeError( "The deploy.update_config must be a dict/map. Got", _config, type(_config), ) return _config
def __init__( self, name: str, definition: dict, module: XResourceModule, settings: ComposeXSettings, ): self.zone_name = None self.records = [] super().__init__(name, definition, module, settings) self.zone_name = set_else_none( "Name", self.definition, set_else_none("ZoneName", self.definition, None) ) if self.zone_name is None: raise ValueError( f"{self.module.res_key}.{self.name} - No ZoneName/Name specified" ) self.requires_vpc = True
def __init__(self, service: ComposeService, default_options: dict): self._service = service self._log_configuration = NoValue self.def_logging = set_else_none("logging", service.definition) self.def_x_logging = set_else_none("x-logging", service.definition) self._log_driver = None self._log_options: dict = {} self.log_driver = set_else_none("driver", self.def_logging, alt_value="awslogs") self.log_options = set_else_none( "options", self.def_logging, alt_value=default_options ) self.cw_retention_period = get_closest_valid_log_retention_period( set_else_none( "RetentionInDays", self.def_x_logging, alt_value=LOG_GROUP_RETENTION.Default, ) )
def __init__( self, name: str, definition: dict, module: XResourceModule, settings: ComposeXSettings, ): self.zone_name = None self.records = [] super().__init__(name, definition, module, settings) self.cloud_control_attributes_mapping = {PUBLIC_DNS_ZONE_ID.title: "Id"} self.zone_name = set_else_none( "ZoneName", self.definition, set_else_none("Name", self.definition, None) ) if self.zone_name is None: raise ValueError( f"{self.module.res_key}.{self.name} - Could not define the Zone Name" )
def __init__(self, name, definition, subnets_list): self.name = name self.definition = deepcopy(definition) self.subnet_name = set_else_none("x-vpc", definition, None) subnet_names = [subnet.title for subnet in subnets_list] if self.subnet_name and self.subnet_name not in subnet_names: raise KeyError( f"networks.{name} - x-vpc.{self.subnet_name} defined. Valid options are", subnet_names, )
def container_start_condition(self) -> str: if (isinstance(self.ecs_healthcheck, HealthCheck) and self.ecs_healthcheck != NoValue): return "HEALTHY" depends_key = "ecs.depends.condition" return set_else_none( depends_key, self.deploy_labels, alt_value="START", )
def shm_size(self): """ Method to import and determine SHM SIZE """ __shm_size = set_else_none("shm_size", self.definition) if not __shm_size: return NoValue if not isinstance(__shm_size, (int, str, float)): raise TypeError(self.name) memory_value = set_memory_to_mb(__shm_size) return If(USE_FARGATE_CON_T, NoValue, memory_value)