def __init__(self, family: ComposeFamily): """ Initialize network settings for the family ServiceConfig :param ecs_composex.ecs.ecs_family.ComposeFamily family: """ self.family = family self._network_mode = "awsvpc" if family.service_compute.launch_type == "EXTERNAL": LOG.warning( f"{family.name} - External mode cannot use awsvpc mode. Falling back to bridge" ) self.network_mode = "bridge" self.ports = [] self.networks = {} self.merge_services_ports() self.merge_networks() self.definition = merge_family_services_networking(family) self.ingress_from_self = False if any([svc.expose_ports for svc in family.services]): self.ingress_from_self = True LOG.info( f"{family.name} - services have export ports, allowing internal ingress" ) self._security_group = None self.extra_security_groups = [] self._subnets = Ref(APP_SUBNETS) self.cloudmap_config = (merge_cloudmap_settings(family, self.ports) if self.ports else {}) self.ingress = Ingress(self.definition[Ingress.master_key], self.ports) self.ingress_from_self = keyisset(self.self_key, self.definition)
def set_services_scaling_list(self, settings): """ Method to map services and families targets of the services defined. :param ecs_composex.common.settings.ComposeXSettings settings: :return: """ if not self.services or isinstance(self.services, dict): return for service in self.services: name_key = get_setting_key("name", service) scaling_key = get_setting_key("scaling", service) if not keyisset(scaling_key, service): LOG.debug( f"{self.module.res_key}.{self.name} - No Scaling set for {service[name_key]}" ) continue service_name = service[name_key] if service_name in settings.families and service_name not in [ f[0].name for f in self.families_scaling ]: self.families_scaling.append( (settings.families[service_name], service[scaling_key])) elif service_name in settings.families and service_name in [ f[0].name for f in self.families_scaling ]: LOG.debug( f"{self.module.res_key}.{self.name} - Family {service_name} has already been added. Skipping" ) elif service_name in [s.name for s in settings.services]: self.handle_family_scaling_expansion(service, settings)
def set_update_log_configuration(self, **kwargs): if kwargs and keyisset("LogDriver", kwargs) and keyisset("Options", kwargs): self.log_configuration = LogConfiguration(**kwargs) return if self.log_driver == "awslogs": self.log_configuration = handle_awslogs_options( self.service, self.log_config ) if self.replace_cw_with_firelens: self.log_configuration = replace_awslogs_with_firelens_configuration( self.service, self.log_configuration ) elif self.log_driver == "awsfirelens": self.log_configuration = handle_firelens_options( self.service, self.log_config )
def set_sid_name(access_definition, access_subkey: str) -> str: """ Defines the name of the SID to use for the policy. Defines access_type :param dict,str access_definition: :param str access_subkey: :return: access_type :rtype: str """ if isinstance(access_definition, dict) and keyisset( access_subkey, access_definition ): if isinstance(access_definition[access_subkey], bool): access_type = access_subkey else: access_type = f"{access_subkey}{access_definition[access_subkey]}" elif isinstance(access_definition, str): access_type = access_definition else: raise ValueError( "The access_definition is not valid", access_definition, type(access_definition), "subkey is", access_subkey, ) return access_type
def handle_families_targets_expansion_dict(self, service_name, service, settings) -> None: """ Method to list all families and services that are targets of the resource. Allows to implement family and service level association to resource :param str service_name: :param dict service: Service definition in compose file :param ecs_composex.common.settings.ComposeXSettings settings: Execution settings """ for svc in settings.services: if svc.name == service_name: the_service = svc break else: raise KeyError( f"Service {service_name} not found in ", [_svc.name for _svc in settings.services], ) for family_name in the_service.families: family_name = NONALPHANUM.sub("", family_name) if family_name not in [f[0].name for f in self.families_targets]: self.families_targets.append(( settings.families[family_name], False, [the_service], service["Access"] if keyisset("Access", service) else {}, service, ))
def get_bucket_config(bucket: Bucket, resource_id: str) -> dict: """ :param ecs_composex.s3.s3_bucket.Bucket bucket: :param str resource_id: """ bucket_config = { S3_BUCKET_NAME: resource_id, S3_BUCKET_ARN: bucket.arn, } client = bucket.lookup_session.client("s3") try: encryption_r = client.get_bucket_encryption(Bucket=resource_id) encryption_attributes = attributes_to_mapping( encryption_r, CONTROL_CLOUD_ATTR_MAPPING) if keyisset( CONTROL_CLOUD_ATTR_MAPPING[S3_BUCKET_KMS_KEY], encryption_attributes, ): bucket_config[S3_BUCKET_KMS_KEY] = encryption_attributes[ S3_BUCKET_KMS_KEY] except ClientError as error: if (not error.response["Error"]["Code"] == "ServerSideEncryptionConfigurationNotFoundError"): raise LOG.warning(error.response["Error"]["Message"]) return bucket_config
def set_service_update_config(family) -> dict: """ Method to determine the update_config for the service. When a family has multiple containers, this applies to all tasks. """ deployment_config = {} min_percents = [ int(service.definition["x-aws-min_percent"]) for service in family.services if keypresent("x-aws-min_percent", service.definition) ] max_percents = [ int(service.definition["x-aws-max_percent"]) for service in family.services if keypresent("x-aws-max_percent", service.definition) ] family_min_percent = define_family_deploy_percents(min_percents, 100) family_max_percent = define_family_deploy_percents(max_percents, 200) rollback = True actions = [ service.update_config["failure_action"] != "rollback" for service in family.services if service.update_config and keyisset("failure_action", service.update_config) ] if any(actions): rollback = False deployment_config.update({ "MinimumHealthyPercent": family_min_percent, "MaximumPercent": family_max_percent, "RollBack": rollback, }) return deployment_config
def get_setting_key(name: str, settings_dict: dict) -> str: """ Allows for flexibility in the syntax, i.e. to make access/Access both valid """ if keyisset(name.title(), settings_dict): return name.title() return name
def set_resources(settings: ComposeXSettings, resource_class, module: XResourceModule): """ Method to define the ComposeXResource for each service. First updates the resources dict :param ecs_composex.common.settings.ComposeXSettings settings: :param ecs_composex.common.compose_resources.XResource resource_class: :param XResourceModule module: """ if not keyisset(module.res_key, settings.compose_content): return resources_ordered_dict = OrderedDict( sorted( settings.compose_content[module.res_key].items(), key=lambda item: item[0], )) del settings.compose_content[module.res_key] settings.compose_content[module.res_key] = resources_ordered_dict for resource_name, resource_definition in resources_ordered_dict.items(): new_definition = resource_class( name=resource_name, definition=resource_definition, module=module, settings=settings, ) LOG.debug(type(new_definition)) LOG.debug(new_definition.__dict__) settings.compose_content[ module.res_key][resource_name] = new_definition
def create_root_template(new_resources: list, module_res_key: str) -> Template: """ Function to create the root stack template for profiles :param list[CodeProfiler] new_resources: :param str module_res_key: :return: The template wit the profiles :rtype: troposphere.Template """ root_tpl = build_template(f"Root stack to manage {module_res_key}") for res in new_resources: try: props = import_record_properties(res.properties, ProfilingGroup, ignore_missing_required=False) if res.parameters and keyisset("AppendStackId", res.parameters): props["ProfilingGroupName"] = Sub( f"{res.properties['ProfilingGroupName']}-${{StackId}}", StackId=STACK_ID_SHORT, ) except KeyError: props = import_record_properties(res.properties, ProfilingGroup, ignore_missing_required=True) props["ProfilingGroupName"] = Sub( f"{res.logical_name}-${{StackId}}", StackId=STACK_ID_SHORT) res.cfn_resource = ProfilingGroup(res.logical_name, **props) res.init_outputs() res.generate_outputs() add_outputs(root_tpl, res.outputs) root_tpl.add_resource(res.cfn_resource) return root_tpl
def determine_resource_type(db_name, properties): """ Function to determine if the properties are the ones of a DB Cluster or DB Instance. By default it will assume Cluster if cannot conclude that it is a DB Instance :param str db_name: :param dict properties: :return: """ if ( keyisset(DB_ENGINE_NAME.title, properties) and properties[DB_ENGINE_NAME.title].startswith("aurora") or all( property_name in DBCluster.props.keys() for property_name in properties.keys() ) ): LOG.info(f"Identified {db_name} to be a RDS Aurora Cluster") return DBCluster elif all( property_name in DBInstance.props.keys() for property_name in properties.keys() ): LOG.info(f"Identified {db_name} to be a RDS Instance") return DBInstance LOG.error( "From the properties defined, we cannot determine whether this is a RDS Cluster or RDS Instance." " Setting to Cluster" ) return None
def __init__(self, title, settings: ComposeXSettings, module: XResourceModule, **kwargs): """ Init method :param str title: :param ecs_composex.common.settings.ComposeXSettings settings: :param kwargs: """ set_resources(settings, CodeProfiler, module) x_resources = settings.compose_content[module.res_key].values() lookup_resources = set_lookup_resources(x_resources) if lookup_resources: if not keyisset(module.mapping_key, settings.mappings): settings.mappings[module.mapping_key] = {} define_lookup_profile_mappings( settings.mappings[module.mapping_key], lookup_resources, settings) new_resources = set_new_resources(x_resources, False) if new_resources: stack_template = create_root_template(new_resources, module.res_key) super().__init__(title, stack_template, **kwargs) else: self.is_void = True for resource in settings.compose_content[module.res_key].values(): resource.stack = self
def resolve_lookup( lookup_resources: list[UserPool], settings: ComposeXSettings, module: XResourceModule, ): """ Iterates over the lookup resources and performs the lookup to create the resource mapping used in the template. :param list[UserPool] lookup_resources: the lookup resources to process :param ecs_composex.common.settings.ComposeXSettings settings: the ComposeX Execution settings. :param module: """ if not keyisset(module.mapping_key, settings.mappings): settings.mappings[module.mapping_key] = {} for resource in lookup_resources: resource.lookup_resource( USER_POOL_RE, get_userpool_config, CfnUserPool.resource_type, "cognito-idp", ) resource.init_outputs() resource.generate_cfn_mappings_from_lookup_properties() resource.generate_outputs() settings.mappings[module.mapping_key].update( {resource.logical_name: resource.mappings}) LOG.info( f"{resource.module.res_key}.{resource.name} Found in AWS Account")
def __init__(self, title, settings: ComposeXSettings, module: XResourceModule, **kwargs): set_resources(settings, NeptuneDBCluster, module) x_resources = settings.compose_content[module.res_key].values() new_resources = set_new_resources(x_resources, True) lookup_resources = set_lookup_resources(x_resources) if new_resources: stack_template = build_template( "Root template for Neptune by ComposeX", [VPC_ID, STORAGE_SUBNETS]) super().__init__(title, stack_template, **kwargs) create_neptune_template(stack_template, new_resources, settings, self) else: self.is_void = True if lookup_resources: if not keyisset(module.mapping_key, settings.mappings): settings.mappings[module.mapping_key] = {} for resource in lookup_resources: resource.lookup_resource( NEPTUNE_DB_CLUSTER_ARN_RE, get_db_cluster_config, CfnDBCluster.resource_type, "rds:cluster", ) resource.generate_cfn_mappings_from_lookup_properties() resource.generate_outputs() settings.mappings[module.mapping_key].update( {resource.logical_name: resource.mappings}) for resource in settings.compose_content[module.res_key].values(): resource.stack = self
def set_content(self, kwargs, content=None, fully_load=True): """ Method to initialize the compose content :param dict kwargs: :param dict content: :param bool fully_load: """ files = ( [] if not keyisset(self.input_file_arg, kwargs) else kwargs[self.input_file_arg] ) content_def = ComposeDefinition(files, content) self.compose_content = content_def.definition source = pkg_files("ecs_composex").joinpath("specs/compose-spec.json") LOG.info(f"Validating against input schema {source}") resolver = jsonschema.RefResolver( f"file://{path.abspath(path.dirname(source))}/", None ) jsonschema.validate( content_def.definition, loads(source.read_text()), resolver=resolver, ) if fully_load: self.set_secrets() self.set_volumes() self.set_services() self.set_families() self.set_efs()
def handle_firelens_options(service: ComposeService, logging_def: dict, set_cw_default: bool = False) -> LogConfiguration: default_cloudwatch_options = { "region": Region, "auto_create_group": True, "log_group_name": service.logical_name, "log_stream_prefix": service.service_name, } if set_cw_default: options = set_else_none("options", logging_def, alt_value=default_cloudwatch_options) else: options = set_else_none("options", logging_def, alt_value=NoValue) config_name_map = { "delivery_stream": "kinesis_firehose", "log_group_name": "cloudwatch", "stream": "kinesis_streams", "bucket": "s3", } for key, value in config_name_map.items(): if keyisset(key, options): options.update({"Name": value}) break return LogConfiguration(LogDriver="awsfirelens", Options=options)
def set_output_settings(self, kwargs): """ Method to set the output settings based on kwargs """ self.format = self.default_format if ( keyisset(self.format_arg, kwargs) and kwargs[self.format_arg] in self.allowed_formats ): self.format = kwargs[self.format_arg] self.output_dir = ( kwargs[self.output_dir_arg] if keyisset(self.output_dir_arg, kwargs) else self.default_output_dir )
def replicas(self): if self.family.stack and keyisset( ecs_params.SERVICE_COUNT.title, self.family.stack.Parameters ): return self.family.stack.Parameters[ecs_params.SERVICE_COUNT.title] else: return max(service.replicas for service in self.family.services)
def add_all_tags(root_template, settings, params=None, xtags=None): """ Function to go through all stacks of a given template and update the template It will recursively render sub stacks defined. If there are no substacks, it will go over the resources of the template add the tags. :param troposphere.Template root_template: the root template to iterate over the resources. :param ecs_composex.common.settings.ComposeXSettings settings: Execution settings :param list params: Parameters to add to template if any :param troposphere.Tags xtags: List of Tags to add to the resources. """ if not params or not xtags: if not keyisset("x-tags", settings.compose_content): xtags = default_tags() params = None else: tags = settings.compose_content["x-tags"] params = generate_tags_parameters(tags) xtags = define_extended_tags(tags) xtags += default_tags() resources = root_template.resources if root_template else [] for resource_name in resources: resource = resources[resource_name] apply_tags_to_resources(settings, resource, params, xtags)
def define_kms_key(self): """ Method to set the KMS Key """ if not self.properties: props = { "Description": Sub( f"{self.name} created in ${{STACK_NAME}}", STACK_NAME=define_stack_name(), ), "Enabled": True, "EnableKeyRotation": True, "KeyUsage": "ENCRYPT_DECRYPT", "PendingWindowInDays": 7, } else: props = import_record_properties(self.properties, Key) if not keyisset("KeyPolicy", props): props.update({"KeyPolicy": define_default_key_policy()}) props.update({"Metadata": metadata}) LOG.debug(props) self.cfn_resource = Key(self.logical_name, **props)
def define_deployment_options(family, props: dict) -> None: """ Function to define the DeploymentConfiguration Default is to have Rollback and CircuitBreaker on. :param ecs_composex.ecs.ecs_family.ComposeFamily family: :param dict props: the troposphere.ecs.Service properties definition to update with deployment config. """ default = DeploymentConfiguration( DeploymentCircuitBreaker=DeploymentCircuitBreaker(Enable=True, Rollback=True), ) deployment_config = set_service_update_config(family) if deployment_config: deploy_config = DeploymentConfiguration( MaximumPercent=int(deployment_config["MaximumPercent"]), MinimumHealthyPercent=int( deployment_config["MinimumHealthyPercent"]), DeploymentCircuitBreaker=DeploymentCircuitBreaker( Enable=True, Rollback=keyisset("RollBack", deployment_config), ), ) props.update({"DeploymentConfiguration": deploy_config}) else: props.update({"DeploymentConfiguration": default})
def handle_key_settings(self, template): """ Method to add to the template for additional KMS key related resources. :param troposphere.Template template: """ if self.parameters and keyisset("Alias", self.parameters): alias_name = self.parameters["Alias"] if not (alias_name.startswith("alias/") or alias_name.startswith("aws")): alias_name = Sub( f"alias/${{STACK_NAME}}/{alias_name}", STACK_NAME=define_stack_name(template), ) elif alias_name.startswith("alias/aws") or alias_name.startswith( "aws"): raise ValueError( f"Alias {alias_name} cannot start with alias/aws.") Alias( f"{self.logical_name}Alias", template=template, AliasName=alias_name, TargetKeyId=Ref(self.cfn_resource), Metadata=metadata, )
def map_resource_env_vars_to_family_services( target, resource, ) -> None: """ Function to deal with the env vars to add to the family stack based on the resource Services definition :param tuple target: :param ecs_composex.compose.x_resources.XResource resource: """ map_resource_env_vars_to_family_service_environment(target, resource) return_values = ( {} if not keyisset("ReturnValues", target[-1]) else target[-1]["ReturnValues"] ) for svc in target[2]: if svc in target[0].managed_sidecars: continue if return_values: extend_container_envvars( svc.container_definition, resource.generate_resource_service_env_vars(target, return_values), ) else: extend_container_envvars( svc.container_definition, resource.generate_ref_env_var(target) )
def init_outputs(self): spacer = "" if (self.properties and keyisset("Name", self.properties) and not self.properties["Name"].startswith(r"/")) or not keyisset( "Name", self.properties): spacer = "/" self.output_properties = { SSM_PARAM_NAME: (self.logical_name, self.cfn_resource, Ref, None), SSM_PARAM_ARN: ( f"{self.logical_name}{SSM_PARAM_ARN.title}", self.cfn_resource, Sub, f"arn:${{{AWS_PARTITION}}}:ssm:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}:parameter{spacer}" f"${{{self.cfn_resource.title}}}", ), }
def set_services_targets_from_dict(self, settings): """ Deals with services set as a dict :param settings: :return: """ for service_name, service_def in self.services.items(): if service_name in settings.families and service_name not in [ f[0].name for f in self.families_targets ]: self.families_targets.append(( settings.families[service_name], True, settings.families[service_name].services, service_def["Access"] if keyisset("Access", service_def) else {}, service_def, )) elif service_name in settings.families and service_name in [ f[0].name for f in self.families_targets ]: LOG.debug( f"{self.module.res_key}.{self.name} - Family {service_name} has already been added. Skipping" ) elif service_name in [s.name for s in settings.services]: self.handle_families_targets_expansion_dict( service_name, service_def, settings)
def handle_user_defined_policies(bucket: Bucket, param_key: str, user_policies_key: str, statement: list): """ Function to add user defined policies :param bucket: :param str param_key: :param str user_policies_key: :param list statement: """ policies = bucket.parameters[param_key][user_policies_key] policy_docs = {} for count, policy_doc in enumerate(policies): if keyisset("Sid", policy_doc): name = policy_doc["Sid"] else: name = f"UserDefined{count}" policy_docs[name] = policy_doc generated_policies = generate_resource_permissions( bucket.logical_name, policy_docs, Sub(f"arn:${{{AWS_PARTITION}}}:s3:::${{{bucket.cfn_resource.title}}}"), True, ) for policy in generated_policies.values(): statement += policy.PolicyDocument["Statement"]
def set_services_targets_scaling_from_dict(self, settings) -> None: """ Deals with services set as a dict :param settings: """ for service_name, service_def in self.services.items(): if not keyisset("Scaling", service_def): LOG.debug( f"{self.module.res_key}.{self.name} - No Scaling set for {service_name}" ) continue if service_name in settings.families and service_name not in [ f[0].name for f in self.families_scaling ]: self.families_scaling.append(( settings.families[service_name], service_def["Scaling"], )) elif service_name in settings.families and service_name in [ f[0].name for f in self.families_scaling ]: LOG.debug( f"{self.module.res_key}.{self.name} - Family {service_name} has already been added. Skipping" ) elif service_name in [s.name for s in settings.services]: self.handle_families_scaling_expansion_dict( service_name, service_def, settings)
def generate_outputs(self): for ( attribute_parameter, output_definition, ) in self.output_properties.items(): output_name = f"{self.title}{attribute_parameter.title}" value = self.set_new_resource_outputs(output_definition) self.attributes_outputs[attribute_parameter] = { "Name": output_name, "Output": Output(output_name, Value=value), "ImportParameter": Parameter( output_name, return_value=attribute_parameter.return_value, Type=attribute_parameter.Type, ), "ImportValue": GetAtt( self.stack, f"Outputs.{output_name}", ), "Original": attribute_parameter, } for attr in self.attributes_outputs.values(): if keyisset("Output", attr): self.outputs.append(attr["Output"])
def get_topic_config(topic: Topic, account_id: str, resource_id: str) -> dict | None: """ Function to create the mapping definition for SNS topics """ topic_config = {TOPIC_NAME: resource_id} client = topic.lookup_session.client("sns") attributes_mapping = { TOPIC_ARN: "Attributes::TopicArn", TOPIC_KMS_KEY: "Attributes::KmsMasterKeyId", } try: topic_r = client.get_topic_attributes(TopicArn=topic.arn) attributes = attributes_to_mapping(topic_r, attributes_mapping) if keyisset(TOPIC_KMS_KEY, attributes) and not attributes[ TOPIC_KMS_KEY ].startswith("arn:aws"): if attributes[TOPIC_KMS_KEY].startswith("alias/aws"): LOG.warning( f"{topic.module.res_key}.{topic.name} - Topic uses the default AWS CMK." ) else: LOG.warning( f"{topic.module.res_key}.{topic.name} - KMS Key provided is not a valid ARN." ) del attributes[TOPIC_KMS_KEY] topic_config.update(attributes) return topic_config except client.exceptions.QueueDoesNotExist: return None except ClientError as error: LOG.error(error) raise
def set_services_mount_points(family): """ Method to set the mount points to the Container Definition of the defined service if the volume["volume"] is none, this is not a shared volume, which then works only when not using Fargate (i.e. EC2 host/ ECS Anywhere) """ for service in family.services: mount_points = [] if not hasattr(service.container_definition, "MountPoints"): setattr(service.container_definition, "MountPoints", mount_points) else: mount_points = getattr(service.container_definition, "MountPoints") for volume in service.volumes: if keyisset("volume", volume): mnt_point = MountPoint( ContainerPath=volume["target"], ReadOnly=volume["read_only"], SourceVolume=volume["volume"].volume_name, ) else: mnt_point = If( USE_FARGATE_CON_T, NoValue, MountPoint( ContainerPath=volume["target"], ReadOnly=volume["read_only"], SourceVolume=NONALPHANUM.sub("", volume["target"]), ), ) if not mount_point_exists(mount_points, mnt_point, family, service): mount_points.append(mnt_point)