def handle_json_validation(resource: SsmParameter, value: str, file_path: str) -> str: """ Function to evaluate the JSON content :param SsmParamter resource: :param str value: Value read from file :param str file_path: :return: """ try: payload = json.loads(value) if keyisset("MinimizeJson", resource.parameters): return json.dumps(payload, separators=(",", ":")) return value except json.decoder.JSONDecodeError: if keyisset("IgnoreInvalidJson", resource.parameters): LOG.warn( f"{resource.name} - The content of {file_path} " "did not pass JSON validation. Skipping due to IgnoreInvalidJson" ) return value else: LOG.error(f"{resource.name} - The content of {file_path} " "did not pass JSON validation.") raise
def handle_yaml_validation(resource: SsmParameter, value: str, file_path: str) -> str: """ Function to evaluate the JSON content :param SsmParamter resource: :param str value: Value read from file :param str file_path: :return: """ try: payload = yaml.load(value, Loader=Loader) if keyisset("RenderToJson", resource.parameters): return json.dumps(payload, separators=(",", ":")) return value except yaml.YAMLError: if keyisset("IgnoreInvalidYaml", resource.parameters): LOG.warn( f"{resource.name} - The content of {file_path} " "did not pass YAML validation. Skipping due to IgnoreInvalidYaml" ) return value else: LOG.error(f"{resource.name} - The content of {file_path} " "did not pass YAML validation.") raise
def generate_ref_env_var(self, target) -> list: """ Method to define all the env var of a resource based on its own defined output attributes """ if not self.ref_parameter: LOG.error( f"{self.module.res_key}.{self.name}. Default ref_parameter not set. Skipping env_vars" ) return [] env_var_name = ENV_VAR_NAME.sub("", self.name.upper().replace("-", "_")) if self.cfn_resource and self.attributes_outputs and self.ref_parameter: ref_env_var = Environment( Name=env_var_name, Value=Ref(self.attributes_outputs[self.ref_parameter] ["ImportParameter"]), ) ref_param_settings = get_parameter_settings( self, self.ref_parameter) add_parameters(target[0].template, [ref_param_settings[1]]) target[0].stack.Parameters.update( {ref_param_settings[0]: ref_param_settings[2]}) elif self.lookup_properties and self.ref_parameter: ref_env_var = Environment( Name=env_var_name, Value=self.attributes_outputs[self.ref_parameter] ["ImportValue"], ) else: raise ValueError( f"{self.module.res_key}.{self.name} - Unable to set the default env var" ) return [ref_env_var]
def create_new_stream(stream: DeliveryStream) -> None: """ Imports the settings from CFN Definitions and define the CFN Resource from properties :param DeliveryStream stream: """ props = import_record_properties( stream.properties, CfnDeliveryStream, ignore_missing_required=True, ignore_missing_sub_required=True, ) stream.cfn_resource = CfnDeliveryStream(stream.logical_name, **props) stream.log_group = LogGroup( f"{stream.logical_name}LogGroup", LogGroupName=Sub(f"firehose/${{STACK_ID}}/{stream.name}", STACK_ID=STACK_ID_SHORT), ) if (stream.cfn_resource.DeliveryStreamType == "KinesisStreamAsSource" and stream.cfn_resource.DeliveryStreamEncryptionConfigurationInput != NoValue): LOG.error( f"{stream.module.res_key}.{stream.name} -" " You can only have ServerSide encryption with DirectPut DeliveryStream. Removing." ) stream.cfn_resource.DeliveryStreamEncryptionConfigurationInput = NoValue set_replace_iam_role(stream) values_validation(stream) stream.init_outputs() stream.generate_outputs()
def get_key_config(key, account_id: str, resource_id: str) -> dict | None: """ :param KmsKey key: :param str account_id: unused :param str resource_id: unused :return: """ key_attributes_mappings = { KMS_KEY_ARN: "KeyMetadata::Arn", KMS_KEY_ID: "KeyMetadata::KeyId", } client = key.lookup_session.client("kms") try: key_desc = client.describe_key(KeyId=key.arn) key_attributes = attributes_to_mapping(key_desc, key_attributes_mappings) key.manager = key_desc["KeyMetadata"]["KeyManager"] try: aliases_r = client.list_aliases(KeyId=key_attributes[KMS_KEY_ID]) if aliases_r["Aliases"]: key_attributes[KMS_KEY_ALIAS_NAME] = aliases_r["Aliases"][0][ "AliasName"] except client.exceptions.NotFoundException: LOG.debug(f"{key.module.res_key}.{key.name} - No KMS Key Alias.") return key_attributes except client.exceptions.QueueDoesNotExist: return None except ClientError as error: LOG.error(error) raise
def validate_schema(self, name, definition, module_name, module_schema: str = None) -> None: """ JSON Validation of the resources module validation """ if not self.module.json_schema and not module_schema: return resolver_source = pkg_files("ecs_composex").joinpath( "specs/compose-spec.json") LOG.debug(f"Validating against input schema {resolver_source}") resolver = jsonschema.RefResolver( base_uri=f"file://{path.abspath(path.dirname(resolver_source))}/", referrer=self.module.json_schema, ) try: jsonschema.validate( definition, module_schema if module_schema else self.module.json_schema, resolver=resolver, ) except jsonschema.exceptions.ValidationError: LOG.error( f"{module_name}.{name} - Definition is not conform to schema.") raise
def get_family_from_engine_version(engine_name, engine_version, session=None, client=None): """ Function to get the engine family from engine name and version :param client: override client for boto3 call :type client: boto3.client :param session: override session for boto3 client :type session: boto3.session.Session :param engine_name: engine name, ie. aurora-mysql :type engine_name: str :param engine_version: engine version, ie. 5.7.12 :type engine_version: str :return: engine_family :rtype: str """ if not client: if not session: session = boto3.session.Session() client = session.client("rds") try: req = client.describe_db_engine_versions(Engine=engine_name, EngineVersion=engine_version) except ClientError as error: LOG.error(error) return None db_family = req["DBEngineVersions"][0]["DBParameterGroupFamily"] return db_family
def validate(self, settings): """ Method to validate the CloudFormation template, either via URL once uploaded to S3 or via TemplateBody """ try: if not settings.no_upload and self.url: validate_wrapper(settings.session, url=self.url) elif settings.no_upload or not self.url: if not self.file_path: self.write(settings) LOG.debug(f"No upload - Validating template body - {self.file_path}") if len(self.body) >= 51200: LOG.warning( f"Template body for {self.file_name} is too big for local validation." " No upload is True, so skipping." ) else: validate_wrapper(settings.session, body=self.body) LOG.debug(f"Template {self.file_name} was validated successfully by CFN") except ClientError as error: LOG.error(error) with open(f"/tmp/{settings.name}.{settings.format}", "w") as failed_file_fd: failed_file_fd.write(self.body) LOG.error( f"Failed validation template written at /tmp/{settings.name}.{settings.format}" ) raise
def apply_extra_parameters(settings, db, db_stack) -> None: """ Function to add extra parameters set in MacroParameters post creation of the DB resource from properties :param ecs_composex.common.settings.ComposeXSettings settings: :param ecs_composex.rds.rds_stack.Rds db: :param ecs_composex.rds.rds_template.RdsDbStack db_stack: """ if not db.parameters: return permissions_boundary = Ref(AWS_NO_VALUE) if keyisset("PermissionsBoundary", db.parameters): permissions_boundary = define_iam_policy( db.parameters["PermissionsBoundary"]) extra_parameters = {"RdsFeatures": (list, add_rds_features)} for name, config in extra_parameters.items(): if not keyisset(name, db.parameters): LOG.debug( f"Feature {name} has not been set in compose file. {db.parameters}" ) if (keyisset(name, db.parameters) and isinstance(db.parameters[name], config[0]) and config[1]): config[1]( settings, db, db_stack, db.parameters[name], permissions_boundary, ) elif keyisset(name, db.parameters) and not isinstance( db.parameters[name], config[0]): LOG.error( f"The property {name} is of type {type(db.parameters[name])}. Expected {config[0]}. Skipping" )
def determine_resource_type(db_name, properties): """ Function to determine if the properties are the ones of a DB Cluster or DB Instance. By default it will assume Cluster if cannot conclude that it is a DB Instance :param str db_name: :param dict properties: :return: """ if ( keyisset(DB_ENGINE_NAME.title, properties) and properties[DB_ENGINE_NAME.title].startswith("aurora") or all( property_name in DBCluster.props.keys() for property_name in properties.keys() ) ): LOG.info(f"Identified {db_name} to be a RDS Aurora Cluster") return DBCluster elif all( property_name in DBInstance.props.keys() for property_name in properties.keys() ): LOG.info(f"Identified {db_name} to be a RDS Instance") return DBInstance LOG.error( "From the properties defined, we cannot determine whether this is a RDS Cluster or RDS Instance." " Setting to Cluster" ) return None
def lookup_ecs_cluster(session, cluster_lookup): """ Function to find the ECS Cluster. :param boto3.session.Session session: Boto3 session to make API calls. :param cluster_lookup: Cluster lookup definition. :return: """ if not isinstance(cluster_lookup, str): raise TypeError("The value for Lookup must be", str, "Got", type(cluster_lookup)) client = session.client("ecs") try: cluster_r = client.describe_clusters(clusters=[cluster_lookup]) if not keyisset("clusters", cluster_r): LOG.warning( f"No cluster named {cluster_lookup} found. Creating one with default settings" ) return get_default_cluster_config() elif (keyisset("clusters", cluster_r) and cluster_r["clusters"][0]["clusterName"] == cluster_lookup): LOG.info( f"Found ECS Cluster {cluster_lookup}. Setting {CLUSTER_NAME_T} accordingly." ) return cluster_r["clusters"][0]["clusterName"] except ClientError as error: LOG.error(error) raise
def determine_resource_type(name, properties): """ Function to determine if the properties are the ones of a DB Cluster or DB Instance. By default it will assume Cluster if cannot conclude that it is a DB Instance :param str name: :param dict properties: :return: """ if all( property_name in CacheCluster.props.keys() for property_name in properties.keys() ): LOG.info(f"Identified {name} to be {CacheCluster.resource_type}") return CacheCluster elif all( property_name in ReplicationGroup.props.keys() for property_name in properties.keys() ): LOG.info(f"Identified {name} to be {ReplicationGroup.resource_type}") return ReplicationGroup LOG.error( "From the properties defined, we cannot determine whether this is a RDS Cluster or RDS Instance." " Setting to Cluster" ) return None
def create_bucket(bucket_name, session): """ Function that checks if the S3 bucket exists and if not attempts to create it. :param bucket_name: name of the s3 bucket :type bucket_name: str :param session: boto3 session to use if wanted to override settings. :type session: boto3.session.Session :returns: True/False, Returns whether the bucket exists or not for upload :rtype: bool """ client = session.client("s3") region = session.region_name location = {"LocationConstraint": region} try: client.create_bucket( ACL="private", Bucket=bucket_name, ObjectLockEnabledForBucket=True, CreateBucketConfiguration=location, ) LOG.info(f"Bucket {bucket_name} successfully created.") except client.exceptions.BucketAlreadyExists: LOG.warning(f"Bucket {bucket_name} already exists.") except client.exceptions.BucketAlreadyOwnedByYou: LOG.info(f"You already own the bucket {bucket_name}") except ClientError as error: LOG.error("Error whilst creating the bucket") LOG.error(error) raise
def generate_security_group_props(allowed_source, service_name): """ Function to parse the allowed source and create the SG Opening options accordingly. :param dict allowed_source: The allowed source defined in configs :param str service_name: :return: security group ingress properties :rtype: dict """ props = { "CidrIp": (allowed_source["ipv4"] if keyisset("ipv4", allowed_source) else Ref(AWS_NO_VALUE)), "CidrIpv6": (allowed_source["ipv6"] if keyisset( "ipv6", allowed_source) else Ref(AWS_NO_VALUE)), } if (keyisset("CidrIp", props) and isinstance(props["CidrIp"], str) and not CIDR_PAT.match(props["CidrIp"])): LOG.error( f"Falty IP Address: {allowed_source} - ecs_service {service_name}") raise ValueError( "Not a valid IPv4 CIDR notation", props["CidrIp"], "Expected", CIDR_REG, ) return props
def handle_predefined_policies(bucket: Bucket, param_key: str, managed_policies_key: str, statement: list) -> None: """ Function to configure and add statements for bucket policy based on predefined Bucket Policies :param bucket: :param str param_key: :param str managed_policies_key: :param list statement: """ unique_policies = list( set(bucket.parameters[param_key]["PredefinedBucketPolicies"])) for policy_name in unique_policies: if policy_name not in bucket.module.iam_policies[ managed_policies_key].keys(): LOG.error( f"Policy {policy_name} is not defined as part of possible permissions set" ) continue policies = generate_resource_permissions( bucket.logical_name, bucket.module.iam_policies[managed_policies_key], Sub(f"arn:${{{AWS_PARTITION}}}:s3:::${{{bucket.cfn_resource.title}}}" ), ) statement += policies[policy_name].PolicyDocument["Statement"]
def __init__(self, service, definition: dict, family, settings: ComposeXSettings): self.service = service self._definition = copy.deepcopy(definition) self.family = family self.source_file = set_else_none("SourceFile", self.definition) self._parser_files = set_else_none("ParserFiles", self.definition, alt_value=[]) self._env_vars = set_else_none("EnvironmentVariables", self.definition) self.managed_destinations = [] self.extra_env_vars = set_else_none( "EnvironmentVariables", self.definition, alt_value={} ) if keyisset("ComposeXManagedAwsDestinations", self.definition): for destination_definition in self.definition[ "ComposeXManagedAwsDestinations" ]: if keyisset("log_group_name", destination_definition): self.managed_destinations.append( FireLensCloudWatchManagedDestination( destination_definition, self, settings ) ) elif keyisset("delivery_stream", destination_definition): self.managed_destinations.append( FireLensFirehoseManagedDestination( destination_definition, self, settings ) ) else: LOG.error("Invalid definition for ComposeXManagedAwsDestinations") LOG.error(destination_definition)
def get_topic_config(topic: Topic, account_id: str, resource_id: str) -> dict | None: """ Function to create the mapping definition for SNS topics """ topic_config = {TOPIC_NAME: resource_id} client = topic.lookup_session.client("sns") attributes_mapping = { TOPIC_ARN: "Attributes::TopicArn", TOPIC_KMS_KEY: "Attributes::KmsMasterKeyId", } try: topic_r = client.get_topic_attributes(TopicArn=topic.arn) attributes = attributes_to_mapping(topic_r, attributes_mapping) if keyisset(TOPIC_KMS_KEY, attributes) and not attributes[ TOPIC_KMS_KEY ].startswith("arn:aws"): if attributes[TOPIC_KMS_KEY].startswith("alias/aws"): LOG.warning( f"{topic.module.res_key}.{topic.name} - Topic uses the default AWS CMK." ) else: LOG.warning( f"{topic.module.res_key}.{topic.name} - KMS Key provided is not a valid ARN." ) del attributes[TOPIC_KMS_KEY] topic_config.update(attributes) return topic_config except client.exceptions.QueueDoesNotExist: return None except ClientError as error: LOG.error(error) raise
def lookup_rds_secret(rds_resource, secret_lookup): """ Lookup RDS DB Secret specified :param ecs_composex.compose.x_resources.network_x_resources.DatabaseXResource rds_resource: :param secret_lookup: :return: """ if keyisset("Arn", secret_lookup): client = rds_resource.lookup_session.client("secretsmanager") try: secret_arn = client.describe_secret( SecretId=secret_lookup["Arn"])["ARN"] except client.exceptions.ResourceNotFoundException: LOG.error(f"{rds_resource.module.res_key}.{rds_resource.name}" f" - Secret {secret_lookup['Arn']} not found") raise except ClientError as error: LOG.error(error) raise elif keyisset("Tags", secret_lookup): secret_arn = find_aws_resource_arn_from_tags_api( rds_resource.lookup["secret"], rds_resource.lookup_session, "secretsmanager:secret", ) else: raise LookupError(f"{rds_resource.module.res_key}.{rds_resource.name}" " - Failed to find the DB Secret") if secret_arn: rds_resource.lookup_properties[ rds_resource.db_secret_arn_parameter] = secret_arn
def match_volumes_services_config(service: ComposeService, vol_config: dict, volumes: list): """ Function to map volume config in services and top-level volumes :param service: :param vol_config: :param volumes: :raises LookupError: """ if keyisset("source", vol_config) and vol_config["source"].startswith(r"/"): vol_config["volume"] = None service.volumes.append(vol_config) LOG.info(f"volumes.{vol_config['source']} - Mapped to {service.name}") return else: for volume in volumes: if not keyisset("source", vol_config) and not keyisset( "volume", volume): LOG.error(f"volumes - Failure to process {volume}") continue if volume.name == vol_config["source"]: volume.services.append(service) vol_config["volume"] = volume service.volumes.append(vol_config) LOG.info(f"volumes.{volume.name} - Mapped to {service.name}") return raise LookupError( f"Volume {vol_config['source']} was not found in {[vol.name for vol in volumes]}" )
def get_replica_group_config(resource, cluster_name, session): client = session.client("elasticache") try: cluster_r = client.describe_replication_groups(ReplicationGroupId=cluster_name) cluster = cluster_r["ReplicationGroups"][0] node_r = client.describe_cache_clusters( CacheClusterId=cluster["MemberClusters"][0] ) sg_id = node_r["CacheClusters"][0]["SecurityGroups"][0]["SecurityGroupId"] resource.port_attr = elasticache_params.REPLICA_PRIMARY_PORT return { elasticache_params.REPLICA_PRIMARY_ADDRESS.title: cluster["NodeGroups"][0][ "PrimaryEndpoint" ]["Address"], elasticache_params.REPLICA_PRIMARY_PORT.title: cluster["NodeGroups"][0][ "PrimaryEndpoint" ]["Port"], elasticache_params.REPLICA_READ_ENDPOINT_ADDRESSES.title: [ cluster["NodeGroups"][0]["ReaderEndpoint"]["Address"] ], elasticache_params.REPLICA_READ_ENDPOINT_PORTS.title: [ cluster["NodeGroups"][0]["ReaderEndpoint"]["Port"] ], elasticache_params.CLUSTER_SG.title: [sg_id], } except client.exceptions.ReplicationGroupNotFoundFault as error: LOG.error(f"Could not fetch information about {cluster_name}") LOG.error(error) return None
def expand_launch_template_tags_specs(lt, tags): """ Function to expand the LaunchTemplate TagSpecifications with defined x-tags. :param lt: the LaunchTemplate object :type: troposphere.ec2.LaunchTemplate :param tags: the Tags as built from x-tags :type tags: troposphere.Tags """ LOG.debug("Setting tags to LaunchTemplate") try: launch_data = getattr(lt, "LaunchTemplateData") if hasattr(launch_data, "TagSpecifications"): tags_specs = getattr(launch_data, "TagSpecifications") if isinstance(tags_specs, list) and tags_specs: for tag_spec in tags_specs: if not isinstance(tag_spec, TagSpecifications): continue original_tags = getattr(tag_spec, "Tags") new_tags = original_tags + tags setattr(tag_spec, "Tags", new_tags) setattr(launch_data, "TagSpecifications", tags_specs) except AttributeError: LOG.error("Failed to get the launch template data") except Exception as error: LOG.error(error)
def get_mod_function(module_name, function_name): """ Function to get function in a given module name from function_name :param module_name: the name of the module in ecs_composex to find and try to import :type module_name: str :param function_name: name of the function to try to get :type function_name: str :return: function, if found, from the module :rtype: function """ composex_module_name = f"ecs_composex.{module_name}" LOG.debug(composex_module_name) function = None try: res_module = import_module(composex_module_name) LOG.debug(res_module) try: function = getattr(res_module, function_name) return function except AttributeError: LOG.info(f"No {function_name} function found - skipping") except ImportError as error: LOG.error(f"Failure to process the module {composex_module_name}") LOG.error(error) return function
def get_db_cluster_engine_parameter_group_defaults(engine_family): """ Returns a dict of all the parameter group parameters and default values :parm str engine_family: Engine family we are getting the cluster settings for, i.e. aurora-mysql5.7 """ client = boto3.client("rds") try: req = client.describe_engine_default_cluster_parameters( DBParameterGroupFamily=engine_family) except ClientError as error: LOG.error(error) return None params_return = {} if "EngineDefaults" in req.keys(): params = req["EngineDefaults"]["Parameters"] for param in params: if ("ParameterValue" in param.keys() and "{" not in param["ParameterValue"] and "IsModifiable" in param.keys() and param["IsModifiable"] is True and not param["ParameterName"].startswith("rds.")): params_return[param["ParameterName"]] = param["ParameterValue"] if param["ParameterName"] == "binlog_format": params_return[param["ParameterName"]] = "MIXED" return params_return
def set_from_x_s3(settings, db, db_stack, bucket_name): """ Function to link the RDS DB to a Bucket defined in x-s3 :param settings: :param db: :param str bucket_name: :return: """ resource = None if not keyisset(S3_KEY, settings.compose_content): raise KeyError( f"No Buckets defined in the Compose file under {S3_KEY}.", settings.compose_content.keys(), ) bucket_name = bucket_name.strip("x-s3::") buckets = settings.compose_content[S3_KEY] if bucket_name not in [res.name for res in buckets.values()]: LOG.error( f"No bucket {bucket_name} in x-s3. Buckets defined: {[res.name for res in buckets.values()]}" ) return for resource in buckets.values(): if bucket_name == resource.name: break if not resource: return if resource.cfn_resource: return get_s3_bucket_arn_from_resource(db_stack, resource) elif resource.lookup and keyisset("s3", settings.mappings): add_update_mapping(db_stack.stack_template, "s3", settings.mappings["s3"]) return resource.lookup_properties[S3_BUCKET_ARN]
def lookup_cluster(self, session): """ Define the ECS Cluster properties and definitions from ECS API. :param boto3.session.Session session: Boto3 session to make API calls. :return: The cluster details :rtype: dict """ if not isinstance(self.lookup, (str, dict)): raise TypeError("The value for Lookup must be", str, dict, "Got", type(self.lookup)) ecs_session = session if isinstance(self.lookup, dict): if keyisset("RoleArn", self.lookup): ecs_session = get_assume_role_session( session, self.lookup["RoleArn"], session_name="EcsClusterLookup@ComposeX", ) cluster_name = self.lookup["ClusterName"] else: cluster_name = self.lookup try: clusters = list_all_ecs_clusters(session=ecs_session) cluster_names = [ CLUSTER_NAME_FROM_ARN.match(c_name).group("name") for c_name in clusters ] clusters_config = describe_all_ecs_clusters_from_ccapi( clusters, return_as_map=True, use_cluster_name=True, session=ecs_session) if cluster_name not in clusters_config.keys(): raise LookupError( f"Failed to find {cluster_name}. Available clusters are", cluster_names, ) the_cluster = clusters_config[cluster_name] LOG.info( f"x-cluster.{cluster_name} found. Setting {CLUSTER_NAME.title} accordingly." ) self.mappings = { CLUSTER_NAME.title: { "Name": the_cluster["ClusterName"] } } self.set_cluster_mappings(the_cluster) self.capacity_providers = evaluate_capacity_providers(the_cluster) if self.capacity_providers: self.default_strategy_providers = get_default_capacity_strategy( the_cluster) self.platform_override = evaluate_fargate_is_set( self.capacity_providers, the_cluster) self.cluster_identifier = FindInMap(self.mappings_key, CLUSTER_NAME.title, "Name") except ClientError as error: LOG.error(error) raise
def set_healthcheck_definition(props, target_definition): """ :param dict props: :param dict target_definition: :return: """ healthcheck_props = { "HealthCheckEnabled": Ref(AWS_NO_VALUE), "HealthCheckIntervalSeconds": Ref(AWS_NO_VALUE), "HealthCheckPath": Ref(AWS_NO_VALUE), "HealthCheckPort": Ref(AWS_NO_VALUE), "HealthCheckProtocol": Ref(AWS_NO_VALUE), "HealthCheckTimeoutSeconds": Ref(AWS_NO_VALUE), "HealthyThresholdCount": Ref(AWS_NO_VALUE), } required_mapping = ( "HealthCheckPort", "HealthCheckProtocol", ) required_rex = re.compile( r"^([\d]{2,5}):(HTTPS|HTTP|TCP_UDP|TCP|TLS|UDP)$") healthcheck_reg = re.compile( r"(^(?:[\d]{2,5}):(?:HTTPS|HTTP|TCP_UDP|TCP|TLS|UDP)):?" r"((?:[\d]{1}|10):(?:[\d]{1}|10):[\d]{1,3}:[\d]{1,3})?:" r"?((?:/[\S][^:]+.$)|(?:/[\S]+)(?::)(?:(?:[\d]{1,4},?){1,}.$)|(?:(?:[\d]{1,4},?){1,}.$))?" ) healthcheck_definition = set_else_none("healthcheck", target_definition) if isinstance(healthcheck_definition, str): groups = healthcheck_reg.search(healthcheck_definition).groups() if not groups[0]: raise ValueError( "You need to define at least the Protocol and port for healthcheck" ) for count, value in enumerate(required_rex.match(groups[0]).groups()): healthcheck_props[required_mapping[count]] = value if groups[1]: handle_ping_settings(healthcheck_props, groups[1]) if groups[2]: try: handle_path_settings(healthcheck_props, groups[2]) except ValueError: LOG.error(target_definition["name"], target_definition["healthcheck"]) raise elif isinstance(healthcheck_definition, dict): healthcheck_props.update(healthcheck_definition) if keyisset("Matcher", healthcheck_definition): healthcheck_props["Matcher"] = Matcher( **healthcheck_definition["Matcher"]) else: raise TypeError( healthcheck_definition, type(healthcheck_definition), "must be one of", (str, dict), ) props.update(healthcheck_props)
def s3_to_firehose( resource: Bucket, dest_resource: DeliveryStream, dest_resource_stack, settings: ComposeXSettings, ) -> None: """ Updates :param Bucket resource: :param DeliveryStream dest_resource: :param dest_resource_stack: :param settings: :return: """ if not dest_resource.cfn_resource: LOG.error( f"{dest_resource.module.res_key}.{dest_resource.name} - Not a new resource" ) for prop_path, bucket_param in FIREHOSE_PROPERTIES.items(): prop_attr = get_dest_resource_nested_property( prop_path, dest_resource.cfn_resource) if skip_if(resource, prop_attr): continue bucket_id = resource.attributes_outputs[bucket_param] if resource.cfn_resource: add_parameters(dest_resource_stack.stack_template, [bucket_id["ImportParameter"]]) setattr( prop_attr[0], prop_attr[1], Ref(bucket_id["ImportParameter"]), ) dest_resource.stack.Parameters.update( {bucket_id["ImportParameter"].title: bucket_id["ImportValue"]}) arn_pointer = Ref(bucket_id["ImportParameter"]) elif not resource.cfn_resource and resource.mappings: add_update_mapping( dest_resource.stack.stack_template, resource.module.mapping_key, settings.mappings[resource.module.mapping_key], ) setattr(prop_attr[0], prop_attr[1], bucket_id["ImportValue"]) arn_pointer = bucket_id["ImportValue"] else: raise ValueError( resource.module.mapping_key, resource.name, "Unable to determine if new or lookup", ) map_x_resource_perms_to_resource( dest_resource, arn_value=arn_pointer, access_definition="s3destination", access_subkey="kinesis_firehose", resource_policies=get_access_types(resource.module.mod_key), resource_mapping_key=resource.module.mapping_key, ) dest_resource.ensure_iam_policies_dependencies()
def handle_cross_account_permissions( family: ComposeFamily, service: ComposeService, settings: ComposeXSettings, parameter_name: str, config_value: str, ): """ Function to automatically add cross-account role access for FireHose to the specified role ARN :param family: :param service: :param settings: :param parameter_name: :param config_value: :return: """ try: validate_iam_role_arn(config_value) except ValueError: LOG.error( f"{family.name}.{service.name} - FireLens config for firehose role_arn is invalid" ) raise policy_title = ( f"{family.logical_name}{service.logical_name}LoggingFirehoseCrossAccount" ) if policy_title in family.template.resources: policy = family.template.resources[policy_title] resource = policy.PolicyDocument["Statement"][0]["Resource"] if isinstance(resource, str): resource = [resource] if config_value not in resource: policy.PolicyDocument["Statement"][0]["Resource"].append( config_value) else: policy = PolicyType( policy_title, PolicyName=Sub( f"{family.logical_name}{service.logical_name}FireHoseCrossAccountAccess${{STACK_ID}}", STACK_ID=STACK_ID_SHORT, ), PolicyDocument={ "Version": "2012-10-17", "Statement": [{ "Sid": "LoggingFirehoseCrossAccount", "Effect": "Allow", "Action": ["sts:AssumeRole"], "Resource": [config_value], }], }, Roles=family.iam_manager.task_role.name, ) add_resource(family.template, policy) return config_value
def lookup_service_discovery_namespace(zone: PrivateNamespace, session: Session, ns_id: str = None) -> dict: """ Function to find and get the PrivateDnsNamespace properties needed by other resources :param PrivateNamespace zone: :param boto3.session.Session session: :param str ns_id: :return: The properties we need :rtype: dict """ client = session.client("servicediscovery") try: namespaces = get_all_dns_namespaces(session) if zone.zone_name not in [z["Name"] for z in namespaces]: raise LookupError("No private namespace found for zone", zone.name, zone.zone_name) zone_r = None if not ns_id: for l_zone in namespaces: if zone.zone_name == l_zone["Name"]: the_zone = l_zone zone_r = client.get_namespace(Id=the_zone["Id"]) break else: zone_r = client.get_namespace(Id=ns_id) if not zone_r: raise LookupError( f"{zone.module.res_key}.{zone.name} - Failed to lookup {zone.zone_name}" ) properties = zone_r["Namespace"]["Properties"] if zone_r["Namespace"]["Type"] == "HTTP": raise TypeError( "Unsupported CloudMap namespace HTTP. " "Only DNS namespaces, private or public, are supported") return { PRIVATE_DNS_ZONE_ID: properties["DnsProperties"]["HostedZoneId"], PRIVATE_DNS_ZONE_NAME: LAST_DOT_RE.sub("", properties["HttpProperties"]["HttpName"]), PRIVATE_NAMESPACE_ID: zone_r["Namespace"]["Id"], } except client.exceptions.NamespaceNotFound: LOG.error(f"Namespace not found for {zone.name}") raise except client.exceptions.InvalidInput: LOG.error("Failed to retrieve the zone info") raise
def upload(self): """ Method to handle uploading the files to S3. """ if not self.can_upload and not self.no_upload: LOG.error("BucketName was not specified, not attempting upload") elif self.no_upload: LOG.debug("No Upload is true. Not uploading") else: self.url = upload_template(self.body, self.bucket, self.file_name, mime=self.mime, validate=False) LOG.info(f"{self.file_name} uploaded successfully to {self.url}")