def create_kms_template(settings): """ :param ecs_composex.common.settings.ComposeXSettings settings: :return: """ mono_template = False template = build_template("Root template for KMS") if not keyisset(RES_KEY, settings.compose_content): return keys = settings.compose_content[RES_KEY] if len(list(keys.keys())) <= CFN_MAX_OUTPUTS: mono_template = True for key_name in keys: key_res_name = NONALPHANUM.sub("", key_name) key = generate_key(key_name, key_res_name, keys[key_name]) if key: values = [ (KMS_KEY_ARN_T, "Arn", GetAtt(key, "Arn")), (KMS_KEY_ID_T, "Name", Ref(key)), ] outputs = ComposeXOutput(key, values, True) if mono_template: template.add_resource(key) handle_key_settings(template, key, keys[key_name]) template.add_output(outputs.outputs) elif not mono_template: key_template = build_template(f"Template for DynamoDB key {key.title}") key_template.add_resource(key) key_template.add_output(outputs.outputs) key_stack = ComposeXStack(key_res_name, stack_template=key_template) template.add_resource(key_stack) return template
def generate_sqs_root_template(settings): """ Function to create the root DynamdoDB template. :param ecs_composex.common.settings.ComposeXSettings settings: Execution settings. :return: """ mono_template = False output_per_resource = 2 if not keyisset(RES_KEY, settings.compose_content): return None queues = settings.compose_content[RES_KEY] if (len(list(queues.keys())) * output_per_resource) <= CFN_MAX_OUTPUTS: mono_template = True template = build_template("DynamoDB for ECS ComposeX") for queue_name in queues: queue_res_name = NONALPHANUM.sub("", queue_name) queue_def = queues[queue_name] queue = define_queue(queue_name, queue_def, queues, mono_template) if queue: values = [ (SQS_URL, "Url", Ref(queue)), (SQS_ARN_T, "Arn", GetAtt(queue, "Arn")), (SQS_NAME_T, "Name", Ref(queue)), ] outputs = ComposeXOutput(queue, values, duplicate_attr=(not mono_template)) if mono_template: template.add_resource(queue) template.add_output(outputs.outputs) elif not mono_template: parameters = {} if hasattr(queue, "RedrivePolicy"): parameters.update({ DLQ_ARN_T: GetAtt( NONALPHANUM.sub( "", queue_def["Properties"]["RedrivePolicy"] ["deadLetterTargetArn"], ), f"Outputs.{SQS_ARN_T}", ) }) queue_template = build_template( f"Template for SQS queue {queue.title}", [DLQ_ARN]) queue_template.add_resource(queue) queue_template.add_output(outputs.outputs) queue_stack = ComposeXStack( queue_res_name, stack_template=queue_template, stack_parameters=parameters, ) template.add_resource(queue_stack) return template
def generate_sns_templates(settings): """ Entrypoint function to generate the SNS topics templates :param settings: :type settings: ecs_composex.common.settings.ComposeXSettings :return: """ allowed_keys = [TOPICS_KEY, SUBSCRIPTIONS_KEY] res_content = settings.compose_content[RES_KEY] if not set(res_content).issubset(allowed_keys): raise KeyError( "SNS Only supports two types of resources", allowed_keys, "provided", res_content.keys(), ) root_template = build_template("SNS Root Template") res_count = define_resources(res_content) if keyisset(TOPICS_KEY, res_content): add_sns_topics(root_template, settings.compose_content, res_count) if keyisset(SUBSCRIPTIONS_KEY, res_content): pass add_topics_outputs(root_template) return root_template
def __init__(self, title, settings: ComposeXSettings, module: XResourceModule, **kwargs): """ Method to initialize the XStack for Events :param str title: title for the stack :param ecs_composex.common.settings.ComposeXSettings settings: Execution settings :param dict kwargs: """ set_resources(settings, Rule, module) x_resources = settings.compose_content[module.res_key].values() lookup_resources = set_lookup_resources(x_resources) if lookup_resources: warnings.warn( f"{module.res_key} does not support Lookup/Use. You can only create new resources" ) new_resources = set_new_resources(x_resources, False) if new_resources: stack_template = build_template( "Events rules for ComposeX", [CLUSTER_NAME, FARGATE_VERSION], ) super().__init__(title, stack_template, **kwargs) create_events_template(self, settings, new_resources) else: self.is_void = True
def create_root_template(new_resources: list, module_res_key: str) -> Template: """ Function to create the root stack template for profiles :param list[CodeProfiler] new_resources: :param str module_res_key: :return: The template wit the profiles :rtype: troposphere.Template """ root_tpl = build_template(f"Root stack to manage {module_res_key}") for res in new_resources: try: props = import_record_properties(res.properties, ProfilingGroup, ignore_missing_required=False) if res.parameters and keyisset("AppendStackId", res.parameters): props["ProfilingGroupName"] = Sub( f"{res.properties['ProfilingGroupName']}-${{StackId}}", StackId=STACK_ID_SHORT, ) except KeyError: props = import_record_properties(res.properties, ProfilingGroup, ignore_missing_required=True) props["ProfilingGroupName"] = Sub( f"{res.logical_name}-${{StackId}}", StackId=STACK_ID_SHORT) res.cfn_resource = ProfilingGroup(res.logical_name, **props) res.init_outputs() res.generate_outputs() add_outputs(root_tpl, res.outputs) root_tpl.add_resource(res.cfn_resource) return root_tpl
def generate_compute_template(settings): """ Function that generates the Compute resources to run ECS services on top of EC2 :param ComposeXSettings settings: The settings for executio :return: ECS Cluster Template :rtype: troposphere.Template """ template = build_template( "Cluster template generated by ECS Compose X", [ USE_FLEET, USE_ONDEMAND, compute_params.ECS_AMI_ID, compute_params.TARGET_CAPACITY, compute_params.MIN_CAPACITY, compute_params.MAX_CAPACITY, vpc_params.APP_SUBNETS, vpc_params.VPC_ID, CLUSTER_NAME, ], ) template.add_condition( compute_conditions.MAX_IS_MIN_T, compute_conditions.MAX_IS_MIN ) template.add_condition(cfn_conditions.USE_SPOT_CON_T, cfn_conditions.USE_SPOT_CON) launch_template = add_hosts_resources(template) add_spotfleet_stack(template, settings, launch_template) return template
def create_s3_template(new_buckets: list[Bucket], template: Template) -> Template: """ Function to create the root S3 template. :param list new_buckets: :param troposphere.Template template: :return: """ mono_template = False if len(list(new_buckets)) <= COMPOSEX_MAX_OUTPUTS: mono_template = True for bucket in new_buckets: generate_bucket(bucket) if bucket.cfn_resource: bucket.init_outputs() bucket.generate_outputs() bucket_template = template if mono_template: bucket_template.add_resource(bucket.cfn_resource) bucket_template.add_output(bucket.outputs) elif not mono_template: bucket_template = build_template( f"Template for S3 Bucket {bucket.name}") bucket_template.add_resource(bucket.cfn_resource) bucket_template.add_output(bucket.outputs) bucket_stack = ComposeXStack(bucket.logical_name, stack_template=bucket_template) template.add_resource(bucket_stack) evaluate_parameters(bucket, bucket_template) return template
def create_kms_template(template, new_keys, xstack): """ Function to create all the KMS Keys based on their definition :param ecs_composex.common.settings.ComposeXSettings settings: """ mono_template = False if len(new_keys) <= CFN_MAX_OUTPUTS: mono_template = True for key in new_keys: key.stack = xstack key.define_kms_key() if key and key.cfn_resource: key.init_outputs() key.generate_outputs() if mono_template: template.add_resource(key.cfn_resource) key.handle_key_settings(template) template.add_output(key.outputs) elif not mono_template: key_template = build_template( f"Template for KMS key {key.logical_name}") key_template.add_resource(key.cfn_resource) key.handle_key_settings(key_template) key_template.add_output(key.outputs) key_stack = ComposeXStack(key.logical_name, stack_template=key_template) template.add_resource(key_stack)
def __init__(self, title, settings: ComposeXSettings, module: XResourceModule, **kwargs): set_resources(settings, NeptuneDBCluster, module) x_resources = settings.compose_content[module.res_key].values() new_resources = set_new_resources(x_resources, True) lookup_resources = set_lookup_resources(x_resources) if new_resources: stack_template = build_template( "Root template for Neptune by ComposeX", [VPC_ID, STORAGE_SUBNETS]) super().__init__(title, stack_template, **kwargs) create_neptune_template(stack_template, new_resources, settings, self) else: self.is_void = True if lookup_resources: if not keyisset(module.mapping_key, settings.mappings): settings.mappings[module.mapping_key] = {} for resource in lookup_resources: resource.lookup_resource( NEPTUNE_DB_CLUSTER_ARN_RE, get_db_cluster_config, CfnDBCluster.resource_type, "rds:cluster", ) resource.generate_cfn_mappings_from_lookup_properties() resource.generate_outputs() settings.mappings[module.mapping_key].update( {resource.logical_name: resource.mappings}) for resource in settings.compose_content[module.res_key].values(): resource.stack = self
def __init__(self, title: str, settings: ComposeXSettings, module: XResourceModule, **kwargs): """ :param str title: :param ecs_composex.common.settings.ComposeXSettings settings: :param dict kwargs: """ set_resources(settings, UserPool, module) x_resources = settings.compose_content[module.res_key].values() lookup_resources = set_lookup_resources(x_resources) if lookup_resources: resolve_lookup(lookup_resources, settings, module) new_resources = set_new_resources(x_resources, False) if new_resources: LOG.error( f"{module.res_key} does not support new resources creation yet." ) stack_template = build_template( f"Root stack to manage {module.mod_key}") super().__init__(title, stack_template, **kwargs) self.is_void = True else: self.is_void = True for resource in x_resources: resource.stack = self
def __init__(self, title, settings: ComposeXSettings, module: XResourceModule, **kwargs): """ :param str title: Name of the stack :param ecs_composex.common.settings.ComposeXSettings settings: :param dict kwargs: """ set_resources(settings, Queue, module) x_resources = settings.compose_content[module.res_key].values() lookup_resources = set_lookup_resources(x_resources) if lookup_resources: resolve_lookup(lookup_resources, settings, module) new_resources = set_new_resources(x_resources, True) if new_resources: template = build_template( "SQS template generated by ECS Compose-X") if lookup_resources: add_update_mapping(template, module.mapping_key, settings.mappings[module.mapping_key]) super().__init__(title, stack_template=template, **kwargs) render_new_queues(settings, new_resources, x_resources, self, template) else: self.is_void = True for resource in settings.compose_content[module.res_key].values(): resource.stack = self
def create_dynamodb_template(new_tables, template, self_stack): """ Function to create the root DynamdoDB template. :param list new_tables: :param troposhere.Template template: :param ComposeXStack self_stack: :return: """ total_outputs = sum( len(table.attributes_outputs.keys()) for table in new_tables) mono_template = False if total_outputs <= CFN_MAX_OUTPUTS: mono_template = True for table in new_tables: if mono_template: table.stack = self_stack define_table(table, template) elif not mono_template: table_template = build_template( f"Template for DynamoDB table {table.title}") table_stack = ComposeXStack(table.logical_name, stack_template=table_template) table.stack = table_stack define_table(table, table_template) template.add_resource(table_stack) return template
def create_efs_stack(settings, new_resources): """ Function to create the root stack and add EFS FS. :param ecs_composex.common.settings.ComposeXSettings settings: :param list new_resources: :return: Root template for EFS :rtype: troposphere.Template """ template = build_template("Root for EFS built by ECS Compose-X", [FS_PORT]) for res in new_resources: res_cfn_props = import_record_properties(res.properties, FileSystem) res.cfn_resource = FileSystem(res.logical_name, **res_cfn_props) res.db_sg = SecurityGroup( f"{res.logical_name}SecurityGroup", GroupName=Sub(f"{res.logical_name}EfsSg"), GroupDescription=Sub(f"SG for EFS {res.cfn_resource.title}"), VpcId=Ref(VPC_ID), ) template.add_resource(res.cfn_resource) template.add_resource(res.db_sg) res.init_outputs() res.generate_outputs() template.add_output(res.outputs) return template
def init_elbv2_template(): """ Function to create a new root ELBv2 stack :return: """ lb_params = [VPC_ID, APP_SUBNETS, PUBLIC_SUBNETS] template = build_template("elbv2 root template for ComposeX", lb_params) return template
def init_doc_db_template(): """ Function to generate the base of the DocDB template. :return: the root template :rtype: troposphere.Template """ template = build_template("Root template for DocumentDB for ComposeX", [VPC_ID, STORAGE_SUBNETS]) return template
def init_vpc_template() -> troposphere.Template: """ Simple wrapper function to create the VPC Template :rtype: troposhere.Template """ template = build_template("Vpc Template generated via ECS Compose-X", ) template.add_mapping("AwsLbAccounts", aws_mappings.AWS_LB_ACCOUNTS) return template
def init_rds_root_template(): """ Function to generate the root template for RDS :return: template :rtype: troposphere.Template """ template = build_template("RDS Root Template", [VPC_ID, STORAGE_SUBNETS]) return template
def init_database_template(db): """ Function to initialize the DB Template :param db: The DB definition :return: template :rtype: troposphere.Template """ template = build_template( f"Template for RDS DB {db.name}", [ VPC_ID, DB_ENGINE_NAME, DB_ENGINE_VERSION, DB_NAME, DB_USERNAME, DB_SNAPSHOT_ID, DB_PASSWORD_LENGTH, DB_INSTANCE_CLASS, DB_STORAGE_CAPACITY, DB_STORAGE_TYPE, STORAGE_SUBNETS, ], ) template.add_condition( rds_conditions.USE_DB_SNAPSHOT_CON_T, rds_conditions.USE_DB_SNAPSHOT_CON ) template.add_condition( rds_conditions.NOT_USE_DB_SNAPSHOT_CON_T, rds_conditions.NOT_USE_DB_SNAPSHOT_CON, ) template.add_condition( rds_conditions.USE_CLUSTER_CON_T, rds_conditions.USE_CLUSTER_CON ) template.add_condition( rds_conditions.NOT_USE_CLUSTER_CON_T, rds_conditions.NOT_USE_CLUSTER_CON ) template.add_condition( rds_conditions.USE_CLUSTER_AND_SNAPSHOT_CON_T, rds_conditions.USE_CLUSTER_AND_SNAPSHOT_CON, ) template.add_condition( rds_conditions.USE_CLUSTER_NOT_SNAPSHOT_CON_T, rds_conditions.USE_CLUSTER_NOT_SNAPSHOT_CON, ) template.add_condition( rds_conditions.NOT_USE_CLUSTER_USE_SNAPSHOT_CON_T, rds_conditions.NOT_USE_CLUSTER_USE_SNAPSHOT_CON, ) template.add_condition( rds_conditions.USE_CLUSTER_OR_SNAPSHOT_CON_T, rds_conditions.USE_CLUSTER_OR_SNAPSHOT_CON, ) return template
def test_zone_create(content, zone_create): """ Tests zone lookup :return: """ print(content) updated_content = content.copy() updated_content.update(zone_create) settings = create_settings(updated_content, "x_dns") root_stack = ComposeXStack("root", build_template()) DnsSettings(root_stack, settings, Ref("VpcId"))
def __init__(self, title, settings: ComposeXSettings, module: XResourceModule, **kwargs): params = { CLUSTER_NAME.title: settings.ecs_cluster, } stack_template = build_template("Root template for Dashboards", [CLUSTER_NAME]) super().__init__(title, stack_template, stack_parameters=params, **kwargs) create_dashboards(settings, self, module)
def init_database_template(db_name): """ Function to initialize the DB Template :param str db_name: Name of the DB as defined in compose file :return: template :rtype: troposphere.Template """ template = build_template( f"Template for RDS DB {db_name}", [ VPC_ID, DB_ENGINE_NAME, DB_ENGINE_VERSION, STORAGE_SUBNETS, DBS_SUBNET_GROUP, DB_NAME, DB_USERNAME, DB_SNAPSHOT_ID, DB_PASSWORD_LENGTH, DB_INSTANCE_CLASS, DB_STORAGE_CAPACITY, DB_STORAGE_TYPE, ], ) template.add_condition(rds_conditions.DBS_SUBNET_GROUP_CON_T, rds_conditions.DBS_SUBNET_GROUP_CON) template.add_condition(rds_conditions.USE_DB_SNAPSHOT_CON_T, rds_conditions.USE_DB_SNAPSHOT_CON) template.add_condition(rds_conditions.NOT_USE_DB_SNAPSHOT_CON_T, rds_conditions.NOT_USE_DB_SNAPSHOT_CON) template.add_condition(rds_conditions.USE_CLUSTER_CON_T, rds_conditions.USE_CLUSTER_CON) template.add_condition(rds_conditions.NOT_USE_CLUSTER_CON_T, rds_conditions.NOT_USE_CLUSTER_CON) template.add_condition( rds_conditions.USE_CLUSTER_AND_SNAPSHOT_CON_T, rds_conditions.USE_CLUSTER_AND_SNAPSHOT_CON, ) template.add_condition( rds_conditions.USE_CLUSTER_NOT_SNAPSHOT_CON_T, rds_conditions.USE_CLUSTER_NOT_SNAPSHOT_CON, ) template.add_condition( rds_conditions.NOT_USE_CLUSTER_USE_SNAPSHOT_CON_T, rds_conditions.NOT_USE_CLUSTER_USE_SNAPSHOT_CON, ) template.add_condition( rds_conditions.USE_CLUSTER_OR_SNAPSHOT_CON_T, rds_conditions.USE_CLUSTER_OR_SNAPSHOT_CON, ) create_db_subnet_group(template, True) return template
def init_stack_for_records(self, root_stack) -> None: """ When creating new Route53 records, if the x-route53 where looked up, we need to initialize the Route53 stack :param ComposeXStack root_stack: The root stack """ if self.module.mapping_key not in root_stack.stack_template.resources: from ecs_composex.route53.route53_stack import XStack stack_template = build_template(self.stack.stack_title) super(XStack, self.stack).__init__(self.module.mapping_key, stack_template) self.stack.is_void = False root_stack.stack_template.add_resource(self.stack)
def test_zone_lookup(content, zone_lookup): """ Tests zone lookup :return: """ updated_content = content.copy() updated_content.update(zone_lookup) settings = create_settings(updated_content, "x_dns") root_stack = ComposeXStack("root", build_template()) try: DnsSettings(root_stack, settings, Ref("VpcId")) except Exception: pass
def init_root_template(): """ Function to initialize the root template :return: template :rtype: troposphere.Template """ template = build_template( "Root template generated via ECS ComposeX", [USE_FLEET, USE_ONDEMAND, CLUSTER_NAME], ) return template
def create_dynamodb_template(settings): """ Function to create the root DynamdoDB template. :param ecs_composex.common.settings.ComposeXSettings settings: :return: """ mono_template = False if not keyisset(RES_KEY, settings.compose_content): return None tables = settings.compose_content[RES_KEY] if len(list(tables.keys())) <= CFN_MAX_OUTPUTS: mono_template = True template = build_template("DynamoDB for ECS ComposeX") for table_name in tables: table_res_name = NONALPHANUM.sub("", table_name) table = generate_table(table_name, table_res_name, tables[table_name]) if table: values = [ (TABLE_ARN_T, "Arn", GetAtt(table, "Arn")), (TABLE_NAME_T, "Name", Ref(table)), ] outputs = ComposeXOutput(table, values, True) if mono_template: template.add_resource(table) template.add_output(outputs.outputs) elif not mono_template: table_template = build_template( f"Template for DynamoDB table {table.title}" ) table_template.add_resource(table) table_template.add_output(outputs.outputs) table_stack = ComposeXStack( table_res_name, stack_template=table_template ) template.add_resource(table_stack) return template
def create_streams_template(new_resources, settings): """ Function to create the root template for Kinesis streams :param list<ecs_composex.kinesis.kinesis_stack.Stream> new_resources: :param ecs_composex.common.settings.ComposeXSettings settings: :return: """ root_template = build_template("Root stack for ecs_composex.kinesis") for res in new_resources: create_new_stream(res) root_template.add_resource(res.cfn_resource) root_template.add_output(res.outputs) return root_template
def create_acm_template(settings): """ Main entrypoint for ACM root template creation :param ecs_composex.common.settings.ComposeXSettings settings: The execution settings :return: root stack template for ACM. :rtype: troposphere.Template """ if not keyisset(RES_KEY, settings.compose_content): return certs = settings.compose_content[RES_KEY] root_acm_tpl = build_template("Root template for ACM") add_certificates(root_acm_tpl, certs) return root_acm_tpl
def __init__( self, title, settings: ComposeXSettings, module: XResourceModule, **kwargs ): set_resources(settings, Rds, module) x_resources = settings.compose_content[module.res_key].values() new_resources = set_new_resources(x_resources, True) lookup_resources = set_lookup_resources(x_resources) if new_resources: stack_template = build_template( "Root stack for RDS DBs", [VPC_ID, STORAGE_SUBNETS] ) super().__init__(title, stack_template, **kwargs) generate_rds_templates(stack_template, new_resources, settings) self.mark_nested_stacks() else: self.is_void = True for resource in settings.compose_content[module.res_key].values(): resource.stack = self if lookup_resources and module.mapping_key not in settings.mappings: settings.mappings[module.mapping_key] = {} for resource in lookup_resources: if keyisset("cluster", resource.lookup): resource.lookup_resource( RDS_DB_CLUSTER_ARN_RE, get_db_cluster_config, CfnDBCluster.resource_type, "rds:cluster", "cluster", ) elif keyisset("db", resource.lookup): resource.lookup_resource( RDS_DB_INSTANCE_ARN_RE, get_db_instance_config, CfnDBInstance.resource_type, "rds:db", "db", ) else: raise KeyError( f"{resource.module.res_key}.{resource.name} - " "You must specify the cluster or instance to lookup" ) if keyisset("secret", resource.lookup): lookup_rds_secret(resource, resource.lookup["secret"]) resource.generate_cfn_mappings_from_lookup_properties() resource.generate_outputs() settings.mappings[module.mapping_key].update( {resource.logical_name: resource.mappings} )
def __init__(self, name: str, settings: ComposeXSettings, **kwargs): stack_template = build_template("Root stack for IAM Roles") add_parameters(stack_template, [CLUSTER_NAME]) super().__init__(name, stack_template, **kwargs) exec_role_managed_policy = add_ecs_execution_role_managed_policy(stack_template) self.Parameters.update( {CLUSTER_NAME.title: settings.ecs_cluster.cluster_identifier} ) new_roles = import_family_roles(settings, exec_role_managed_policy) for role in new_roles: self.stack_template.add_resource(role.cfn_resource) role.stack = self if not role.attributes_outputs: role.generate_outputs() add_outputs(stack_template, role.outputs)
def init_stack_for_resources(self, settings) -> None: """ When creating new Route53 records, if the x-route53 where looked up, we need to initialize the Route53 stack :param ComposeXStack root_stack: The root stack """ if self.stack.is_void: stack_template = build_template("Root stack for x-cloudmap resources") super(XStack, self.stack).__init__(MOD_KEY, stack_template) self.stack.is_void = False add_update_mapping( self.stack.stack_template, self.module.mapping_key, settings.mappings[self.module.mapping_key], )