def add_security_group_ingress(service_stack: ComposeXStack, db_name: str,
sg_id, port):
"""
Function to add a SecurityGroupIngress rule into the ECS Service template
:param ecs_composex.ecs.ServicesStack service_stack: The root stack for the services
:param str db_name: the name of the database to use for imports
:param sg_id: The security group Id to use for ingress. DB Security group, not service's
:param port: The port for Ingress to the DB.
"""
service_template = service_stack.stack_template
add_resource(
service_template,
SecurityGroupIngress(
f"AllowFrom{service_stack.title}to{db_name}",
template=service_template,
GroupId=sg_id,
FromPort=port,
ToPort=port,
Description=Sub(f"Allow FROM {service_stack.title} TO {db_name}"),
SourceSecurityGroupId=GetAtt(service_template.resources[SG_T],
"GroupId"),
SourceSecurityGroupOwnerId=Ref("AWS::AccountId"),
IpProtocol="6",
),
)
def add_security_group(family) -> None:
"""
Creates a new EC2 SecurityGroup and assigns to ecs_service.network_settings
Adds the security group to the family template resources.
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
"""
family.service_networking.security_group = SecurityGroup(
SG_T,
GroupDescription=Sub(
f"SG for ${{{SERVICE_NAME.title}}} - ${{STACK_NAME}}",
STACK_NAME=define_stack_name(),
),
Tags=Tags({
"Name":
Sub(
f"${{{SERVICE_NAME.title}}}-${{STACK_NAME}}",
STACK_NAME=define_stack_name(),
),
"StackName":
StackName,
"MicroserviceName":
Ref(SERVICE_NAME),
}),
VpcId=Ref(VPC_ID),
)
add_resource(family.template, family.service_networking.security_group)
def create_alarms(template: Template, new_alarms: list[Alarm]) -> None:
"""
Main function to create new alarms
Rules out CompositeAlarms first, creates "Simple" alarms, and then link these to ComopsiteAlarms if so declared.
"""
for alarm in new_alarms:
if (alarm.properties and not alarm.parameters or
(alarm.parameters
and not keyisset("CompositeExpression", alarm.parameters))):
try:
import_record_properties(
alarm.properties,
CompositeAlarm,
ignore_missing_required=False,
)
except KeyError:
props = import_record_properties(alarm.properties, CWAlarm)
alarm.cfn_resource = CWAlarm(alarm.logical_name, **props)
if alarm.cfn_resource.title not in template.resources:
alarm.init_outputs()
alarm.generate_outputs()
add_resource(template, alarm.cfn_resource)
add_outputs(template, alarm.outputs)
elif alarm.parameters and keyisset("CompositeExpression",
alarm.parameters):
continue
add_composite_alarms(template, new_alarms)
def define_table(table, template):
"""
Function to create the DynamoDB table resource
:param table:
:type table: ecs_composex.common.compose_resources.Table
"""
table_props = import_record_properties(table.properties, dynamodb.Table)
table_props.update({
"Metadata":
metadata,
"Tags":
Tags(
Name=table.name,
ResourceName=table.logical_name,
CreatedByComposex=True,
RootStackName=Ref(ROOT_STACK_NAME),
),
})
cfn_table = dynamodb.Table(table.logical_name, **table_props)
table.cfn_resource = cfn_table
if table.scaling:
add_autoscaling(table, template)
table.init_outputs()
table.generate_outputs()
add_resource(template, table.cfn_resource)
add_outputs(template, table.outputs)
def create_log_group(
family: ComposeFamily,
group_name,
grant_task_role_access: bool = False,
) -> LogGroup:
"""
Function to create a new Log Group for the services
:return:
"""
if LOG_GROUP_T not in family.template.resources:
svc_log = LogGroup(
LOG_GROUP_T,
RetentionInDays=Ref(LOG_GROUP_RETENTION),
LogGroupName=group_name,
)
add_resource(family.template, svc_log)
else:
svc_log = family.template.resources[LOG_GROUP_T]
roles = [family.iam_manager.exec_role.name]
if grant_task_role_access:
roles.append(family.iam_manager.task_role.name)
define_iam_permissions(
"logs",
family,
family.template,
"CloudWatchLogsAccess",
LOGGING_IAM_PERMISSIONS_MODEL,
access_definition="LogGroupOwner",
resource_arns=[GetAtt(svc_log, "Arn")],
roles=roles,
)
return svc_log
def handle_cross_account_permissions(
family: ComposeFamily,
service: ComposeService,
settings: ComposeXSettings,
parameter_name: str,
config_value: str,
):
"""
Function to automatically add cross-account role access for FireHose to the specified role ARN
:param family:
:param service:
:param settings:
:param parameter_name:
:param config_value:
:return:
"""
try:
validate_iam_role_arn(config_value)
except ValueError:
LOG.error(
f"{family.name}.{service.name} - FireLens config for firehose role_arn is invalid"
)
raise
policy_title = (
f"{family.logical_name}{service.logical_name}LoggingFirehoseCrossAccount"
)
if policy_title in family.template.resources:
policy = family.template.resources[policy_title]
resource = policy.PolicyDocument["Statement"][0]["Resource"]
if isinstance(resource, str):
resource = [resource]
if config_value not in resource:
policy.PolicyDocument["Statement"][0]["Resource"].append(
config_value)
else:
policy = PolicyType(
policy_title,
PolicyName=Sub(
f"{family.logical_name}{service.logical_name}FireHoseCrossAccountAccess${{STACK_ID}}",
STACK_ID=STACK_ID_SHORT,
),
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Sid": "LoggingFirehoseCrossAccount",
"Effect": "Allow",
"Action": ["sts:AssumeRole"],
"Resource": [config_value],
}],
},
Roles=family.iam_manager.task_role.name,
)
add_resource(family.template, policy)
return config_value
def add_composite_alarms(template: Template, new_alarms: list[Alarm]) -> None:
for alarm in new_alarms:
if not alarm.cfn_resource and ((alarm.parameters and keyisset(
"CompositeExpression", alarm.parameters)) or alarm.properties):
alarm.is_composite = True
create_composite_alarm(alarm, new_alarms)
add_resource(template, alarm.cfn_resource)
alarm.init_outputs()
alarm.generate_outputs()
add_outputs(template, alarm.outputs)
def add_x_resources(settings: ComposeXSettings) -> None:
"""
Processes the modules / resources that are defining the environment settings
"""
for name, module in settings.mod_manager.modules.items():
LOG.info(f"Processing {name}")
x_stack = module.stack_class(
module.mapping_key,
settings=settings,
module=module,
Parameters={ROOT_STACK_NAME_T: Ref(AWS_STACK_NAME)},
)
if x_stack and x_stack.is_void:
settings.x_resources_void.append({module.res_key: x_stack})
elif (x_stack and hasattr(x_stack, "title")
and hasattr(x_stack, "stack_template") and not x_stack.is_void):
add_resource(settings.root_stack.stack_template, x_stack)
def create_registry(family, namespace, port_config, settings):
"""
Creates the settings for the ECS Service Registries and adds the resources to the appropriate template
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
:param ecs_composex.cloudmap.cloudmap_stack.PrivateNamespace namespace:
:param dict port_config:
:param ecs_composex.common.settings.ComposeXSettings settings:
"""
if family.ecs_service.registries:
LOG.warn(f"{family.name} already has a CloudMap mapping. "
f"Only one can be set. Ignoring mapping to {namespace.name}")
return
if namespace.cfn_resource:
add_parameters(
family.template,
[
namespace.attributes_outputs[PRIVATE_NAMESPACE_ID]
["ImportParameter"]
],
)
family.stack.Parameters.update({
namespace.attributes_outputs[PRIVATE_NAMESPACE_ID]["ImportParameter"].title:
namespace.attributes_outputs[PRIVATE_NAMESPACE_ID]["ImportValue"]
})
namespace_id = Ref(namespace.attributes_outputs[PRIVATE_NAMESPACE_ID]
["ImportParameter"])
elif namespace.lookup_properties:
add_update_mapping(
family.template,
namespace.module.mapping_key,
settings.mappings[namespace.module.mapping_key],
)
namespace_id = namespace.attributes_outputs[PRIVATE_NAMESPACE_ID][
"ImportValue"]
else:
raise AttributeError(
f"{namespace.module.res_key}.{namespace.name} - Cannot define if new or lookup !?"
)
sd_service = SdService(
f"{namespace.logical_name}EcsServiceDiscovery{family.logical_name}",
Description=Sub(f"{family.name} service"),
NamespaceId=namespace_id,
HealthCheckCustomConfig=HealthCheckCustomConfig(FailureThreshold=1.0),
DnsConfig=DnsConfig(
RoutingPolicy="MULTIVALUE",
NamespaceId=Ref(AWS_NO_VALUE),
DnsRecords=[
DnsRecord(TTL="15", Type="A"),
DnsRecord(TTL="15", Type="SRV"),
],
),
Name=family.family_hostname,
)
service_registry = ServiceRegistry(
f"ServiceRegistry{port_config['target']}",
RegistryArn=GetAtt(sd_service, "Arn"),
Port=int(port_config["target"]),
)
add_resource(family.template, sd_service)
family.ecs_service.registries.append(service_registry)
def add_managed_ssm_parameter(family: ComposeFamily,
settings: ComposeXSettings,
content: dict) -> SsmParameter:
"""
Handles x-logging.FireLens.Advanced.Rendered
:param family:
:param settings:
:param content:
"""
ssm_parameter_title = f"{family.logical_name}FireLensConfigurationSsm"
ssm_parameter_definition = {
"Properties": {
"DataType": "text",
"Type": "String",
"Value": dumps(content),
},
"MacroParameters": {
"EncodeToBase64": True
},
"Services": {
family.name: {
"Access": "RO"
}
},
}
if "x-ssm_parameter" not in settings.mod_manager.modules:
ssm_module = settings.mod_manager.add_module("x-ssm_parameter")
settings.compose_content[ssm_module.res_key]: dict = {
ssm_parameter_title: ssm_parameter_definition
}
else:
ssm_module = settings.mod_manager.modules["x-ssm_parameter"]
settings.compose_content[ssm_module.res_key].update(
{ssm_parameter_title: ssm_parameter_definition})
if not ssm_module:
raise LookupError("Failed to import x-ssm_parameter module!")
if ssm_module.mapping_key not in settings.stacks:
ssm_stack = ssm_module.stack_class(ssm_module.mod_key, settings,
ssm_module)
settings.stacks[ssm_module.mapping_key] = ssm_stack
add_resource(settings.root_stack.stack_template, ssm_stack)
ssm_parameter = settings.compose_content[
ssm_module.res_key][ssm_parameter_title]
else:
ssm_stack = settings.stacks[ssm_module.mapping_key]
ssm_parameter = SsmParameter(ssm_parameter_title,
ssm_parameter_definition, ssm_module,
settings)
render_new_parameters([ssm_parameter], ssm_stack)
ssm_parameter.stack = ssm_stack
ssm_parameter.init_outputs()
ssm_parameter.generate_outputs()
add_resource(ssm_parameter.stack.stack_template,
ssm_parameter.cfn_resource)
ssm_parameter.to_ecs(settings, settings.mod_manager)
settings.compose_content[ssm_module.res_key][
ssm_parameter.name] = ssm_parameter
return ssm_parameter
def create_autoscaling_target_and_policy(
table: Table,
template: Template,
scalable_property: str,
scale_definition: dict,
index: str = None,
) -> tuple:
"""
Defines the autoscaling target and policy for the a given resource and dimension.
:param Table table:
:param Template template:
:param str scalable_property:
:param dict scale_definition:
:param str index:
:return: The target and the associated policy
"""
property_mapping: dict = {
"WriteCapacityUnits": {
"PredefinedMetricType": "DynamoDBWriteCapacityUtilization"
},
"ReadCapacityUnits": {
"PredefinedMetricType": "DynamoDBReadCapacityUtilization"
},
}
scablable_resource = (
Sub(f"table/${{{table.cfn_resource.title}}}")
if not index
else Sub(f"table/${{{table.cfn_resource.title}}}/index/{index}")
)
target_title = (
f"{table.logical_name}{scalable_property}ScalableTarget"
if not index
else f"{table.logical_name}{scalable_property}Index{index}ScalableTarget"
)
scaling_target = ScalableTarget(
target_title,
MinCapacity=scale_definition["MinCapacity"],
MaxCapacity=scale_definition["MaxCapacity"],
ServiceNamespace="dynamodb",
ScalableDimension=f"dynamodb:table:{scalable_property}"
if not index
else f"dynamodb:index:{scalable_property}",
RoleARN=Sub(
"arn:aws:iam::${AWS::AccountId}:role/aws-service-role/"
"dynamodb.application-autoscaling.${AWS::URLSuffix}/"
"AWSServiceRoleForApplicationAutoScaling_DynamoDBTable"
),
ResourceId=scablable_resource,
)
scaling_policy = ScalingPolicy(
f"{scaling_target.title}ScalingPolicy",
DependsOn=[scaling_target.title],
PolicyName=f"{scalable_property}AutoScalingPolicy",
PolicyType="TargetTrackingScaling",
ScalingTargetId=Ref(scaling_target),
TargetTrackingScalingPolicyConfiguration=TargetTrackingScalingPolicyConfiguration(
TargetValue=scale_definition["TargetValue"],
ScaleInCooldown=set_else_none(
"ScaleInCooldown", scale_definition, alt_value=60
),
ScaleOutCooldown=set_else_none(
"ScaleOutCooldown", scale_definition, alt_value=60
),
PredefinedMetricSpecification=PredefinedMetricSpecification(
PredefinedMetricType=property_mapping[scalable_property][
"PredefinedMetricType"
]
),
),
)
target = add_resource(template, scaling_target)
policy = add_resource(template, scaling_policy)
return target, policy
def set_ecs_cw_policy(
family: ComposeFamily,
prometheus_parameter: Parameter,
cw_config_parameter: Parameter,
) -> None:
"""
Renders the IAM policy to grant the TaskRole access to CW, ECS and SSM Parameters
:param family: The Service family
:param troposphere.ssm.Parameter prometheus_parameter:
:param troposphere.ssm.Parameter cw_config_parameter:
"""
ecs_sd_policy = PolicyType(
"CWAgentAccessForPrometheusScraping",
PolicyName="CWAgentAccessForPrometheusScraping",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Sid":
"EnableCreationAndManagementOfPrometheusLogEvents",
"Effect":
"Allow",
"Action": ["logs:GetLogEvents", "logs:PutLogEvents"],
"Resource":
Sub(f"arn:${{{AWS_PARTITION}}}:logs:*:${{{AWS_ACCOUNT_ID}}}:"
"log-group:/aws/ecs/containerinsights/*:log-stream:*"),
},
{
"Sid":
"EnableCreationAndManagementOfPrometheusCloudwatchLogGroupsAndStreams",
"Effect":
"Allow",
"Action": [
"logs:CreateLogStream",
"logs:DescribeLogStreams",
"logs:PutRetentionPolicy",
"logs:CreateLogGroup",
],
"Resource":
Sub(f"arn:${{{AWS_PARTITION}}}:logs:*:${{{AWS_ACCOUNT_ID}}}:"
"log-group:/aws/ecs/containerinsights/*"),
},
{
"Sid": "ECSTaskDefinitionsAccess",
"Effect": "Allow",
"Action": ["ecs:DescribeTaskDefinition"],
"Resource": "*",
},
{
"Sid":
"ServiceDiscoveryAccess",
"Effect":
"Allow",
"Action": [
"ecs:DescribeTasks",
"ecs:ListTasks",
"ecs:DescribeContainerInstances",
"ecs:DescribeServices",
"ecs:ListServices",
],
"Resource":
"*",
"Condition": {
"ArnEquals": {
"ecs:cluster":
Sub(f"arn:${{{AWS_PARTITION}}}:ecs:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}"
f":cluster/${{{ecs_params.CLUSTER_NAME.title}}}"
)
}
},
},
{
"Sid":
"ExtractFromCloudWatchAgentServerPolicy",
"Effect":
"Allow",
"Action": ["ssm:GetParameter*"],
"Resource": [
Sub("arn:aws:ssm:*:${AWS::AccountId}:parameter/AmazonCloudWatch-*"
),
Sub(f"arn:${{{AWS_PARTITION}}}:ssm:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}"
f":parameter${{{prometheus_parameter.title}}}"),
Sub(f"arn:${{{AWS_PARTITION}}}:ssm:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}"
f":parameter${{{cw_config_parameter.title}}}"),
],
},
],
},
Roles=[
family.iam_manager.exec_role.name,
family.iam_manager.task_role.name,
],
)
add_resource(family.template, ecs_sd_policy)