def generate_ref_env_var(self, target) -> list:
"""
Method to define all the env var of a resource based on its own defined output attributes
"""
if not self.ref_parameter:
LOG.error(
f"{self.module.res_key}.{self.name}. Default ref_parameter not set. Skipping env_vars"
)
return []
env_var_name = ENV_VAR_NAME.sub("",
self.name.upper().replace("-", "_"))
if self.cfn_resource and self.attributes_outputs and self.ref_parameter:
ref_env_var = Environment(
Name=env_var_name,
Value=Ref(self.attributes_outputs[self.ref_parameter]
["ImportParameter"]),
)
ref_param_settings = get_parameter_settings(
self, self.ref_parameter)
add_parameters(target[0].template, [ref_param_settings[1]])
target[0].stack.Parameters.update(
{ref_param_settings[0]: ref_param_settings[2]})
elif self.lookup_properties and self.ref_parameter:
ref_env_var = Environment(
Name=env_var_name,
Value=self.attributes_outputs[self.ref_parameter]
["ImportValue"],
)
else:
raise ValueError(
f"{self.module.res_key}.{self.name} - Unable to set the default env var"
)
return [ref_env_var]
def import_env_variables(environment) -> list:
"""
Function to import Docker compose env variables into ECS Env Variables
:param environment: Environment variables as defined on the ecs_service definition
:type environment: dict
:return: list of Environment
:rtype: list<troposphere.ecs.Environment>
"""
env_vars = []
if isinstance(environment, list):
env_vars_to_map = set_environment_dict_from_list(environment)
elif isinstance(environment, dict):
env_vars_to_map = environment
else:
raise TypeError(
"Enviroment must be a list of string or a dict of key/value where value is a string"
)
for key, value in env_vars_to_map.items():
if not isinstance(value, str):
env_vars.append(Environment(Name=key, Value=str(environment[key])))
else:
env_vars.append(
Environment(
Name=key,
Value=define_string_interpolation(value),
))
return env_vars
def generate_resource_envvars(resource_name, resource, attribute, arn=None):
"""
Function to generate environment variables that can be added to a container definition
shall the ecs_service need to know about the Queue
:param str resource_name: The name of the resource
:param dict resource: The resource definition as defined in docker-compose file.
:param str attribute: the attribute of the resource we are using for Import
:param str arn: The ARN of the resource if already looked up.
:return: environment key/pairs
:rtype: list<troposphere.ecs.Environment>
"""
env_names = []
export_strings = (
generate_export_strings(resource_name, attribute) if not arn else arn
)
if keyisset("Settings", resource) and keyisset("EnvNames", resource["Settings"]):
for env_name in resource["Settings"]["EnvNames"]:
env_names.append(Environment(Name=env_name, Value=export_strings,))
if resource_name not in resource["Settings"]["EnvNames"]:
env_names.append(Environment(Name=resource_name, Value=export_strings,))
else:
env_names.append(Environment(Name=resource_name, Value=export_strings,))
return env_names
def handle_firelens_advanced_settings(
family: ComposeFamily,
settings: ComposeXSettings) -> FireLensFamilyManagedConfiguration:
"""
Handles x-logging.FireLens.Advanced.Rendered
:param ComposeFamily family:
:param ComposeXSettings settings:
"""
advanced_config = FireLensFamilyManagedConfiguration(family, settings)
advanced_config.set_update_ssm_parameter(settings)
env_vars = [
Environment(Name=name, Value=str(value))
for name, value in advanced_config.extra_env_vars.items()
if isinstance(value, (int, float, str, bool))
]
env_vars += [
Environment(Name=name, Value=value)
for name, value in advanced_config.extra_env_vars.items()
if isinstance(value, Ref)
]
extend_container_envvars(
family.logging.firelens_service.container_definition, env_vars)
extend_container_envvars(
family.logging.firelens_config_service.container_definition, env_vars)
family.logging.firelens_config_service.add_to_family(family,
is_dependency=True)
family.logging.firelens_config_service.logging.set_update_log_configuration(
LogDriver="awslogs",
Options={
"awslogs-group": Ref(family.logging.family_log_group),
"awslogs-region": Region,
"awslogs-stream-prefix":
family.logging.firelens_config_service.name,
},
)
set_volumes(family)
family.logging.firelens_service.firelens_config = {
"Type": "fluentbit",
"Options": {
"config-file-value":
f"{advanced_config.volume_mount}{advanced_config.config_file_name}",
"config-file-type": "file",
"enable-ecs-log-metadata": True,
},
}
return advanced_config
def set_update_container_env_var(self, target: tuple, parameter,
env_var_name: str) -> list:
"""
Function that will set or update the value of a given env var from Return value of a resource.
:param tuple target:
:param parameter:
"""
if isinstance(parameter, str):
try:
attr_parameter = self.property_to_parameter_mapping[parameter]
except KeyError:
LOG.error(
f"{self.module.res_key}.{self.name} - No return value {parameter} available."
)
return []
elif isinstance(parameter, Parameter):
attr_parameter = parameter
else:
raise TypeError("parameter is", type(parameter), "must be one of",
[str, Parameter])
env_vars = []
params_to_add = []
attr_id = self.attributes_outputs[attr_parameter]
if self.cfn_resource:
env_vars.append(
Environment(
Name=env_var_name,
Value=Ref(attr_id["ImportParameter"]),
))
params_to_add.append(attr_parameter)
elif self.lookup_properties:
env_vars.append(
Environment(
Name=env_var_name,
Value=attr_id["ImportValue"],
))
if params_to_add:
params_values = {}
settings = [
get_parameter_settings(self, param) for param in params_to_add
]
resource_params_to_add = []
for setting in settings:
resource_params_to_add.append(setting[1])
params_values[setting[0]] = setting[2]
add_parameters(target[0].template, resource_params_to_add)
target[0].stack.Parameters.update(params_values)
return env_vars
def import_env_variables(environment):
"""
Function to import Docker compose env variables into ECS Env Variables
:param environment: Environment variables as defined on the ecs_service definition
:type environment: dict
:return: list of Environment
:rtype: list<troposphere.ecs.Environment>
"""
env_vars = []
for key in environment:
if not isinstance(environment[key], str):
env_vars.append(Environment(Name=key, Value=str(environment[key])))
else:
env_vars.append(Environment(Name=key, Value=environment[key]))
return env_vars
def expand_backends(self, root_stack, services):
"""
Method to set the backends to the service node.
:param ecs_composex.ServicesStack root_stack: the root stack to put a dependency from.
:param dict services: the services in the mesh stack.
"""
backends = []
task_def = self.stack.stack_template.resources[ecs_params.TASK_T]
container_envvars = []
for backend in self.backends:
LOG.debug(backend)
virtual_service = services[backend]
backends_nodes = virtual_service.get_backend_nodes()
LOG.debug(backends_nodes)
self.create_ingress_rule(root_stack, backends_nodes)
root_stack.stack_template.resources[self.stack.title].DependsOn.append(
virtual_service.service.title
)
backend_parameter = Parameter(
f"{backend}VirtualServiceBackend",
template=self.stack.stack_template,
Type="String",
)
self.stack.Parameters.update(
{
backend_parameter.title: GetAtt(
virtual_service.service, "VirtualServiceName"
)
}
)
backends.append(
appmesh.Backend(
VirtualService=appmesh.VirtualServiceBackend(
VirtualServiceName=Ref(backend_parameter)
)
)
)
container_envvars.append(
Environment(
Name=f"{backend.upper()}_BACKEND", Value=Ref(backend_parameter)
)
)
for container in task_def.ContainerDefinitions:
extend_container_envvars(container, env_vars=container_envvars)
node_spec = getattr(self.node, "Spec")
setattr(node_spec, "Backends", backends)
def _gen_container_definitions_for_sidecar(self, sidecar, log_config,
env_config):
cd = dict()
if 'command' in sidecar:
cd['Command'] = sidecar['command']
return ContainerDefinition(
Name=container_name(sidecar.get('name')),
Environment=[
Environment(Name=k, Value=v) for (k, v) in env_config
],
MemoryReservation=int(sidecar.get('memory_reservation')),
Image=sidecar.get('image'),
LogConfiguration=log_config,
Essential=False,
**cd)
def set_add_region_when_external(self):
from troposphere.ecs import Environment
env_var_to_add = Environment(Name="AWS_DEFAULT_REGION", Value=Region)
region_conditional = If(
ecs_conditions.USE_EXTERNAL_LT_T, env_var_to_add, NoValue
)
for service in self.services:
environment = getattr(service.container_definition, "Environment")
if (
not environment
or environment == NoValue
or not isinstance(environment, list)
):
environment = []
setattr(service.container_definition, "Environment", environment)
if "AWS_DEFAULT_REGION" not in [
_env.Name for _env in environment if isinstance(_env, Environment)
]:
environment.append(region_conditional)
def ecs_redirect(self, cluster, url):
self._ecs_redirect = True
self.template.add_resource(
TaskDefinition(
"RedirectTaskDef",
Volumes=[],
Family=Sub("${AWS::StackName}-redirect"),
NetworkMode="bridge",
ContainerDefinitions=[
ContainerDefinition(
Name="redirect",
Cpu=1,
Environment=[Environment(Name="REDIRECT", Value=url)],
Essential=True,
Hostname=Sub("${AWS::StackName}-redirect"),
Image="cusspvz/redirect:0.0.2",
Memory=512,
MemoryReservation=128,
PortMappings=[
PortMapping(ContainerPort=80, Protocol="tcp")
])
]))
self.template.add_resource(
Service("RedirectService",
TaskDefinition=Ref("RedirectTaskDef"),
Cluster=cluster,
DesiredCount=1,
DeploymentConfiguration=DeploymentConfiguration(
MaximumPercent=200, MinimumHealthyPercent=100),
LoadBalancers=[
EcsLoadBalancer(
ContainerName="redirect",
ContainerPort=80,
TargetGroupArn=Ref("DefaultTargetGroup"))
]))
def add_ecs_task(self):
'''
Add ECS Task
'''
self.cfn_template.add_resource(
TaskDefinition(
title=constants.TASK,
Volumes=[Volume(Name='anchore_db_vol')],
TaskRoleArn=GetAtt(constants.TASK_ROLE, 'Arn'),
ContainerDefinitions=[
ContainerDefinition(
Name='anchore-engine',
Hostname='anchore-engine',
Cpu=int('512'),
MemoryReservation=int('1536'),
Essential=bool('true'),
Image=ImportValue(
Sub('${Environment}-${AnchoreEngineImage}')),
PortMappings=[
PortMapping(
ContainerPort=int('8228'),
HostPort=int('8228'),
Protocol='tcp',
),
PortMapping(
ContainerPort=int('8338'),
HostPort=int('8338'),
Protocol='tcp',
),
],
DockerSecurityOptions=['apparmor:docker-default'],
Environment=[
Environment(Name='ANCHORE_HOST_ID',
Value='anchore-engine'),
Environment(Name='ANCHORE_ENDPOINT_HOSTNAME',
Value='anchore-engine'),
Environment(Name='ANCHORE_DB_HOST',
Value='anchore-db'),
Environment(Name='ANCHORE_DB_PASSWORD',
Value=Ref('AnchoreDBPassword')),
Environment(Name='AWS_DEFAULT_REGION',
Value=Ref('AWS::Region')),
Environment(Name='region',
Value=Ref('AWS::Region')),
],
LogConfiguration=LogConfiguration(
LogDriver='awslogs',
Options={
"awslogs-group":
Ref('EngineLogGroup'),
"awslogs-region":
Ref('AWS::Region'),
"awslogs-stream-prefix":
Join('', ['anchore-engine', 'logs'])
}),
Links=['anchore-db']),
ContainerDefinition(
Name='anchore-db',
Hostname='anchore-db',
Cpu=int('512'),
MemoryReservation=int('1536'),
Essential=bool('true'),
Image=Ref('ArchoreDatabaseImage'),
PortMappings=[
PortMapping(
ContainerPort=int('5432'),
HostPort=int('5432'),
Protocol='tcp',
)
],
DockerSecurityOptions=['apparmor:docker-default'],
MountPoints=[
MountPoint(ContainerPath=Ref('PGDATA'),
SourceVolume='anchore_db_vol')
],
Environment=[
Environment(Name='POSTGRES_PASSWORD',
Value=Ref('AnchoreDBPassword')),
Environment(Name='PGDATA', Value=Ref('PGDATA')),
Environment(Name='AWS_DEFAULT_REGION',
Value=Ref('AWS::Region')),
Environment(Name='region',
Value=Ref('AWS::Region')),
],
LogConfiguration=LogConfiguration(
LogDriver='awslogs',
Options={
"awslogs-group":
Ref('DatabaseLogGroup'),
"awslogs-region":
Ref('AWS::Region'),
"awslogs-stream-prefix":
Join('', ['anchore-db', 'logs'])
}))
]))
return self.cfn_template
Name="bigid-web",
Memory="1024",
Essential=True,
Image=Join("", [
repo_id,
"/bigid/bigid-web",
]),
PortMappings=[PortMapping(ContainerPort="3000", HostPort="3000")],
Links=["bigid-orch"],
ExtraHosts=[
HostEntry(Hostname="bigid-mongo",
IpAddress=GetAtt(mongo_instance, "PrivateIp"))
],
Environment=[
Environment(
Name="BIGID_MONGO_USER",
Value=mongo_user,
),
Environment(
Name="BIGID_MONGO_PWD",
Value=mongo_pass,
),
Environment(
Name="WEB_URL_EXT",
Value=Join(
"",
["http://",
GetAtt(load_balancer, "DNSName"), ":3000"]),
),
Environment(
Name="AWS_ACCESS_KEY",
Value=Ref(aws_access_key),
Name="WebWorker",
# 1024 is full CPU
Cpu=8,
Memory=2048,
Essential=True,
Image=Join("", [
Ref(AWS_ACCOUNT_ID),
".dkr.ecr.",
Ref(AWS_REGION),
".amazonaws.com/",
Ref(repository),
":",
app_revision,
]),
PortMappings=[PortMapping(
ContainerPort=10,
HostPort=8000,
)],
Environment=[
Environment(
Name="AWS_STORAGE_BUCKET_NAME",
Value='blah',
),
Environment(Name="CDN_DOMAIN_NAME", Value="DomainName")
],
)
],
)
print(template.to_yaml())
Cpu='512',
Memory='1GB',
NetworkMode='awsvpc',
TaskRoleArn=ImportValue(Sub("${DependencyStackName}-TaskRoleArn")),
ExecutionRoleArn=ImportValue(
Sub("${DependencyStackName}-TaskExecutionRoleArn")),
ContainerDefinitions=[
ContainerDefinition(
Name='ghost',
Image=Ref(ghost_image),
Essential=True,
PortMappings=[PortMapping(ContainerPort=2368)],
Environment=[
Environment(
Name='url',
Value=ImportValue(
Sub("${DependencyStackName}-ALBURL")),
),
Environment(Name='database__client', Value='mysql2'),
Environment(
Name='database__connection__host',
Value=ImportValue(
Sub("${DependencyStackName}-GhostDBHost")),
),
Environment(Name='database__connection__user',
Value='ghost'),
Environment(Name='database__connection__database',
Value='ghost'),
Environment(Name='AWSREGION', Value=Ref('AWS::Region'))
],
LogConfiguration=LogConfiguration(
TaskDefinition(
"task",
ContainerDefinitions=[
ContainerDefinition(Image=Join("", [
Ref("AWS::AccountId"), ".dkr.ecr.",
Ref("AWS::Region"), ".amazonaws.com", "/", "helloworld", ":",
Ref("Tag")
]),
Memory=32,
Cpu=256,
Name="helloworld",
PortMappings=[
ecs.PortMapping(ContainerPort=3000)
],
Environment=[
Environment(Name='HELLOWORLD_VERSION',
Value=Ref("Tag"))
],
LogConfiguration=LogConfiguration(
LogDriver="awslogs",
Options={
'awslogs-group': "/aws/ecs/helloworld",
'awslogs-region': Ref("AWS::Region"),
})),
],
))
t.add_resource(
Role(
"ServiceRole",
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
PortMapping(
ContainerPort=web_worker_port,
HostPort=web_worker_port,
)
],
LogConfiguration=LogConfiguration(LogDriver="awslogs",
Options={
'awslogs-group':
Ref(container_log_group),
'awslogs-region':
Ref(AWS_REGION),
'awslogs-stream-prefix':
Ref(AWS_STACK_NAME),
}),
Environment=[
Environment(Name=k, Value=v) for k, v in environment_variables
] + [
Environment(Name="PORT", Value=web_worker_port),
],
)
],
)
app_service_role = iam.Role(
"AppServiceRole",
template=template,
AssumeRolePolicyDocument=dict(Statement=[
dict(
Effect="Allow",
Principal=dict(Service=["ecs.amazonaws.com"]),
Action=["sts:AssumeRole"],
envs = container.get('env', [])
entrypoint = container.get('entrypoint')
command = container.get('command')
volumes = container.get('volumes', [])
ports = container.get('ports', [])
depends = container.get('depends', [])
linux_parameters = container.get('linux_parameters')
definition = ContainerDefinition(
Image=container.get('image', 'kspckan/netkan'),
Memory=container.get('memory', '128'),
Name=container['name'],
Secrets=[
Secret(Name=x, ValueFrom='{}{}'.format(PARAM_NAMESPACE, x))
for x in secrets
],
Environment=[Environment(Name=x[0], Value=x[1]) for x in envs],
MountPoints=[],
PortMappings=[],
DependsOn=[],
Links=[],
)
if entrypoint:
entrypoint = entrypoint if isinstance(entrypoint,
list) else [entrypoint]
definition.EntryPoint = entrypoint
if command:
command = command if isinstance(command, list) else [command]
definition.Command = command
if linux_parameters:
definition.LinuxParameters = linux_parameters
for volume in volumes:
def _add_service(self, service_name, config):
launch_type = self.LAUNCH_TYPE_FARGATE if 'fargate' in config else self.LAUNCH_TYPE_EC2
env_config = build_config(
self.env,
self.application_name,
self.env_sample_file_path
)
container_definition_arguments = {
"Environment": [
Environment(Name=k, Value=v) for (k, v) in env_config
],
"Name": service_name + "Container",
"Image": self.ecr_image_uri + ':' + self.current_version,
"Essential": 'true',
"LogConfiguration": self._gen_log_config(service_name),
"MemoryReservation": int(config['memory_reservation']),
"Cpu": 0
}
if 'http_interface' in config:
container_definition_arguments['PortMappings'] = [
PortMapping(
ContainerPort=int(
config['http_interface']['container_port']
)
)
]
if config['command'] is not None:
container_definition_arguments['Command'] = [config['command']]
cd = ContainerDefinition(**container_definition_arguments)
task_role = self.template.add_resource(Role(
service_name + "Role",
AssumeRolePolicyDocument=PolicyDocument(
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", ["ecs-tasks.amazonaws.com"])
)
]
)
))
launch_type_td = {}
if launch_type == self.LAUNCH_TYPE_FARGATE:
launch_type_td = {
'RequiresCompatibilities': ['FARGATE'],
'ExecutionRoleArn': boto3.resource('iam').Role('ecsTaskExecutionRole').arn,
'NetworkMode': 'awsvpc',
'Cpu': str(config['fargate']['cpu']),
'Memory': str(config['fargate']['memory'])
}
td = TaskDefinition(
service_name + "TaskDefinition",
Family=service_name + "Family",
ContainerDefinitions=[cd],
TaskRoleArn=Ref(task_role),
**launch_type_td
)
self.template.add_resource(td)
desired_count = self._get_desired_task_count_for_service(service_name)
deployment_configuration = DeploymentConfiguration(
MinimumHealthyPercent=100,
MaximumPercent=200
)
if 'http_interface' in config:
alb, lb, service_listener, alb_sg = self._add_alb(cd, service_name, config, launch_type)
if launch_type == self.LAUNCH_TYPE_FARGATE:
# if launch type is ec2, then services inherit the ec2 instance security group
# otherwise, we need to specify a security group for the service
service_security_group = SecurityGroup(
pascalcase("FargateService" + self.env + service_name),
GroupName=pascalcase("FargateService" + self.env + service_name),
SecurityGroupIngress=[{
'IpProtocol': 'TCP',
'SourceSecurityGroupId': Ref(alb_sg),
'ToPort': int(config['http_interface']['container_port']),
'FromPort': int(config['http_interface']['container_port']),
}],
VpcId=Ref(self.vpc),
GroupDescription=pascalcase("FargateService" + self.env + service_name)
)
self.template.add_resource(service_security_group)
launch_type_svc = {
'NetworkConfiguration': NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
],
SecurityGroups=[
Ref(service_security_group)
]
)
)
}
else:
launch_type_svc = {
'Role': Ref(self.ecs_service_role),
'PlacementStrategies': self.PLACEMENT_STRATEGIES
}
svc = Service(
service_name,
LoadBalancers=[lb],
Cluster=self.cluster_name,
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DependsOn=service_listener.title,
LaunchType=launch_type,
**launch_type_svc,
)
self.template.add_output(
Output(
service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')
)
)
self.template.add_output(
Output(
service_name + "URL",
Description="The URL at which the service is accessible",
Value=Sub("https://${" + alb.name + ".DNSName}")
)
)
self.template.add_resource(svc)
else:
launch_type_svc = {}
if launch_type == self.LAUNCH_TYPE_FARGATE:
# if launch type is ec2, then services inherit the ec2 instance security group
# otherwise, we need to specify a security group for the service
service_security_group = SecurityGroup(
pascalcase("FargateService" + self.env + service_name),
GroupName=pascalcase("FargateService" + self.env + service_name),
SecurityGroupIngress=[],
VpcId=Ref(self.vpc),
GroupDescription=pascalcase("FargateService" + self.env + service_name)
)
self.template.add_resource(service_security_group)
launch_type_svc = {
'NetworkConfiguration': NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
],
SecurityGroups=[
Ref(service_security_group)
]
)
)
}
else:
launch_type_svc = {
'PlacementStrategies': self.PLACEMENT_STRATEGIES
}
svc = Service(
service_name,
Cluster=self.cluster_name,
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DeploymentConfiguration=deployment_configuration,
LaunchType=launch_type,
**launch_type_svc
)
self.template.add_output(
Output(
service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')
)
)
self.template.add_resource(svc)
self._add_service_alarms(svc)
app_revision,
]),
PortMappings=[PortMapping(
ContainerPort=web_worker_port,
HostPort=web_worker_port,
)],
LogConfiguration=LogConfiguration(
LogDriver="awslogs",
Options={
'awslogs-group': Ref(container_log_group),
'awslogs-region': Ref(AWS_REGION),
'awslogs-stream-prefix': Ref(AWS_STACK_NAME),
}
),
Environment=[
Environment(Name=k, Value=v)
for k, v in environment_variables
] + [
Environment(Name="PORT", Value=web_worker_port),
],
)
],
)
app_service_role = iam.Role(
"AppServiceRole",
template=template,
AssumeRolePolicyDocument=dict(Statement=[dict(
Effect="Allow",
Principal=dict(Service=["ecs.amazonaws.com"]),
definition = ContainerDefinition(
Image=Sub(
'${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/been_hugged/${name}',
name=container.get('image', container['name'])
),
Memory=container.get('memory', '96'),
Name=container['name'],
Secrets=[
Secret(
Name=x.upper(),
ValueFrom='/hugged/{}'.format(x)
) for x in secrets
],
Environment=[
Environment(
Name=x[0].upper(), Value=x[1]
) for x in envs
],
MountPoints=[],
PortMappings=[],
DependsOn=[],
Links=[],
)
if entrypoint:
entrypoint = entrypoint if isinstance(entrypoint, list) else [entrypoint]
definition.EntryPoint = entrypoint
if command:
command = command if isinstance(command, list) else [command]
definition.Command = command
for volume in volumes:
volume_name = '{}{}'.format(
def build_cloudformation_resource(
self,
container_configurations,
ecr_image_uri,
fallback_task_role,
fallback_task_execution_role,
):
environment = self.environment
service_name = self.service_name
config = self.configuration
launch_type = get_launch_type(config)
task_family_name = f'{environment}{service_name}Family'[:255]
td_kwargs = dict()
td_kwargs['PlacementConstraints'] = [
PlacementConstraint(Type=constraint['type'],
Expression=constraint['expression'])
for constraint in config.get('placement_constraints', [])
]
td_kwargs['TaskRoleArn'] = config.get('task_role_arn') if 'task_role_arn' in config \
else fallback_task_role
td_kwargs['ExecutionRoleArn'] = config.get('task_execution_role_arn') \
if 'task_execution_role_arn' in config \
else fallback_task_execution_role
if ('udp_interface' in config) or ('tcp_interface' in config):
td_kwargs['NetworkMode'] = 'awsvpc'
log_config = self._gen_log_config(config)
env_config = container_configurations[container_name(
service_name)].get('environment', {})
secrets_config = container_configurations[container_name(
service_name)].get('secrets', {})
cd_kwargs = {
"Environment": [
Environment(Name=name, Value=env_config[name])
for name in env_config
],
"Secrets": [
Secret(Name=name, ValueFrom=secrets_config[name])
for name in secrets_config
],
"Name":
container_name(service_name),
"Image":
ecr_image_uri,
"Essential":
True,
"LogConfiguration":
log_config,
"MemoryReservation":
int(config['memory_reservation']),
"Cpu":
0,
'Memory':
int(config.get('memory_hard_limit', HARD_LIMIT_MEMORY_IN_MB)),
}
if config.get('command'):
# command can be one of list(string) / string
command = config.get('command')
if isinstance(command, str):
cd_kwargs['Command'] = [command]
elif isinstance(command, list):
cd_kwargs['Command'] = command
else:
raise UnrecoverableException(
'command should either be string or list(string)')
if 'stop_timeout' in config:
cd_kwargs['StopTimeout'] = int(config['stop_timeout'])
if 'system_controls' in config:
cd_kwargs['SystemControls'] = [
SystemControl(Namespace=system_control['namespace'],
Value=system_control['value'])
for system_control in config['system_controls']
]
if 'dns_servers' in config:
cd_kwargs['DnsServers'] = config.get('dns_servers', [])
if launch_type == LAUNCH_TYPE_FARGATE:
if 'udp_interface' in config:
raise NotImplementedError(
'udp interface not yet implemented in fargate type, please use ec2 type'
)
elif 'tcp_interface' in config:
raise NotImplementedError(
'tcp interface not yet implemented in fargate type, please use ec2 type'
)
if 'http_interface' in config:
cd_kwargs['PortMappings'] = [
PortMapping(ContainerPort=int(config['http_interface']
['container_port']))
]
elif 'udp_interface' in config:
cd_kwargs['PortMappings'] = [
PortMapping(ContainerPort=int(
config['udp_interface']['container_port']),
HostPort=int(
config['udp_interface']['container_port']),
Protocol='udp'),
PortMapping(ContainerPort=int(
config['udp_interface']['health_check_port']),
HostPort=int(
config['udp_interface']['health_check_port']),
Protocol='tcp')
]
elif 'tcp_interface' in config:
cd_kwargs['PortMappings'] = [
PortMapping(ContainerPort=int(
config['tcp_interface']['container_port']),
Protocol='tcp')
]
if 'container_health_check' in config:
configured_health_check = config['container_health_check']
ecs_health_check = {
'Command': ['CMD-SHELL', configured_health_check['command']]
}
if 'start_period' in configured_health_check:
ecs_health_check['StartPeriod'] = int(
configured_health_check['start_period'])
if 'retries' in configured_health_check:
ecs_health_check['Retries'] = int(
configured_health_check['retries'])
if 'interval' in configured_health_check:
ecs_health_check['Interval'] = int(
configured_health_check['interval'])
if 'timeout' in configured_health_check:
ecs_health_check['Timeout'] = int(
configured_health_check['timeout'])
cd_kwargs['HealthCheck'] = HealthCheck(**ecs_health_check)
if 'sidecars' in config:
links = []
for sidecar in config['sidecars']:
sidecar_name = sidecar.get('name')
links.append("{}:{}".format(container_name(sidecar_name),
sidecar_name))
cd_kwargs['Links'] = links
if 'container_labels' in config:
cd_kwargs['DockerLabels'] = config.get('container_labels')
if 'ulimits' in config:
cd_kwargs['Ulimits'] = [
Ulimit(
Name=ulimit['name'],
SoftLimit=ulimit['soft_limit'],
HardLimit=ulimit['hard_limit'],
) for ulimit in config.get('ulimits', [])
]
cd = ContainerDefinition(**cd_kwargs)
container_definitions = [cd]
if 'sidecars' in config:
for sidecar in config['sidecars']:
container_definitions.append(
self._gen_container_definitions_for_sidecar(
sidecar, log_config,
container_configurations.get(
container_name(sidecar.get('name')), {})), )
if launch_type == LAUNCH_TYPE_FARGATE:
td_kwargs['RequiresCompatibilities'] = [LAUNCH_TYPE_FARGATE]
td_kwargs['NetworkMode'] = 'awsvpc'
td_kwargs['Cpu'] = str(config['fargate']['cpu'])
td_kwargs['Memory'] = str(config['fargate']['memory'])
return TaskDefinition(self._resource_name(service_name),
Family=task_family_name,
ContainerDefinitions=container_definitions,
**td_kwargs)
def main():
template = Template()
template.add_version("2010-09-09")
template.set_description("AWS CloudFormation ECS Service")
# Add the Parameters
Application = template.add_parameter(
Parameter(
"Application",
Type="String",
))
DockerImage = template.add_parameter(
Parameter(
"DockerImage",
Type="String",
))
USERNAME = template.add_parameter(Parameter(
"USERNAME",
Type="String",
))
ClusterName = template.add_parameter(
Parameter(
"ClusterName",
Type="String",
))
ContainerPort = template.add_parameter(
Parameter(
"ContainerPort",
Type="String",
))
HostPort = template.add_parameter(Parameter(
"HostPort",
Type="String",
))
HostedZoneName = template.add_parameter(
Parameter(
"HostedZoneName",
Type="String",
))
CertArn = template.add_parameter(Parameter(
"CertArn",
Type="String",
))
ExecutionRoleArn = template.add_parameter(
Parameter("ExecutionRoleArn",
Type="String",
Description="Execution Role to get creadentials from ssm"))
HealthCheckPath = template.add_parameter(
Parameter(
"HealthCheckPath",
Type="String",
))
HealthCheckIntervalSeconds = template.add_parameter(
Parameter(
"HealthCheckIntervalSeconds",
Type="String",
))
HealthyThresholdCount = template.add_parameter(
Parameter(
"HealthyThresholdCount",
Type="String",
))
HealthCheckTimeoutSeconds = template.add_parameter(
Parameter(
"HealthCheckTimeoutSeconds",
Type="String",
))
UnhealthyThresholdCount = template.add_parameter(
Parameter(
"UnhealthyThresholdCount",
Type="String",
))
VpcId = template.add_parameter(Parameter(
"VpcId",
Type="String",
))
Subnets = template.add_parameter(
Parameter(
"Subnets",
Type="List<AWS::EC2::Subnet::Id>",
))
# Add the application ELB
NetworkLB = template.add_resource(
elb.LoadBalancer("NetworkLB",
Name=Join("", [Ref(Application), "-nlb"]),
Scheme="internet-facing",
Subnets=Ref(Subnets),
Type='network'))
NlbTargetGroup = template.add_resource(
elb.TargetGroup(
"NlbTargetGroup",
Name='ecs-service-targetgroup',
HealthCheckIntervalSeconds=Ref(HealthCheckIntervalSeconds),
HealthCheckProtocol="TCP",
HealthyThresholdCount=Ref(HealthyThresholdCount),
Port=80,
Protocol="TCP",
UnhealthyThresholdCount=Ref(UnhealthyThresholdCount),
VpcId=Ref(VpcId)))
NlbListener = template.add_resource(
elb.Listener(
"Listener",
DependsOn=["NlbTargetGroup", "NetworkLB"],
Certificates=[elb.Certificate(CertificateArn=Ref(CertArn))],
Port="443",
Protocol="TLS",
LoadBalancerArn=Ref(NetworkLB),
DefaultActions=[
elb.Action(Type="forward", TargetGroupArn=Ref(NlbTargetGroup))
]))
Task_Definition = template.add_resource(
TaskDefinition(
'TaskDefinition',
Memory='500',
ExecutionRoleArn=Ref(ExecutionRoleArn),
ContainerDefinitions=[
ContainerDefinition(
Name=Join("", [Ref(Application)]),
Image=Ref(DockerImage),
Essential=True,
Secrets=[Secret(Name='USERNAME', ValueFrom=Ref(USERNAME))],
Environment=[
Environment(Name="DOCKER_LABELS", Value="true")
],
DockerLabels={
'aws-account': Ref("AWS::AccountId"),
'region': Ref("AWS::Region"),
'stack': Ref("AWS::StackName")
},
PortMappings=[
PortMapping(ContainerPort=Ref(ContainerPort),
HostPort=Ref(HostPort))
])
]))
app_service = template.add_resource(
Service("AppService",
DependsOn=["Listener", "TaskDefinition"],
Cluster=Ref(ClusterName),
DesiredCount=1,
TaskDefinition=Ref(Task_Definition),
ServiceName=Join("", [Ref(Application), "-ecs-service"]),
LoadBalancers=[
ecs.LoadBalancer(ContainerName=Join(
"", [Ref(Application)]),
ContainerPort=Ref(ContainerPort),
TargetGroupArn=Ref(NlbTargetGroup))
]))
AppDNSRecord = template.add_resource(
RecordSetType(
"AppDNSRecord",
DependsOn=["AppService"],
HostedZoneName=Join("", [Ref(HostedZoneName), "."]),
Name=Join("", [Ref(Application), ".",
Ref(HostedZoneName), "."]),
Type="CNAME",
TTL="900",
ResourceRecords=[GetAtt(NetworkLB, "DNSName")]))
template.add_output(
Output("URL",
Description="DomainName",
Value=Join("", ["https://", Ref(AppDNSRecord)])))
with open("ecs-ec2-service-cf.yaml", "w") as yamlout:
yamlout.write(template.to_yaml())
t.add_resource(
TaskDefinition(
"task",
ContainerDefinitions=[
ContainerDefinition(
Image=Join("", [
Ref("AWS::AccountId"), ".dkr.ecr.",
Ref("AWS::Region"), ".amazonaws.com", "/",
ImportValue("helloworld-repo"), ":",
Ref("Tag")
]),
Memory=512,
Cpu=2048,
Name="helloworld",
Environment=[
Environment(Name="node_env", Value=Ref("NodeEnv"))
],
PortMappings=[ecs.PortMapping(ContainerPort=80)])
],
))
t.add_resource(
Role(
"ServiceRole",
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", ["ecs.amazonaws.com"]))
]),
Path="/",
ManagedPolicyArns=[
shibboleth_task_definition = t.add_resource(TaskDefinition(
'ShibbolethTaskDefinition',
TaskRoleArn=Ref(task_role_arn),
ContainerDefinitions=[
ContainerDefinition(
Name='shibboleth',
Image=Ref(shibboleth_image),
MemoryReservation=1024,
Essential=True,
PortMappings=[PortMapping(ContainerPort=8080)],
Environment=[
Environment(
Name='S3PATH',
Value=Ref(s3path)
)
]
)
]
))
shibboleth_service = t.add_resource(Service(
'ShibbolethService',
Cluster=Ref(cluster),
DesiredCount=1,
TaskDefinition=Ref(shibboleth_task_definition),
LoadBalancers=[
LoadBalancer(
ContainerName='shibboleth',
ContainerPort=8080,
application_revision,
]),
PortMappings=[PortMapping(
HostPort=0,
ContainerPort=web_worker_port,
)],
LogConfiguration=LogConfiguration(
LogDriver="awslogs",
Options={
'awslogs-group': Ref(web_log_group),
'awslogs-region': Ref(AWS_REGION),
}
),
Environment=[
Environment(
Name="AWS_STORAGE_BUCKET_NAME",
Value=assets_bucket,
),
Environment(
Name="CDN_DOMAIN_NAME",
Value=distribution,
),
Environment(
Name="DOMAIN_NAME",
Value=api_domain_name,
),
Environment(
Name="PORT",
Value=web_worker_port,
),
Environment(
Name="SECRET_KEY",
def generate_resource_service_env_vars(self, target: tuple,
target_definition: dict) -> list:
"""
Generates env vars based on ReturnValues set for a give service. When the resource is new, adds the
parameter to the services stack appropriately.
"""
res_return_names = {}
for prop_param in self.attributes_outputs.keys():
if prop_param.return_value:
res_return_names[prop_param.return_value] = prop_param
else:
res_return_names[prop_param.title] = prop_param
env_vars = []
params_to_add = []
if self.ref_parameter and self.ref_parameter.title in target_definition.keys(
):
LOG.debug(
f"{self.module.res_key}.{self.module.res_key} - Ref parameter {self.ref_parameter.title} override."
)
elif (self.ref_parameter
and self.ref_parameter.title not in target_definition.keys()):
env_var_name = ENV_VAR_NAME.sub(
"",
self.name.replace("-", "_").upper())
target_definition[self.ref_parameter.title] = env_var_name
LOG.info(
f"{self.module.res_key}.{self.name} - Auto-added {env_var_name} for Ref value"
)
else:
LOG.warning(
f"{self.module.res_key}.{self.name} - Ref parameter not defined on {self.__class__}"
)
for prop_name, env_var_name in target_definition.items():
if prop_name in res_return_names:
if self.cfn_resource:
env_vars.append(
Environment(
Name=env_var_name,
Value=Ref(self.attributes_outputs[res_return_names[
prop_name]]["ImportParameter"]),
))
params_to_add.append(res_return_names[prop_name])
elif self.lookup_properties:
env_vars.append(
Environment(
Name=env_var_name,
Value=self.attributes_outputs[
res_return_names[prop_name]]["ImportValue"],
))
if params_to_add:
params_values = {}
settings = [
get_parameter_settings(self, param) for param in params_to_add
]
resource_params_to_add = []
for setting in settings:
resource_params_to_add.append(setting[1])
params_values[setting[0]] = setting[2]
add_parameters(target[0].template, resource_params_to_add)
target[0].stack.Parameters.update(params_values)
return env_vars
def _add_service(self, service_name, config):
env_config = build_config(
self.env,
self.application_name,
self.env_sample_file_path
)
container_definition_arguments = {
"Environment": [
Environment(Name=k, Value=v) for (k, v) in env_config
],
"Name": service_name + "Container",
"Image": self.ecr_image_uri + ':' + self.current_version,
"Essential": 'true',
"LogConfiguration": self._gen_log_config(service_name),
"MemoryReservation": int(config['memory_reservation']),
"Cpu": 0
}
if 'http_interface' in config:
container_definition_arguments['PortMappings'] = [
PortMapping(
ContainerPort=int(
config['http_interface']['container_port']
)
)
]
if config['command'] is not None:
container_definition_arguments['Command'] = [config['command']]
cd = ContainerDefinition(**container_definition_arguments)
task_role = self.template.add_resource(Role(
service_name + "Role",
AssumeRolePolicyDocument=PolicyDocument(
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", ["ecs-tasks.amazonaws.com"])
)
]
)
))
td = TaskDefinition(
service_name + "TaskDefinition",
Family=service_name + "Family",
ContainerDefinitions=[cd],
TaskRoleArn=Ref(task_role)
)
self.template.add_resource(td)
desired_count = self._get_desired_task_count_for_service(service_name)
deployment_configuration = DeploymentConfiguration(
MinimumHealthyPercent=100,
MaximumPercent=200
)
if 'http_interface' in config:
alb, lb, service_listener = self._add_alb(cd, service_name, config)
svc = Service(
service_name,
LoadBalancers=[lb],
Cluster=self.cluster_name,
Role=Ref(self.ecs_service_role),
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DependsOn=service_listener.title,
PlacementStrategies=self.PLACEMENT_STRATEGIES
)
self.template.add_output(
Output(
service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')
)
)
self.template.add_output(
Output(
service_name + "URL",
Description="The URL at which the service is accessible",
Value=Sub("https://${" + alb.name + ".DNSName}")
)
)
self.template.add_resource(svc)
else:
svc = Service(
service_name,
Cluster=self.cluster_name,
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DeploymentConfiguration=deployment_configuration,
PlacementStrategies=self.PLACEMENT_STRATEGIES
)
self.template.add_output(
Output(
service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')
)
)
self.template.add_resource(svc)
self._add_service_alarms(svc)
def add_envoy_container_definition(self, family):
"""
Method to expand the containers configuration and add the Envoy SideCar.
"""
proxy_config = ProxyConfiguration(
ContainerName="envoy",
Type="APPMESH",
ProxyConfigurationProperties=[
Environment(Name="IgnoredUID", Value="1337"),
Environment(
Name="ProxyIngressPort",
Value="15000",
),
Environment(Name="ProxyEgressPort", Value="15001"),
Environment(Name="IgnoredGID", Value=""),
Environment(
Name="EgressIgnoredIPs",
Value="169.254.170.2,169.254.169.254",
),
Environment(Name="EgressIgnoredPorts", Value=""),
Environment(
Name="AppPorts",
Value=",".join(
[f"{port.Port}" for port in self.port_mappings]),
),
],
)
envoy_service = ManagedSidecar(
"envoy",
{
"image":
"public.ecr.aws/appmesh/aws-appmesh-envoy:v1.21.1.1-prod",
"user":
"******",
"deploy": {
"resources": {
"limits": {
"cpus": 0.125,
"memory": "256MB"
}
}
},
"environment": {
"ENABLE_ENVOY_XRAY_TRACING": 0,
},
"ports": [
{
"target": 15000,
"published": 15000,
"protocol": "tcp"
},
{
"target": 15001,
"published": 15001,
"protocol": "tcp"
},
],
"healthcheck": {
"test": [
"CMD-SHELL",
"curl -s http://localhost:9901/server_info | grep state | grep -q LIVE",
],
"interval":
"5s",
"timeout":
"2s",
"retries":
3,
"start_period":
10,
},
"ulimits": {
"nofile": {
"soft": 15000,
"hard": 15000
}
},
"x-iam": {
"Policies": [{
"PolicyName": "AppMeshAccess",
"PolicyDocument": {
"Version":
"2012-10-17",
"Statement": [
{
"Sid": "AppMeshAccess",
"Effect": "Allow",
"Action":
["appmesh:StreamAggregatedResources"],
"Resource": ["*"],
},
{
"Sid":
"ServiceDiscoveryAccess",
"Effect":
"Allow",
"Action": [
"servicediscovery:Get*",
"servicediscovery:Describe*",
"servicediscovery:List*",
"servicediscovery:DiscoverInstances*",
],
"Resource":
"*",
},
],
},
}]
},
},
)
envoy_service.container_definition.Environment.append(
Environment(
Name="APPMESH_VIRTUAL_NODE_NAME",
Value=Sub(
f"mesh/${{{appmesh_params.MESH_NAME.title}}}/virtualNode/${{{self.node.title}.VirtualNodeName}}"
),
))
family.add_managed_sidecar(envoy_service)
setattr(family.task_definition, "ProxyConfiguration", proxy_config)
PortMappings=[
PortMapping(
HostPort=0,
ContainerPort=web_worker_port,
)
],
LogConfiguration=LogConfiguration(LogDriver="awslogs",
Options={
'awslogs-group':
Ref(web_log_group),
'awslogs-region':
Ref(AWS_REGION),
}),
Environment=[
Environment(
Name="AWS_STORAGE_BUCKET_NAME",
Value=Ref(assets_bucket),
),
Environment(
Name="CDN_DOMAIN_NAME",
Value=GetAtt(distribution, "DomainName"),
),
Environment(
Name="DOMAIN_NAME",
Value=domain_name,
),
Environment(
Name="PORT",
Value=web_worker_port,
),
Environment(
Name="SECRET_KEY",
ContainerDefinition(
Image=Join("", [
Ref("AWS::AccountId"), ".dkr.ecr.",
Ref("AWS::Region"), ".amazonaws.com", "/",
ImportValue("community-mother-api-dev-repo"), ":",
Ref("Tag")
]),
Memory=957,
Cpu=2048,
Name=Join("-", [
Select(0, Split("-", Ref("AWS::StackName"))),
Select(1, Split("-", Ref("AWS::StackName"))),
Select(2, Split("-", Ref("AWS::StackName")))
]),
Environment=[
Environment(Name="application_env",
Value=Ref("ApplicationEnv"))
],
PortMappings=[ecs.PortMapping(ContainerPort=80)])
],
))
t.add_resource(
Role(
"ServiceRole",
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", ["ecs.amazonaws.com"]))
]),
Path="/",
ManagedPolicyArns=[