def set_volumes(family):
"""
Method to create the volumes definition to the Task Definition
:return:
"""
family_task_volumes = define_shared_volumes(family)
host_volumes = define_host_volumes(family)
if not hasattr(family.task_definition, "Volumes"):
family_definition_volumes = []
setattr(family.task_definition, "Volumes", family_definition_volumes)
else:
family_definition_volumes = getattr(family.task_definition, "Volumes")
for volume in family_task_volumes:
if volume.type == "volume" and volume.driver == "local":
volume.cfn_volume = Volume(
Host=NoValue,
Name=volume.volume_name,
DockerVolumeConfiguration=If(
USE_FARGATE_CON_T,
NoValue,
DockerVolumeConfiguration(
Scope="task" if not volume.is_shared else "shared",
Autoprovision=NoValue
if not volume.is_shared else True,
),
),
)
if volume.cfn_volume:
family_definition_volumes.append(volume.cfn_volume)
for volume_config in host_volumes:
cfn_volume = If(
USE_FARGATE_CON_T,
NoValue,
Volume(
Host=Host(SourcePath=volume_config["source"]),
DockerVolumeConfiguration=NoValue,
Name=NONALPHANUM.sub("", volume_config["target"]),
),
)
family_definition_volumes.append(cfn_volume)
set_services_mount_points(family)
def expand_family_with_efs_volumes(efs_root_stack_title, new_efs, settings):
"""
Function to add the EFS Volume definition to the task definition for the service to use.
:param efs_root_stack_title: Root stack title for EFS
:param new_efs:
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
fs_id_parameter = new_efs.attributes_outputs[FS_ID]["ImportParameter"]
fs_id_getatt = new_efs.attributes_outputs[FS_ID]["ImportValue"]
for target in new_efs.families_targets:
if target[0].service_compute.launch_type == "EXTERNAL":
LOG.warning(
f"x-efs - {target[0].name} - When using EXTERNAL Launch Type, networking settings cannot be set."
)
return
access_points = []
target[0].stack.Parameters.update(
{fs_id_parameter.title: fs_id_getatt})
add_parameters(target[0].template, [fs_id_parameter])
task_definition = target[0].template.resources[TASK_T]
efs_config_kwargs = {"FilesystemId": Ref(fs_id_parameter)}
if (new_efs.parameters
and keyisset("EnforceIamAuth", new_efs.parameters)
or [service.user for service in target[2]]):
add_efs_definition_to_target_family(new_efs, target)
efs_access_point = target[0].template.add_resource(
AccessPoint(
f"{new_efs.logical_name}{target[0].logical_name}EfsAccessPoint",
FileSystemId=Ref(fs_id_parameter),
))
access_points.append(efs_access_point)
efs_config_kwargs.update({
"AuthorizationConfig":
AuthorizationConfig(AccessPointId=Ref(efs_access_point),
IAM="ENABLED"),
"TransitEncryption":
"ENABLED",
})
efs_volume_definition = Volume(
EFSVolumeConfiguration=EFSVolumeConfiguration(**efs_config_kwargs),
Name=new_efs.volume.volume_name,
)
volumes = get_volumes(task_definition)
volumes.append(efs_volume_definition)
override_efs_settings(new_efs, target, fs_id_parameter, access_points,
volumes)
add_task_iam_access_to_access_point(target[0], access_points, new_efs)
def override_service_volume(new_efs, fs_id, target, access_points, volumes):
"""
Function to override service volume if a specific definition was set for it
"""
for service in target[2]:
if not service.user:
continue
mount_points = get_service_mount_points(service)
for count, mount_pt in enumerate(mount_points):
if mount_pt.SourceVolume == new_efs.volume.volume_name:
container_volume_name = (
f"{new_efs.volume.volume_name}{service.logical_name}")
setattr(
mount_pt,
"SourceVolume",
container_volume_name,
)
sub_service_specific_access_point = AccessPoint(
f"{new_efs.logical_name}{service.logical_name}ServiceEfsAccessPoint",
FileSystemId=Ref(fs_id),
PosixUser=PosixUser(
Uid=service.user,
Gid=service.group if service.group else service.user,
),
RootDirectory=RootDirectory(
CreationInfo=CreationInfo(
OwnerUid=service.user,
OwnerGid=service.group
if service.group else service.user,
Permissions="0755",
),
Path=mount_pt.ContainerPath,
),
)
target[0].template.add_resource(
sub_service_specific_access_point)
access_points.append(sub_service_specific_access_point)
volumes.append(
Volume(
EFSVolumeConfiguration=EFSVolumeConfiguration(
FilesystemId=Ref(fs_id),
AuthorizationConfig=AuthorizationConfig(
AccessPointId=Ref(
sub_service_specific_access_point),
IAM="ENABLED",
),
),
Name=container_volume_name,
))
def add_ecs_task(self):
'''
Add ECS Task
'''
self.cfn_template.add_resource(
TaskDefinition(
title=constants.TASK,
Volumes=[Volume(Name='anchore_db_vol')],
TaskRoleArn=GetAtt(constants.TASK_ROLE, 'Arn'),
ContainerDefinitions=[
ContainerDefinition(
Name='anchore-engine',
Hostname='anchore-engine',
Cpu=int('512'),
MemoryReservation=int('1536'),
Essential=bool('true'),
Image=ImportValue(
Sub('${Environment}-${AnchoreEngineImage}')),
PortMappings=[
PortMapping(
ContainerPort=int('8228'),
HostPort=int('8228'),
Protocol='tcp',
),
PortMapping(
ContainerPort=int('8338'),
HostPort=int('8338'),
Protocol='tcp',
),
],
DockerSecurityOptions=['apparmor:docker-default'],
Environment=[
Environment(Name='ANCHORE_HOST_ID',
Value='anchore-engine'),
Environment(Name='ANCHORE_ENDPOINT_HOSTNAME',
Value='anchore-engine'),
Environment(Name='ANCHORE_DB_HOST',
Value='anchore-db'),
Environment(Name='ANCHORE_DB_PASSWORD',
Value=Ref('AnchoreDBPassword')),
Environment(Name='AWS_DEFAULT_REGION',
Value=Ref('AWS::Region')),
Environment(Name='region',
Value=Ref('AWS::Region')),
],
LogConfiguration=LogConfiguration(
LogDriver='awslogs',
Options={
"awslogs-group":
Ref('EngineLogGroup'),
"awslogs-region":
Ref('AWS::Region'),
"awslogs-stream-prefix":
Join('', ['anchore-engine', 'logs'])
}),
Links=['anchore-db']),
ContainerDefinition(
Name='anchore-db',
Hostname='anchore-db',
Cpu=int('512'),
MemoryReservation=int('1536'),
Essential=bool('true'),
Image=Ref('ArchoreDatabaseImage'),
PortMappings=[
PortMapping(
ContainerPort=int('5432'),
HostPort=int('5432'),
Protocol='tcp',
)
],
DockerSecurityOptions=['apparmor:docker-default'],
MountPoints=[
MountPoint(ContainerPath=Ref('PGDATA'),
SourceVolume='anchore_db_vol')
],
Environment=[
Environment(Name='POSTGRES_PASSWORD',
Value=Ref('AnchoreDBPassword')),
Environment(Name='PGDATA', Value=Ref('PGDATA')),
Environment(Name='AWS_DEFAULT_REGION',
Value=Ref('AWS::Region')),
Environment(Name='region',
Value=Ref('AWS::Region')),
],
LogConfiguration=LogConfiguration(
LogDriver='awslogs',
Options={
"awslogs-group":
Ref('DatabaseLogGroup'),
"awslogs-region":
Ref('AWS::Region'),
"awslogs-stream-prefix":
Join('', ['anchore-db', 'logs'])
}))
]))
return self.cfn_template
def _add_service(self, service_name, config):
launch_type = self.LAUNCH_TYPE_FARGATE if 'fargate' in config else self.LAUNCH_TYPE_EC2
env_config = build_config(self.env, self.application_name,
self.env_sample_file_path)
container_definition_arguments = {
"Environment":
[Environment(Name=k, Value=v) for (k, v) in env_config],
"Name": service_name + "Container",
"Image": self.ecr_image_uri + ':' + self.current_version,
"Essential": 'true',
"LogConfiguration": self._gen_log_config(service_name),
"MemoryReservation": int(config['memory_reservation']),
"Cpu": 0
}
if 'http_interface' in config:
container_definition_arguments['PortMappings'] = [
PortMapping(ContainerPort=int(config['http_interface']
['container_port']))
]
if config['command'] is not None:
container_definition_arguments['Command'] = [config['command']]
if 'volume' in config:
container_definition_arguments['MountPoints'] = [
MountPoint(SourceVolume=service_name + '-efs-volume',
ContainerPath=config['volume']['container_path'])
]
cd = ContainerDefinition(**container_definition_arguments)
task_role = self.template.add_resource(
Role(service_name + "Role",
AssumeRolePolicyDocument=PolicyDocument(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal(
"Service", ["ecs-tasks.amazonaws.com"]))
])))
launch_type_td = {}
if launch_type == self.LAUNCH_TYPE_FARGATE:
launch_type_td = {
'RequiresCompatibilities': ['FARGATE'],
'ExecutionRoleArn':
boto3.resource('iam').Role('ecsTaskExecutionRole').arn,
'NetworkMode':
'awsvpc',
'Cpu':
str(config['fargate']['cpu']),
'Memory':
str(config['fargate']['memory'])
}
if 'custom_metrics' in config:
launch_type_td['NetworkMode'] = 'awsvpc'
if 'volume' in config:
launch_type_td['Volumes'] = [
Volume(
Name=service_name + '-efs-volume',
EFSVolumeConfiguration=EFSVolumeConfiguration(
FilesystemId=config['volume']['efs_id'],
RootDirectory=config['volume']['efs_directory_path']))
]
td = TaskDefinition(service_name + "TaskDefinition",
Family=service_name + "Family",
ContainerDefinitions=[cd],
TaskRoleArn=Ref(task_role),
**launch_type_td)
if 'custom_metrics' in config:
sd = SD(
service_name + "ServiceRegistry",
DnsConfig=DnsConfig(
RoutingPolicy="MULTIVALUE",
DnsRecords=[DnsRecord(TTL="60", Type="SRV")],
NamespaceId=ImportValue(
"{self.env}Cloudmap".format(**locals()))),
Tags=Tags(
{'METRICS_PATH': config['custom_metrics']['metrics_path']},
{'METRICS_PORT': config['custom_metrics']['metrics_port']
}))
self.template.add_resource(sd)
self.template.add_resource(td)
desired_count = self._get_desired_task_count_for_service(service_name)
deployment_configuration = DeploymentConfiguration(
MinimumHealthyPercent=100, MaximumPercent=200)
if 'http_interface' in config:
alb, lb, service_listener, alb_sg = self._add_alb(
cd, service_name, config, launch_type)
if launch_type == self.LAUNCH_TYPE_FARGATE:
# if launch type is ec2, then services inherit the ec2 instance security group
# otherwise, we need to specify a security group for the service
if 'custom_metrics' in config:
launch_type_svc = {
"ServiceRegistries": [
ServiceRegistry(
RegistryArn=GetAtt(sd, 'Arn'),
Port=int(
config['custom_metrics']['metrics_port']))
]
}
else:
service_security_group = SecurityGroup(
pascalcase("FargateService" + self.env + service_name),
GroupName=pascalcase("FargateService" + self.env +
service_name),
SecurityGroupIngress=[{
'IpProtocol':
'TCP',
'SourceSecurityGroupId':
Ref(alb_sg),
'ToPort':
int(config['http_interface']['container_port']),
'FromPort':
int(config['http_interface']['container_port']),
}],
VpcId=Ref(self.vpc),
GroupDescription=pascalcase("FargateService" +
self.env + service_name))
self.template.add_resource(service_security_group)
launch_type_svc['NetworkConfiguration'] = NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
],
SecurityGroups=[
ImportValue("{self.env}Ec2Host".format(
**locals())) if 'custom_metrics' in
config else Ref(service_security_group)
]))
else:
if 'custom_metrics' in config:
launch_type_svc = {
"ServiceRegistries": [
ServiceRegistry(
RegistryArn=GetAtt(sd, 'Arn'),
Port=int(
config['custom_metrics']['metrics_port']))
],
"NetworkConfiguration":
NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
SecurityGroups=[
ImportValue("{self.env}Ec2Host".format(
**locals()))
],
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
])),
'PlacementStrategies':
self.PLACEMENT_STRATEGIES
}
else:
launch_type_svc = {
'Role': Ref(self.ecs_service_role),
'PlacementStrategies': self.PLACEMENT_STRATEGIES
}
svc = Service(
service_name,
LoadBalancers=[lb],
Cluster=self.cluster_name,
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DependsOn=service_listener.title,
LaunchType=launch_type,
**launch_type_svc,
)
self.template.add_output(
Output(service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')))
self.template.add_output(
Output(
service_name + "URL",
Description="The URL at which the service is accessible",
Value=Sub("https://${" + alb.name + ".DNSName}")))
self.template.add_resource(svc)
else:
launch_type_svc = {}
if launch_type == self.LAUNCH_TYPE_FARGATE:
# if launch type is ec2, then services inherit the ec2 instance security group
# otherwise, we need to specify a security group for the service
if 'custom_metrics' in config:
launch_type_svc = {
"ServiceRegistries": [
ServiceRegistry(
RegistryArn=GetAtt(sd, 'Arn'),
Port=int(
config['custom_metrics']['metrics_port']))
],
'NetworkConfiguration':
NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
],
SecurityGroups=[
ImportValue("{self.env}Ec2Host".format(
**locals()))
]))
}
else:
service_security_group = SecurityGroup(
pascalcase("FargateService" + self.env + service_name),
GroupName=pascalcase("FargateService" + self.env +
service_name),
SecurityGroupIngress=[],
VpcId=Ref(self.vpc),
GroupDescription=pascalcase("FargateService" +
self.env + service_name))
self.template.add_resource(service_security_group)
launch_type_svc = {
'NetworkConfiguration':
NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
],
SecurityGroups=[Ref(service_security_group)]))
}
else:
if 'custom_metrics' in config:
launch_type_svc = {
"ServiceRegistries": [
ServiceRegistry(
RegistryArn=GetAtt(sd, 'Arn'),
Port=int(
config['custom_metrics']['metrics_port']))
],
"NetworkConfiguration":
NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
SecurityGroups=[
ImportValue("{self.env}Ec2Host".format(
**locals()))
],
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
])),
'PlacementStrategies':
self.PLACEMENT_STRATEGIES
}
else:
launch_type_svc = {
'PlacementStrategies': self.PLACEMENT_STRATEGIES
}
svc = Service(service_name,
Cluster=self.cluster_name,
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DeploymentConfiguration=deployment_configuration,
LaunchType=launch_type,
**launch_type_svc)
self.template.add_output(
Output(service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')))
self.template.add_resource(svc)
self._add_service_alarms(svc)
)
if entrypoint:
entrypoint = entrypoint if isinstance(entrypoint,
list) else [entrypoint]
definition.EntryPoint = entrypoint
if command:
command = command if isinstance(command, list) else [command]
definition.Command = command
if linux_parameters:
definition.LinuxParameters = linux_parameters
for volume in volumes:
volume_name = '{}{}'.format(
name,
''.join([i for i in volume[0].capitalize() if i.isalpha()]))
task.Volumes.append(
Volume(Name=volume_name,
Host=Host(SourcePath=('/mnt/{}'.format(volume[0])))))
definition.MountPoints.append(
MountPoint(ContainerPath=volume[1], SourceVolume=volume_name))
for port in ports:
definition.PortMappings.append(
PortMapping(
ContainerPort=port,
HostPort=port,
Protocol='tcp',
))
for depend in depends:
definition.DependsOn.append(
ContainerDependency(
Condition='START',
ContainerName=depend,
))