def environment(self):
env_dict = self.get_variables()["Environment"]
if not env_dict:
return NoValue
env_list = []
for k, v in env_dict.items():
env_list.append(ecs.Environment(Name=str(k), Value=str(v)))
return env_list
def environment(self):
env_dict = self.get_variables()["Environment"]
if not env_dict:
return NoValue
env_list = []
# Sort it first to avoid dict sort issues on different machines
sorted_env = sorted(env_dict.items(), key=lambda pair: pair[0])
for k, v in sorted_env:
env_list.append(ecs.Environment(Name=str(k), Value=str(v)))
return env_list
def create_task_definition(self, t):
return t.add_resource(ecs.TaskDefinition(
self.resource_name_format % ('TaskDefinition'),
Family=Sub('${AWS::StackName}-app'),
ContainerDefinitions=[ecs.ContainerDefinition(
Name=self.container_name,
Image=self.docker_image,
Memory=self.memory,
PortMappings=[ecs.PortMapping(ContainerPort=self.port)],
Environment=[
ecs.Environment(Name=key, Value=value)
for (key, value) in sorted(self.get_envs(t).items())
]
)]
))
def build(self, t):
env = []
logConf = None
data = deepcopy(self.data)
if data.get('Environment'):
for v in data.get('Environment'):
e = ecs.Environment(Name=self.jinja_txt(v['name']),
Value=self.jinja_txt(v['value']))
env.append(e)
del data['Environment']
if data.get('Image'):
data['Image'] = self.jinja_txt(data['Image'])
if data.get('LogConfiguration'):
logConf = ecs.LogConfiguration(
LogDriver=data.get('LogConfiguration').get('LogDriver'),
Options={})
if data.get('LogConfiguration').get('Options').get(
'awslogs-group'):
logConf.Options['awslogs-group'] = self.jinja_txt(
data['LogConfiguration']['Options']['awslogs-group'])
if data.get('LogConfiguration').get('Options').get(
'awslogs-region'):
logConf.Options['awslogs-region'] = data['LogConfiguration'][
'Options']['awslogs-region']
if data.get('LogConfiguration').get('Options').get(
'awslogs-stream-prefix'):
logConf.Options['awslogs-stream-prefix'] = data[
'LogConfiguration']['Options']['awslogs-stream-prefix']
del data['LogConfiguration']
ctr = ecs.ContainerDefinition(**data)
ctr.Environment = env
if logConf is not None:
ctr.LogConfiguration = logConf
return ctr
def munge_container_attributes(self):
image = ':'.join([
self.vars['ContainerImage'],
str(self.vars['ContainerImageVersion']),
])
# munge ECR image path
if self.vars['UseECR']:
image = Join(
'.', [AccountId, 'dkr.ecr', Region, 'amazonaws.com/' + image])
# set required attributes
required = dict(
Name=self.vars['ContainerName'],
Image=image,
PortMappings=[
ecs.PortMapping(
ContainerPort=self.vars['ContainerPort'],
Protocol=self.vars['ContainerProtocol'],
)
],
LogConfiguration=ecs.LogConfiguration(
LogDriver='awslogs',
Options={
'awslogs-group': Ref(self.log_group),
'awslogs-region': Region,
'awslogs-stream-prefix': self.vars['ContainerName'],
},
),
)
# deal with additional attributes
if self.vars['AdditionalContainerAttributes']:
added = self.vars['AdditionalContainerAttributes']
# deal with troposphere AWSProperty objects
if 'Environment' in added:
added['Environment'] = [
ecs.Environment(**m) for m in added['Environment']
]
if 'ExtraHosts' in added:
added['ExtraHosts'] = [
ecs.HostEntry(**m) for m in added['ExtraHosts']
]
if 'LinuxParameters' in added:
added['LinuxParameters'] = [
ecs.LinuxParameters(**m) for m in added['LinuxParameters']
]
if 'MountPoints' in added:
added['MountPoints'] = [
ecs.MountPoint(**m) for m in added['MountPoints']
]
if 'Ulimit' in added:
added['Ulimit'] = [ecs.Ulimit(**m) for m in added['Ulimit']]
if 'VolumesFrom' in added:
added['VolumesFrom'] = [
ecs.VolumesFrom(**m) for m in added['VolumesFrom']
]
# munge memory
if not 'Memory' in added and not 'MemoryReservation' in added:
added['MemoryReservation'] = self.vars['Memory']
attributes = added.copy()
else:
attributes = dict()
# merge in required attributes.
attributes.update(required)
return attributes
def get_empire_environment(self):
database_url = Join("", [
"postgres://",
Ref("DatabaseUser"), ":",
Ref("DatabasePassword"), "@",
Ref("DatabaseHost"), "/empire"
])
return [
ecs.Environment(Name="EMPIRE_ENVIRONMENT",
Value=Ref("Environment")),
ecs.Environment(Name="EMPIRE_SCHEDULER",
Value=Ref("EmpireScheduler")),
ecs.Environment(Name="EMPIRE_REPORTER", Value=Ref("Reporter")),
ecs.Environment(Name="EMPIRE_S3_TEMPLATE_BUCKET",
Value=Ref("TemplateBucket")),
ecs.Environment(Name="EMPIRE_GITHUB_CLIENT_ID",
Value=Ref("GitHubClientId")),
ecs.Environment(Name="EMPIRE_GITHUB_CLIENT_SECRET",
Value=Ref("GitHubClientSecret")),
ecs.Environment(Name="EMPIRE_DATABASE_URL", Value=database_url),
ecs.Environment(Name="EMPIRE_TOKEN_SECRET",
Value=Ref("TokenSecret")),
ecs.Environment(Name="AWS_REGION", Value=Ref("AWS::Region")),
ecs.Environment(Name="EMPIRE_PORT", Value="8081"),
ecs.Environment(Name="EMPIRE_AWS_DEBUG", Value=Ref("AwsDebug")),
ecs.Environment(Name="EMPIRE_GITHUB_ORGANIZATION",
Value=Ref("GitHubOrganization")),
ecs.Environment(Name="EMPIRE_GITHUB_WEBHOOKS_SECRET",
Value=Ref("GitHubWebhooksSecret")),
ecs.Environment(Name="EMPIRE_GITHUB_DEPLOYMENTS_ENVIRONMENT",
Value=Ref("GitHubDeploymentsEnvironment")),
ecs.Environment(Name="EMPIRE_EVENTS_BACKEND",
Value=Ref("EventsBackend")),
ecs.Environment(Name="EMPIRE_SNS_TOPIC",
Value=If(
"EnableSNSEvents",
If("CreateSNSTopic", Ref(EVENTS_TOPIC),
Ref("EventsSNSTopicName")),
"AWS::NoValue")),
ecs.Environment(Name="EMPIRE_TUGBOAT_URL",
Value=Ref("TugboatUrl")),
ecs.Environment(Name="EMPIRE_LOGS_STREAMER",
Value=Ref("LogsStreamer")),
ecs.Environment(Name="EMPIRE_ECS_CLUSTER",
Value=Ref("MinionCluster")),
ecs.Environment(Name="EMPIRE_ECS_SERVICE_ROLE",
Value=Ref("ServiceRole")),
ecs.Environment(Name="EMPIRE_ROUTE53_INTERNAL_ZONE_ID",
Value=Ref("InternalZoneId")),
ecs.Environment(Name="EMPIRE_EC2_SUBNETS_PRIVATE",
Value=Join(",", Ref("PrivateSubnets"))),
ecs.Environment(Name="EMPIRE_EC2_SUBNETS_PUBLIC",
Value=Join(",", Ref("PublicSubnets"))),
ecs.Environment(Name='EMPIRE_ELB_VPC_ID', Value=Ref('VpcId')),
ecs.Environment(Name="EMPIRE_ELB_SG_PRIVATE",
Value=Ref("PrivateAppELBSG")),
ecs.Environment(Name="EMPIRE_ELB_SG_PUBLIC",
Value=Ref("PublicAppELBSG")),
ecs.Environment(Name="EMPIRE_ELB_VPC_ID", Value=Ref("VpcId")),
ecs.Environment(Name="EMPIRE_GITHUB_DEPLOYMENTS_IMAGE_BUILDER",
Value="conveyor"),
ecs.Environment(Name="EMPIRE_CONVEYOR_URL",
Value=Ref("ConveyorUrl")),
ecs.Environment(Name="EMPIRE_RUN_LOGS_BACKEND",
Value=Ref("RunLogsBackend")),
ecs.Environment(Name="EMPIRE_CUSTOM_RESOURCES_TOPIC",
Value=Ref("CustomResourcesTopic")),
ecs.Environment(Name="EMPIRE_CUSTOM_RESOURCES_QUEUE",
Value=Ref("CustomResourcesQueue")),
ecs.Environment(Name="EMPIRE_CLOUDWATCH_LOG_GROUP",
Value=If("EnableCloudwatchLogs", Ref(RUN_LOGS),
"AWS::NoValue")),
If('RequireCommitMessages',
ecs.Environment(Name='EMPIRE_MESSAGES_REQUIRED', Value='true'),
Ref('AWS::NoValue')),
]
# ECS Cluster
# =============
cluster = ecs.Cluster("JicketCluster")
cluster.ClusterName = "Jicket"
t.add_resource(cluster)
# ECS Task Definition
# =====================
taskdef = ecs.TaskDefinition("JicketTask")
contdef = ecs.ContainerDefinition()
contdef.Cpu = 0
contdef.Environment = [
ecs.Environment(Name="JICKET_IMAP_HOST", Value=jicketimaphost),
ecs.Environment(Name="JICKET_JIRA_USER", Value=jicketjirauser),
ecs.Environment(Name="JICKET_TICKET_ADDRESS", Value=jicketticketaddress),
ecs.Environment(Name="JICKET_SMTP_HOST", Value=jicketsmtphost),
ecs.Environment(Name="JICKET_JIRA_PASS", Value=jicketjirapass),
ecs.Environment(Name="JICKET_JIRA_PROJECT", Value=jicketjiraproject),
ecs.Environment(Name="JICKET_JIRA_URL", Value=jicketjiraurl),
ecs.Environment(Name="JICKET_THREAD_TEMPLATE", Value=jicketthreadtemplate),
ecs.Environment(Name="JICKET_IMAP_PASS", Value=jicketimappass),
ecs.Environment(Name="JICKET_IMAP_USER", Value=jicketimapuser),
]
contdef.Image = "kwpcommunications/jicket:latest"
contdef.MemoryReservation = 512
contdef.Name = "jicket"
logconf = ecs.LogConfiguration()
logconf.LogDriver = "awslogs"
def GenerateSerializerLayer():
t = Template()
t.add_description("""\
Serializer Layer
""")
serializername_param = t.add_parameter(
Parameter(
"SerializerName",
Description="Serializer Name (default: SampleSerializer)",
Type="String",
Default="SampleSerializer",
))
stackname_param = t.add_parameter(
Parameter(
"StackName",
Description="Environment Name (default: hackathon)",
Type="String",
Default="hackathon",
))
clusterid_param = t.add_parameter(
Parameter(
"ClusterId",
Type="String",
Description="ClusterId to run the serializer on",
))
docker_id_param = t.add_parameter(
Parameter("DockerId",
Description="DockerId (default: centos:latest)",
Type="String",
Default="centos:latest"))
execution_role_param = t.add_parameter(
Parameter(
"ExecutionRole",
Description="Lambda Execution Role",
Type="String",
))
hashkeyname_param = t.add_parameter(
Parameter(
"HaskKeyElementName",
Description="HashType PrimaryKey Name (default: id)",
Type="String",
AllowedPattern="[a-zA-Z0-9]*",
MinLength="1",
MaxLength="2048",
ConstraintDescription="must contain only alphanumberic characters",
Default="id"))
dirtylist_param = t.add_parameter(
Parameter("DirtyList",
Description="DirtyList Table Name",
Type="String"))
runlog_param = t.add_parameter(
Parameter("RunLog", Description="RunLog Table Name", Type="String"))
canonical_prefix_param = t.add_parameter(
Parameter("CanonicalPrefix",
Description="Canonical Tables Prefix",
Type="String",
Default="CANON_"))
serializer_table = t.add_resource(
dynamodb.Table(
"sampleSerializerTable",
AttributeDefinitions=[
dynamodb.AttributeDefinition(Ref(hashkeyname_param), "S"),
],
KeySchema=[dynamodb.Key(Ref(hashkeyname_param), "HASH")],
ProvisionedThroughput=dynamodb.ProvisionedThroughput(1, 1)))
task_definition = t.add_resource(
ecs.TaskDefinition(
'TaskDefinition',
ContainerDefinitions=[
ecs.ContainerDefinition(
Name=Join('', [Ref(stackname_param), "Task"]),
Image=Ref(docker_id_param),
Environment=[
ecs.Environment(Name="LAYER",
Value=Ref(stackname_param)),
ecs.Environment(Name="SERIALIZER",
Value=Ref(serializername_param)),
ecs.Environment(Name="SERIALIZER_TABLE",
Value=Ref(serializer_table)),
ecs.Environment(Name="CANONICAL_PREFIX",
Value=Ref(canonical_prefix_param)),
ecs.Environment(Name="DIRTYLIST",
Value=Ref(dirtylist_param)),
ecs.Environment(Name="RUNLOG", Value=Ref(runlog_param))
],
Memory=512,
)
],
Volumes=[],
))
t.add_output([
Output(
"taskdefinitionid",
Description="Task Definition Id",
Value=Ref(task_definition),
)
])
return t
PortMappings=[
ecs.PortMapping(HostPort=0,
ContainerPort=Ref(container_port),
Protocol="tcp"),
],
Essential=True,
Command=["/usr/sbin/apache2ctl", "-D", "FOREGROUND"],
Name=Ref(container_name),
Image=Join(
"", [Ref(ecr), "/",
Ref(image_name), ":",
Ref(image_tag)]),
Cpu=200,
MemoryReservation=512,
Environment=[
ecs.Environment(Name="AWSStackName",
Value=Ref("AWS::StackName")),
ecs.Environment(Name="AWSRegion",
Value=Ref("AWS::Region")),
ecs.Environment(Name="SCR_HOST", Value=Ref(scr_hostname)),
ecs.Environment(Name="ALB",
Value=ImportValue(
Sub("${AlbStack}-AlbPrivateDNSName")))
],
)
],
))
"""
We need new listeners
"""
# app_lb_listener = t.add_resource(elasticloadbalancingv2.Listener(
"awslogs-group":
'/ecs/aws-currency-exchange-service-h2-xray',
"awslogs-region": 'us-east-2',
"awslogs-stream-prefix": 'ecs'
}),
Name='aws-currency-exchange-service-h2-xray',
PortMappings=[
ecs.PortMapping(ContainerPort=8000,
HostPort=8000,
Protocol='tcp')
]),
ecs.ContainerDefinition(
Environment=[
ecs.Environment(
Name='APPMESH_VIRTUAL_NODE_NAME',
Value=
'mesh/my-service-mesh/virtualNode/currency-exchange-service-vn'
),
ecs.Environment(Name='ENABLE_ENVOY_XRAY_TRACING',
Value='1'),
ecs.Environment(Name='ENVOY_LOG_LEVEL', Value='trace')
],
Essential=True,
HealthCheck=ecs.HealthCheck(Command=[
'CMD-SHELL',
'curl -s http://localhost:9901/server_info | grep state | grep -q LIVE'
],
Interval=5,
Timeout=2,
Retries=3,
StartPeriod=10),
def __init__(self, key):
Environments_Base = [
ecs.Environment(
Name='EnvRegion',
Value=Ref('AWS::Region')
),
ecs.Environment(
Name='Env',
Value=Ref('Env')
),
ecs.Environment(
Name='EnvAbbr',
Value=Ref('EnvShort')
),
ecs.Environment(
Name='EnvBrand',
Value=cfg.BrandDomain
),
ecs.Environment(
Name='EnvRole',
Value=Ref('EnvRole')
),
ecs.Environment(
Name='EnvStackName',
Value=Ref('AWS::StackName')
),
ecs.Environment(
Name='EnvClusterStackName',
Value=get_endvalue('ClusterStack')
),
]
Containers = []
for n, v in cfg.ContainerDefinitions.items():
Environments = []
MountPoints = []
name = f'ContainerDefinitions{n}' # Ex. ContainerDefinitions1
# parameters
# if ContainerDefinitions have RepoName
# use different EnvApp version
if n == 1 or 'RepoName' in v:
nameenvapp = f'EnvApp{n}Version' # Ex. EnvApp1Version
EnvApp = Parameter(nameenvapp)
EnvApp.Description = nameenvapp
EnvApp.AllowedPattern = '^[a-zA-Z0-9-_.]*$'
EnvApp.Default = '1'
add_obj(EnvApp)
# outputs
o_EnvAppOut = Output(nameenvapp)
o_EnvAppOut.Value = Ref(nameenvapp)
# and use different output for RepoName
if cfg.RepoName != 'None':
if 'RepoName' in v:
o_Repo = Output(f'{name}RepoName')
o_Repo.Value = get_endvalue(f'{name}RepoName')
else:
o_Repo = Output('RepoName')
o_Repo.Value = get_endvalue('RepoName')
add_obj(o_Repo)
add_obj([
o_EnvAppOut,
])
# outputs
EnvValue_Out_String = []
EnvValue_Out_Map = {}
if 'Envs' in v:
for m, w in v['Envs'].items():
envname = f'{name}Envs{m}'
envkeyname = w['Name']
# parameters
EnvValue = Parameter(f'{envname}Value')
EnvValue.Description = (
f'{envkeyname} - empty for default based on env/role')
# If key NoParam is present skip adding Parameters
# (usefull as they have a limited max number)
if 'NoParam' not in w:
add_obj(EnvValue)
Environment = ECSEnvironment(envname, key=w)
Environments.append(Environment)
# outputs
EnvValue_Out_String.append(
'%s=${%s}' % (envkeyname, envkeyname))
EnvValue_Out_Map.update({
envkeyname: Environment.Value
})
o_EnvValueOut = Output(f'{name}Envs')
o_EnvValueOut.Value = Sub(
','.join(EnvValue_Out_String), **EnvValue_Out_Map)
add_obj(o_EnvValueOut)
Environments.extend(Environments_Base)
# parameters
if 'Cpu' in v:
p_Cpu = Parameter(f'{name}Cpu')
p_Cpu.Description = (
'Cpu Share for containers - '
'empty for default based on env/role')
add_obj(p_Cpu)
if 'Memory' in v:
p_Memory = Parameter(f'{name}Memory')
p_Memory.Description = (
'Memory hard limit for containers - '
'empty for default based on env/role')
add_obj(p_Memory)
if 'MemoryReservation' in v:
p_MemoryReservation = Parameter(f'{name}MemoryReservation')
p_MemoryReservation.Description = (
'Memory soft limit for containers - '
'empty for default based on env/role')
add_obj(p_MemoryReservation)
if 'Command' in v:
p_Command = Parameter(f'{name}Command')
p_Command.Description = 'Command to execute'
p_Command.Type = 'CommaDelimitedList'
add_obj(p_Command)
Container = ECSContainerDefinition(name, key=v, index=n)
Container.Environment = Environments
# Trick to force reload of Service using parameter
Container.DockerLabels = If(
'DockerLabelLastUpdateOverride',
{'LastUpdate': Ref('DockerLabelLastUpdate')},
Ref('AWS::NoValue'),
)
Containers.append(Container)
# outputs
Constraints_Out_String = []
Constraints_Out_Map = {}
if 'Cpu' in v:
Constraints_Out_String.append(
'Cpu:${Cpu}')
Constraints_Out_Map.update({
'Cpu': Container.Cpu
})
if 'Memory' in v:
Constraints_Out_String.append(
'Memory:${Memory}')
Constraints_Out_Map.update({
'Memory': Container.Memory
})
if 'MemoryReservation' in v:
Constraints_Out_String.append(
'MemoryReservation:${MemoryReservation}')
Constraints_Out_Map.update({
'MemoryReservation': Container.MemoryReservation
})
if Constraints_Out_String:
o_Constraints = Output(f'{name}Constraints')
o_Constraints.Value = Sub(
','.join(Constraints_Out_String), **Constraints_Out_Map)
add_obj(o_Constraints)
if 'Command' in v:
o_Command = Output(f'{name}Command')
o_Command.Value = Join(',', get_endvalue(f'{name}Command'))
add_obj(o_Command)
# Resources
R_TaskDefinition = ECSTaskDefinition('TaskDefinition')
R_TaskDefinition.ContainerDefinitions = Containers
if cfg.Volumes:
Volumes = []
for n, v in cfg.Volumes.items():
Volume = ECSVolume(n, key=v)
Volumes.append(Volume)
R_TaskDefinition.Volumes = Volumes
add_obj([
R_TaskDefinition,
])
elasticloadbalancingv2.Action(
Type='forward',
TargetGroupArn=keycloakTargetGroup.Ref(),
)
]))
keycloakTask = t.add_resource(
ecs.TaskDefinition(
'KeycloakTask',
ContainerDefinitions=[
ecs.ContainerDefinition(
'KeycloakContainer',
Name='keycloak',
Image=Ref('KeycloakImage'),
Environment=[
ecs.Environment(Name='KEYCLOAK_USER',
Value=Ref('KeycloakUser')),
ecs.Environment(Name='KEYCLOAK_PASSWORD',
Value=Ref('KeycloakPassword')),
ecs.Environment(Name='DB_VENDOR', Value='mysql'),
ecs.Environment(Name='DB_ADDR',
Value=ImportValue(
Sub('${CoreStack}-MySQL-Address'))),
ecs.Environment(Name='DB_PORT',
Value=ImportValue(
Sub('${CoreStack}-MySQL-Port'))),
ecs.Environment(Name='DB_DATABASE',
Value=Ref('MySQLDbName')),
ecs.Environment(Name='DB_USER', Value=Ref('MySQLUser')),
ecs.Environment(Name='DB_PASSWORD',
Value=Ref('MySQLPass')),
ecs.Environment(Name='PROXY_ADDRESS_FORWARDING',