def test_successful_build_config_from_without_duplicates(
self, mock_secrets_manager, mock_parameter_store):
env_name = "staging"
cloudlift_service_name = "Dummy"
sample_env_file_path = "test-env.sample"
essential_container_name = "mainService"
secrets_name = "main"
mock_store = MagicMock()
mock_parameter_store.return_value = mock_store
mock_store.get_existing_config.return_value = ({
'PORT':
'80',
"LABEL":
"arn_for_secret_at_v1"
}, {})
mock_secrets_manager.get_config.return_value = {}
actual_configurations = build_config(env_name, cloudlift_service_name,
"", sample_env_file_path,
essential_container_name, None)
expected_configurations = {
"mainService": {
"secrets": {},
"environment": {
"PORT": "80",
"LABEL": "arn_for_secret_at_v1"
}
}
}
self.assertDictEqual(expected_configurations, actual_configurations)
def test_successful_build_config_from_only_param_store(
self, mock_secrets_manager, mock_parameter_store):
env_name = "staging"
cloudlift_service_name = "Dummy"
sample_env_file_path = "test-env.sample"
essential_container_name = "mainServiceContainer"
mock_store = MagicMock()
mock_parameter_store.return_value = mock_store
mock_store.get_existing_config.return_value = ({
'PORT': '80',
'LABEL': 'Dummy'
}, {})
actual_configurations = build_config(env_name, cloudlift_service_name,
"", sample_env_file_path,
essential_container_name, None)
expected_configurations = {
"mainServiceContainer": {
"environment": {
"LABEL": "Dummy",
"PORT": "80"
},
"secrets": {}
}
}
self.assertDictEqual(expected_configurations, actual_configurations)
mock_secrets_manager.get_config.assert_not_called()
def create(self):
log_warning(
"Create task definition to {self.region}".format(**locals()))
if not os.path.exists(self.env_sample_file):
raise UnrecoverableException('env.sample not found. Exiting.')
ecr_client = EcrClient(self.name, self.region, self.build_args)
ecr_client.set_version(self.version)
log_intent("name: " + self.name + " | environment: " +
self.environment + " | version: " + str(ecr_client.version))
log_bold("Checking image in ECR")
ecr_client.build_and_upload_image()
log_bold("Creating task definition\n")
env_config = build_config(self.environment, self.name,
self.env_sample_file)
container_definition_arguments = {
"environment": [{
"name": k,
"value": v
} for (k, v) in env_config],
"name": pascalcase(self.name) + "Container",
"image": _complete_image_url(ecr_client),
"essential": True,
"logConfiguration": self._gen_log_config(pascalcase(self.name)),
"memoryReservation": 1024
}
ecs_client = EcsClient(region=self.region)
ecs_client.register_task_definition(self._task_defn_family(),
[container_definition_arguments],
[], None)
log_bold("Task definition successfully created\n")
def test_build_config_for_secrets_manager_from_multiple_files_already_present(
self, mock_getcwd, m_secrets_manager):
env_name = "staging"
service_name = "Dummy"
sample_env_file_path = "test-env.sample"
essential_container_name = "mainService"
ecs_service_name = "dummy-ecs-service"
secrets = {
'key1': 'actualValue1',
'key2': 'actualValue2',
'key3': 'actualValue3',
'key4': 'actualValue4',
}
def get_config(secret_name, env):
if secret_name == 'dummy-secrets-staging':
return {
'secrets': {
'key1': 'actualValue1',
'key2': 'actualValue2',
'key3': 'actualValue3',
}
}
if secret_name == 'dummy-secrets-staging/app1':
return {
'secrets': {
'key4': 'actualValue4',
'key5': 'actualValue5',
}
}
if secret_name == get_automated_injected_secret_name(
env_name, service_name, ecs_service_name):
return {'ARN': 'injected-secret-arn', 'secrets': secrets}
return {'secrets': {}}
m_secrets_manager.get_config.side_effect = get_config
mock_getcwd.return_value = os.path.join(
os.path.dirname(__file__),
'../env_sample_files/env_sample_files_without_duplicate_keys',
)
result = build_config(env_name, service_name, ecs_service_name,
sample_env_file_path, essential_container_name,
"dummy-secrets-staging")
m_secrets_manager.set_secrets_manager_config.assert_not_called()
self.assertEqual(
{
essential_container_name: {
'environment': {},
'secrets': {
'CLOUDLIFT_INJECTED_SECRETS': 'injected-secret-arn'
}
}
}, result)
def test_failure_build_config_for_if_sample_config_has_additional_keys(
self, m_secrets_manager, m_parameter_store):
env_name = "staging"
service_name = "Dummy"
sample_env_file_path = "test-env.sample"
essential_container_name = "mainService"
mock_store = MagicMock()
m_parameter_store.return_value = mock_store
mock_store.get_existing_config.return_value = ({
'PORT':
'80',
"LABEL":
"arn_for_secret_at_v1"
}, {})
m_secrets_manager.get_config.return_value = {}
with pytest.raises(UnrecoverableException) as pytest_wrapped_e:
build_config(env_name, service_name, "", sample_env_file_path,
essential_container_name, None)
self.assertEqual(pytest_wrapped_e.type, UnrecoverableException)
self.assertEqual(
str(pytest_wrapped_e.value),
'"There is no config value for the keys {\'ADDITIONAL_CONFIG\'}"')
def update(self):
log_warning(
"Update task definition to {self.region}".format(**locals()))
if not os.path.exists(self.env_sample_file):
raise UnrecoverableException('env.sample not found. Exiting.')
ecr_client = EcrClient(self.name, self.region, self.build_args)
ecr_client.set_version(self.version)
log_intent("name: " + self.name + " | environment: " +
self.environment + " | version: " + str(ecr_client.version))
log_bold("Checking image in ECR")
ecr_client.build_and_upload_image()
log_bold("Updating task definition\n")
env_config = build_config(self.environment, self.name,
self.env_sample_file)
ecs_client = EcsClient(region=self.region)
deployment = DeployAction(ecs_client, self.cluster_name, None)
task_defn = self._apply_changes_over_current_task_defn(
env_config, ecs_client, ecr_client, deployment)
deployment.update_task_definition(task_defn)
log_bold("Task definition successfully updated\n")
def test_build_config_ignores_additional_keys_in_parameter_store(
self, m_secrets_mgr, m_parameter_store):
env_name = "staging"
service_name = "Dummy"
sample_env_file_path = "test-env.sample"
essential_container_name = "mainService"
secrets_name = "main"
mock_store = MagicMock()
m_parameter_store.return_value = mock_store
mock_store.get_existing_config.return_value = ({
"LABEL":
"dummyvalue",
'PORT':
'80',
'ADDITIONAL_KEY_1':
'true'
}, {})
m_secrets_mgr.get_config.return_value = {
'secrets': {},
'ARN': "dummy_arn"
}
actual_configurations = build_config(env_name, service_name, "",
sample_env_file_path,
essential_container_name, None)
expected_configurations = {
"mainService": {
"secrets": {},
"environment": {
"PORT": "80",
"LABEL": "dummyvalue"
}
}
}
self.assertDictEqual(expected_configurations, actual_configurations)
def _add_service(self, service_name, config):
launch_type = self.LAUNCH_TYPE_FARGATE if 'fargate' in config else self.LAUNCH_TYPE_EC2
env_config = build_config(
self.env,
self.application_name,
self.env_sample_file_path
)
container_definition_arguments = {
"Environment": [
Environment(Name=k, Value=v) for (k, v) in env_config
],
"Name": service_name + "Container",
"Image": self.ecr_image_uri + ':' + self.current_version,
"Essential": 'true',
"LogConfiguration": self._gen_log_config(service_name),
"MemoryReservation": int(config['memory_reservation']),
"Cpu": 0
}
if 'http_interface' in config:
container_definition_arguments['PortMappings'] = [
PortMapping(
ContainerPort=int(
config['http_interface']['container_port']
)
)
]
if config['command'] is not None:
container_definition_arguments['Command'] = [config['command']]
cd = ContainerDefinition(**container_definition_arguments)
task_role = self.template.add_resource(Role(
service_name + "Role",
AssumeRolePolicyDocument=PolicyDocument(
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", ["ecs-tasks.amazonaws.com"])
)
]
)
))
launch_type_td = {}
if launch_type == self.LAUNCH_TYPE_FARGATE:
launch_type_td = {
'RequiresCompatibilities': ['FARGATE'],
'ExecutionRoleArn': boto3.resource('iam').Role('ecsTaskExecutionRole').arn,
'NetworkMode': 'awsvpc',
'Cpu': str(config['fargate']['cpu']),
'Memory': str(config['fargate']['memory'])
}
td = TaskDefinition(
service_name + "TaskDefinition",
Family=service_name + "Family",
ContainerDefinitions=[cd],
TaskRoleArn=Ref(task_role),
**launch_type_td
)
self.template.add_resource(td)
desired_count = self._get_desired_task_count_for_service(service_name)
deployment_configuration = DeploymentConfiguration(
MinimumHealthyPercent=100,
MaximumPercent=200
)
if 'http_interface' in config:
alb, lb, service_listener, alb_sg = self._add_alb(cd, service_name, config, launch_type)
if launch_type == self.LAUNCH_TYPE_FARGATE:
# if launch type is ec2, then services inherit the ec2 instance security group
# otherwise, we need to specify a security group for the service
service_security_group = SecurityGroup(
pascalcase("FargateService" + self.env + service_name),
GroupName=pascalcase("FargateService" + self.env + service_name),
SecurityGroupIngress=[{
'IpProtocol': 'TCP',
'SourceSecurityGroupId': Ref(alb_sg),
'ToPort': int(config['http_interface']['container_port']),
'FromPort': int(config['http_interface']['container_port']),
}],
VpcId=Ref(self.vpc),
GroupDescription=pascalcase("FargateService" + self.env + service_name)
)
self.template.add_resource(service_security_group)
launch_type_svc = {
'NetworkConfiguration': NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
],
SecurityGroups=[
Ref(service_security_group)
]
)
)
}
else:
launch_type_svc = {
'Role': Ref(self.ecs_service_role),
'PlacementStrategies': self.PLACEMENT_STRATEGIES
}
svc = Service(
service_name,
LoadBalancers=[lb],
Cluster=self.cluster_name,
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DependsOn=service_listener.title,
LaunchType=launch_type,
**launch_type_svc,
)
self.template.add_output(
Output(
service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')
)
)
self.template.add_output(
Output(
service_name + "URL",
Description="The URL at which the service is accessible",
Value=Sub("https://${" + alb.name + ".DNSName}")
)
)
self.template.add_resource(svc)
else:
launch_type_svc = {}
if launch_type == self.LAUNCH_TYPE_FARGATE:
# if launch type is ec2, then services inherit the ec2 instance security group
# otherwise, we need to specify a security group for the service
service_security_group = SecurityGroup(
pascalcase("FargateService" + self.env + service_name),
GroupName=pascalcase("FargateService" + self.env + service_name),
SecurityGroupIngress=[],
VpcId=Ref(self.vpc),
GroupDescription=pascalcase("FargateService" + self.env + service_name)
)
self.template.add_resource(service_security_group)
launch_type_svc = {
'NetworkConfiguration': NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
Subnets=[
Ref(self.private_subnet1),
Ref(self.private_subnet2)
],
SecurityGroups=[
Ref(service_security_group)
]
)
)
}
else:
launch_type_svc = {
'PlacementStrategies': self.PLACEMENT_STRATEGIES
}
svc = Service(
service_name,
Cluster=self.cluster_name,
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DeploymentConfiguration=deployment_configuration,
LaunchType=launch_type,
**launch_type_svc
)
self.template.add_output(
Output(
service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')
)
)
self.template.add_resource(svc)
self._add_service_alarms(svc)
def _add_service(self, service_name, config):
env_config = build_config(
self.env,
self.application_name,
self.env_sample_file_path
)
container_definition_arguments = {
"Environment": [
Environment(Name=k, Value=v) for (k, v) in env_config
],
"Name": service_name + "Container",
"Image": self.ecr_image_uri + ':' + self.current_version,
"Essential": 'true',
"LogConfiguration": self._gen_log_config(service_name),
"MemoryReservation": int(config['memory_reservation']),
"Cpu": 0
}
if 'http_interface' in config:
container_definition_arguments['PortMappings'] = [
PortMapping(
ContainerPort=int(
config['http_interface']['container_port']
)
)
]
if config['command'] is not None:
container_definition_arguments['Command'] = [config['command']]
cd = ContainerDefinition(**container_definition_arguments)
task_role = self.template.add_resource(Role(
service_name + "Role",
AssumeRolePolicyDocument=PolicyDocument(
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", ["ecs-tasks.amazonaws.com"])
)
]
)
))
td = TaskDefinition(
service_name + "TaskDefinition",
Family=service_name + "Family",
ContainerDefinitions=[cd],
TaskRoleArn=Ref(task_role)
)
self.template.add_resource(td)
desired_count = self._get_desired_task_count_for_service(service_name)
deployment_configuration = DeploymentConfiguration(
MinimumHealthyPercent=100,
MaximumPercent=200
)
if 'http_interface' in config:
alb, lb, service_listener = self._add_alb(cd, service_name, config)
svc = Service(
service_name,
LoadBalancers=[lb],
Cluster=self.cluster_name,
Role=Ref(self.ecs_service_role),
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DependsOn=service_listener.title,
PlacementStrategies=self.PLACEMENT_STRATEGIES
)
self.template.add_output(
Output(
service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')
)
)
self.template.add_output(
Output(
service_name + "URL",
Description="The URL at which the service is accessible",
Value=Sub("https://${" + alb.name + ".DNSName}")
)
)
self.template.add_resource(svc)
else:
svc = Service(
service_name,
Cluster=self.cluster_name,
TaskDefinition=Ref(td),
DesiredCount=desired_count,
DeploymentConfiguration=deployment_configuration,
PlacementStrategies=self.PLACEMENT_STRATEGIES
)
self.template.add_output(
Output(
service_name + 'EcsServiceName',
Description='The ECS name which needs to be entered',
Value=GetAtt(svc, 'Name')
)
)
self.template.add_resource(svc)
self._add_service_alarms(svc)
def test_build_config_for_secrets_manager_from_multiple_files(
self, mock_getcwd, m_secrets_manager, m_parameter_store):
env_name = "staging"
service_name = "Dummy"
sample_env_file_path = "test-env.sample"
essential_container_name = "mainService"
ecs_service_name = "dummy-ecs-service"
injected_secret_name = get_automated_injected_secret_name(
env_name, service_name, ecs_service_name)
class MockSecretManager:
injected_secret_name_call_count = 0
@classmethod
def get_config(cls, secret_name, env):
if secret_name == 'dummy-secrets-staging':
return {
'secrets': {
'key1': 'actualValue1',
'key2': 'actualValue2',
'key3': 'actualValue3',
}
}
if secret_name == 'dummy-secrets-staging/app1':
return {
'secrets': {
'key4': 'actualValue4',
'key5': 'actualValue5',
}
}
if secret_name == injected_secret_name and cls.injected_secret_name_call_count == 0:
cls.injected_secret_name_call_count += 1
raise Exception('not found')
if secret_name == injected_secret_name:
return {'ARN': 'injected-secret-arn'}
return {'secrets': {}}
m_secrets_manager.get_config.side_effect = MockSecretManager.get_config
mock_getcwd.return_value = os.path.join(
os.path.dirname(__file__),
'../env_sample_files/env_sample_files_without_duplicate_keys',
)
result = build_config(env_name, service_name, ecs_service_name,
sample_env_file_path, essential_container_name,
"dummy-secrets-staging")
m_secrets_manager.set_secrets_manager_config.assert_called_with(
env_name, injected_secret_name, {
'key1': 'actualValue1',
'key2': 'actualValue2',
'key3': 'actualValue3',
'key4': 'actualValue4',
})
self.assertEqual(
{
essential_container_name: {
'environment': {},
'secrets': {
'CLOUDLIFT_INJECTED_SECRETS': 'injected-secret-arn'
}
}
}, result)
