def test_successful_build_config_from_without_duplicates( self, mock_secrets_manager, mock_parameter_store): env_name = "staging" cloudlift_service_name = "Dummy" sample_env_file_path = "test-env.sample" essential_container_name = "mainService" secrets_name = "main" mock_store = MagicMock() mock_parameter_store.return_value = mock_store mock_store.get_existing_config.return_value = ({ 'PORT': '80', "LABEL": "arn_for_secret_at_v1" }, {}) mock_secrets_manager.get_config.return_value = {} actual_configurations = build_config(env_name, cloudlift_service_name, "", sample_env_file_path, essential_container_name, None) expected_configurations = { "mainService": { "secrets": {}, "environment": { "PORT": "80", "LABEL": "arn_for_secret_at_v1" } } } self.assertDictEqual(expected_configurations, actual_configurations)
def test_successful_build_config_from_only_param_store( self, mock_secrets_manager, mock_parameter_store): env_name = "staging" cloudlift_service_name = "Dummy" sample_env_file_path = "test-env.sample" essential_container_name = "mainServiceContainer" mock_store = MagicMock() mock_parameter_store.return_value = mock_store mock_store.get_existing_config.return_value = ({ 'PORT': '80', 'LABEL': 'Dummy' }, {}) actual_configurations = build_config(env_name, cloudlift_service_name, "", sample_env_file_path, essential_container_name, None) expected_configurations = { "mainServiceContainer": { "environment": { "LABEL": "Dummy", "PORT": "80" }, "secrets": {} } } self.assertDictEqual(expected_configurations, actual_configurations) mock_secrets_manager.get_config.assert_not_called()
def create(self): log_warning( "Create task definition to {self.region}".format(**locals())) if not os.path.exists(self.env_sample_file): raise UnrecoverableException('env.sample not found. Exiting.') ecr_client = EcrClient(self.name, self.region, self.build_args) ecr_client.set_version(self.version) log_intent("name: " + self.name + " | environment: " + self.environment + " | version: " + str(ecr_client.version)) log_bold("Checking image in ECR") ecr_client.build_and_upload_image() log_bold("Creating task definition\n") env_config = build_config(self.environment, self.name, self.env_sample_file) container_definition_arguments = { "environment": [{ "name": k, "value": v } for (k, v) in env_config], "name": pascalcase(self.name) + "Container", "image": _complete_image_url(ecr_client), "essential": True, "logConfiguration": self._gen_log_config(pascalcase(self.name)), "memoryReservation": 1024 } ecs_client = EcsClient(region=self.region) ecs_client.register_task_definition(self._task_defn_family(), [container_definition_arguments], [], None) log_bold("Task definition successfully created\n")
def test_build_config_for_secrets_manager_from_multiple_files_already_present( self, mock_getcwd, m_secrets_manager): env_name = "staging" service_name = "Dummy" sample_env_file_path = "test-env.sample" essential_container_name = "mainService" ecs_service_name = "dummy-ecs-service" secrets = { 'key1': 'actualValue1', 'key2': 'actualValue2', 'key3': 'actualValue3', 'key4': 'actualValue4', } def get_config(secret_name, env): if secret_name == 'dummy-secrets-staging': return { 'secrets': { 'key1': 'actualValue1', 'key2': 'actualValue2', 'key3': 'actualValue3', } } if secret_name == 'dummy-secrets-staging/app1': return { 'secrets': { 'key4': 'actualValue4', 'key5': 'actualValue5', } } if secret_name == get_automated_injected_secret_name( env_name, service_name, ecs_service_name): return {'ARN': 'injected-secret-arn', 'secrets': secrets} return {'secrets': {}} m_secrets_manager.get_config.side_effect = get_config mock_getcwd.return_value = os.path.join( os.path.dirname(__file__), '../env_sample_files/env_sample_files_without_duplicate_keys', ) result = build_config(env_name, service_name, ecs_service_name, sample_env_file_path, essential_container_name, "dummy-secrets-staging") m_secrets_manager.set_secrets_manager_config.assert_not_called() self.assertEqual( { essential_container_name: { 'environment': {}, 'secrets': { 'CLOUDLIFT_INJECTED_SECRETS': 'injected-secret-arn' } } }, result)
def test_failure_build_config_for_if_sample_config_has_additional_keys( self, m_secrets_manager, m_parameter_store): env_name = "staging" service_name = "Dummy" sample_env_file_path = "test-env.sample" essential_container_name = "mainService" mock_store = MagicMock() m_parameter_store.return_value = mock_store mock_store.get_existing_config.return_value = ({ 'PORT': '80', "LABEL": "arn_for_secret_at_v1" }, {}) m_secrets_manager.get_config.return_value = {} with pytest.raises(UnrecoverableException) as pytest_wrapped_e: build_config(env_name, service_name, "", sample_env_file_path, essential_container_name, None) self.assertEqual(pytest_wrapped_e.type, UnrecoverableException) self.assertEqual( str(pytest_wrapped_e.value), '"There is no config value for the keys {\'ADDITIONAL_CONFIG\'}"')
def update(self): log_warning( "Update task definition to {self.region}".format(**locals())) if not os.path.exists(self.env_sample_file): raise UnrecoverableException('env.sample not found. Exiting.') ecr_client = EcrClient(self.name, self.region, self.build_args) ecr_client.set_version(self.version) log_intent("name: " + self.name + " | environment: " + self.environment + " | version: " + str(ecr_client.version)) log_bold("Checking image in ECR") ecr_client.build_and_upload_image() log_bold("Updating task definition\n") env_config = build_config(self.environment, self.name, self.env_sample_file) ecs_client = EcsClient(region=self.region) deployment = DeployAction(ecs_client, self.cluster_name, None) task_defn = self._apply_changes_over_current_task_defn( env_config, ecs_client, ecr_client, deployment) deployment.update_task_definition(task_defn) log_bold("Task definition successfully updated\n")
def test_build_config_ignores_additional_keys_in_parameter_store( self, m_secrets_mgr, m_parameter_store): env_name = "staging" service_name = "Dummy" sample_env_file_path = "test-env.sample" essential_container_name = "mainService" secrets_name = "main" mock_store = MagicMock() m_parameter_store.return_value = mock_store mock_store.get_existing_config.return_value = ({ "LABEL": "dummyvalue", 'PORT': '80', 'ADDITIONAL_KEY_1': 'true' }, {}) m_secrets_mgr.get_config.return_value = { 'secrets': {}, 'ARN': "dummy_arn" } actual_configurations = build_config(env_name, service_name, "", sample_env_file_path, essential_container_name, None) expected_configurations = { "mainService": { "secrets": {}, "environment": { "PORT": "80", "LABEL": "dummyvalue" } } } self.assertDictEqual(expected_configurations, actual_configurations)
def _add_service(self, service_name, config): launch_type = self.LAUNCH_TYPE_FARGATE if 'fargate' in config else self.LAUNCH_TYPE_EC2 env_config = build_config( self.env, self.application_name, self.env_sample_file_path ) container_definition_arguments = { "Environment": [ Environment(Name=k, Value=v) for (k, v) in env_config ], "Name": service_name + "Container", "Image": self.ecr_image_uri + ':' + self.current_version, "Essential": 'true', "LogConfiguration": self._gen_log_config(service_name), "MemoryReservation": int(config['memory_reservation']), "Cpu": 0 } if 'http_interface' in config: container_definition_arguments['PortMappings'] = [ PortMapping( ContainerPort=int( config['http_interface']['container_port'] ) ) ] if config['command'] is not None: container_definition_arguments['Command'] = [config['command']] cd = ContainerDefinition(**container_definition_arguments) task_role = self.template.add_resource(Role( service_name + "Role", AssumeRolePolicyDocument=PolicyDocument( Statement=[ Statement( Effect=Allow, Action=[AssumeRole], Principal=Principal("Service", ["ecs-tasks.amazonaws.com"]) ) ] ) )) launch_type_td = {} if launch_type == self.LAUNCH_TYPE_FARGATE: launch_type_td = { 'RequiresCompatibilities': ['FARGATE'], 'ExecutionRoleArn': boto3.resource('iam').Role('ecsTaskExecutionRole').arn, 'NetworkMode': 'awsvpc', 'Cpu': str(config['fargate']['cpu']), 'Memory': str(config['fargate']['memory']) } td = TaskDefinition( service_name + "TaskDefinition", Family=service_name + "Family", ContainerDefinitions=[cd], TaskRoleArn=Ref(task_role), **launch_type_td ) self.template.add_resource(td) desired_count = self._get_desired_task_count_for_service(service_name) deployment_configuration = DeploymentConfiguration( MinimumHealthyPercent=100, MaximumPercent=200 ) if 'http_interface' in config: alb, lb, service_listener, alb_sg = self._add_alb(cd, service_name, config, launch_type) if launch_type == self.LAUNCH_TYPE_FARGATE: # if launch type is ec2, then services inherit the ec2 instance security group # otherwise, we need to specify a security group for the service service_security_group = SecurityGroup( pascalcase("FargateService" + self.env + service_name), GroupName=pascalcase("FargateService" + self.env + service_name), SecurityGroupIngress=[{ 'IpProtocol': 'TCP', 'SourceSecurityGroupId': Ref(alb_sg), 'ToPort': int(config['http_interface']['container_port']), 'FromPort': int(config['http_interface']['container_port']), }], VpcId=Ref(self.vpc), GroupDescription=pascalcase("FargateService" + self.env + service_name) ) self.template.add_resource(service_security_group) launch_type_svc = { 'NetworkConfiguration': NetworkConfiguration( AwsvpcConfiguration=AwsvpcConfiguration( Subnets=[ Ref(self.private_subnet1), Ref(self.private_subnet2) ], SecurityGroups=[ Ref(service_security_group) ] ) ) } else: launch_type_svc = { 'Role': Ref(self.ecs_service_role), 'PlacementStrategies': self.PLACEMENT_STRATEGIES } svc = Service( service_name, LoadBalancers=[lb], Cluster=self.cluster_name, TaskDefinition=Ref(td), DesiredCount=desired_count, DependsOn=service_listener.title, LaunchType=launch_type, **launch_type_svc, ) self.template.add_output( Output( service_name + 'EcsServiceName', Description='The ECS name which needs to be entered', Value=GetAtt(svc, 'Name') ) ) self.template.add_output( Output( service_name + "URL", Description="The URL at which the service is accessible", Value=Sub("https://${" + alb.name + ".DNSName}") ) ) self.template.add_resource(svc) else: launch_type_svc = {} if launch_type == self.LAUNCH_TYPE_FARGATE: # if launch type is ec2, then services inherit the ec2 instance security group # otherwise, we need to specify a security group for the service service_security_group = SecurityGroup( pascalcase("FargateService" + self.env + service_name), GroupName=pascalcase("FargateService" + self.env + service_name), SecurityGroupIngress=[], VpcId=Ref(self.vpc), GroupDescription=pascalcase("FargateService" + self.env + service_name) ) self.template.add_resource(service_security_group) launch_type_svc = { 'NetworkConfiguration': NetworkConfiguration( AwsvpcConfiguration=AwsvpcConfiguration( Subnets=[ Ref(self.private_subnet1), Ref(self.private_subnet2) ], SecurityGroups=[ Ref(service_security_group) ] ) ) } else: launch_type_svc = { 'PlacementStrategies': self.PLACEMENT_STRATEGIES } svc = Service( service_name, Cluster=self.cluster_name, TaskDefinition=Ref(td), DesiredCount=desired_count, DeploymentConfiguration=deployment_configuration, LaunchType=launch_type, **launch_type_svc ) self.template.add_output( Output( service_name + 'EcsServiceName', Description='The ECS name which needs to be entered', Value=GetAtt(svc, 'Name') ) ) self.template.add_resource(svc) self._add_service_alarms(svc)
def _add_service(self, service_name, config): env_config = build_config( self.env, self.application_name, self.env_sample_file_path ) container_definition_arguments = { "Environment": [ Environment(Name=k, Value=v) for (k, v) in env_config ], "Name": service_name + "Container", "Image": self.ecr_image_uri + ':' + self.current_version, "Essential": 'true', "LogConfiguration": self._gen_log_config(service_name), "MemoryReservation": int(config['memory_reservation']), "Cpu": 0 } if 'http_interface' in config: container_definition_arguments['PortMappings'] = [ PortMapping( ContainerPort=int( config['http_interface']['container_port'] ) ) ] if config['command'] is not None: container_definition_arguments['Command'] = [config['command']] cd = ContainerDefinition(**container_definition_arguments) task_role = self.template.add_resource(Role( service_name + "Role", AssumeRolePolicyDocument=PolicyDocument( Statement=[ Statement( Effect=Allow, Action=[AssumeRole], Principal=Principal("Service", ["ecs-tasks.amazonaws.com"]) ) ] ) )) td = TaskDefinition( service_name + "TaskDefinition", Family=service_name + "Family", ContainerDefinitions=[cd], TaskRoleArn=Ref(task_role) ) self.template.add_resource(td) desired_count = self._get_desired_task_count_for_service(service_name) deployment_configuration = DeploymentConfiguration( MinimumHealthyPercent=100, MaximumPercent=200 ) if 'http_interface' in config: alb, lb, service_listener = self._add_alb(cd, service_name, config) svc = Service( service_name, LoadBalancers=[lb], Cluster=self.cluster_name, Role=Ref(self.ecs_service_role), TaskDefinition=Ref(td), DesiredCount=desired_count, DependsOn=service_listener.title, PlacementStrategies=self.PLACEMENT_STRATEGIES ) self.template.add_output( Output( service_name + 'EcsServiceName', Description='The ECS name which needs to be entered', Value=GetAtt(svc, 'Name') ) ) self.template.add_output( Output( service_name + "URL", Description="The URL at which the service is accessible", Value=Sub("https://${" + alb.name + ".DNSName}") ) ) self.template.add_resource(svc) else: svc = Service( service_name, Cluster=self.cluster_name, TaskDefinition=Ref(td), DesiredCount=desired_count, DeploymentConfiguration=deployment_configuration, PlacementStrategies=self.PLACEMENT_STRATEGIES ) self.template.add_output( Output( service_name + 'EcsServiceName', Description='The ECS name which needs to be entered', Value=GetAtt(svc, 'Name') ) ) self.template.add_resource(svc) self._add_service_alarms(svc)
def test_build_config_for_secrets_manager_from_multiple_files( self, mock_getcwd, m_secrets_manager, m_parameter_store): env_name = "staging" service_name = "Dummy" sample_env_file_path = "test-env.sample" essential_container_name = "mainService" ecs_service_name = "dummy-ecs-service" injected_secret_name = get_automated_injected_secret_name( env_name, service_name, ecs_service_name) class MockSecretManager: injected_secret_name_call_count = 0 @classmethod def get_config(cls, secret_name, env): if secret_name == 'dummy-secrets-staging': return { 'secrets': { 'key1': 'actualValue1', 'key2': 'actualValue2', 'key3': 'actualValue3', } } if secret_name == 'dummy-secrets-staging/app1': return { 'secrets': { 'key4': 'actualValue4', 'key5': 'actualValue5', } } if secret_name == injected_secret_name and cls.injected_secret_name_call_count == 0: cls.injected_secret_name_call_count += 1 raise Exception('not found') if secret_name == injected_secret_name: return {'ARN': 'injected-secret-arn'} return {'secrets': {}} m_secrets_manager.get_config.side_effect = MockSecretManager.get_config mock_getcwd.return_value = os.path.join( os.path.dirname(__file__), '../env_sample_files/env_sample_files_without_duplicate_keys', ) result = build_config(env_name, service_name, ecs_service_name, sample_env_file_path, essential_container_name, "dummy-secrets-staging") m_secrets_manager.set_secrets_manager_config.assert_called_with( env_name, injected_secret_name, { 'key1': 'actualValue1', 'key2': 'actualValue2', 'key3': 'actualValue3', 'key4': 'actualValue4', }) self.assertEqual( { essential_container_name: { 'environment': {}, 'secrets': { 'CLOUDLIFT_INJECTED_SECRETS': 'injected-secret-arn' } } }, result)