Exemple #1
0
    def test_successful_build_config_from_without_duplicates(
            self, mock_secrets_manager, mock_parameter_store):
        env_name = "staging"
        cloudlift_service_name = "Dummy"
        sample_env_file_path = "test-env.sample"
        essential_container_name = "mainService"
        secrets_name = "main"
        mock_store = MagicMock()
        mock_parameter_store.return_value = mock_store
        mock_store.get_existing_config.return_value = ({
            'PORT':
            '80',
            "LABEL":
            "arn_for_secret_at_v1"
        }, {})
        mock_secrets_manager.get_config.return_value = {}

        actual_configurations = build_config(env_name, cloudlift_service_name,
                                             "", sample_env_file_path,
                                             essential_container_name, None)

        expected_configurations = {
            "mainService": {
                "secrets": {},
                "environment": {
                    "PORT": "80",
                    "LABEL": "arn_for_secret_at_v1"
                }
            }
        }
        self.assertDictEqual(expected_configurations, actual_configurations)
Exemple #2
0
    def test_successful_build_config_from_only_param_store(
            self, mock_secrets_manager, mock_parameter_store):
        env_name = "staging"
        cloudlift_service_name = "Dummy"
        sample_env_file_path = "test-env.sample"
        essential_container_name = "mainServiceContainer"

        mock_store = MagicMock()
        mock_parameter_store.return_value = mock_store
        mock_store.get_existing_config.return_value = ({
            'PORT': '80',
            'LABEL': 'Dummy'
        }, {})

        actual_configurations = build_config(env_name, cloudlift_service_name,
                                             "", sample_env_file_path,
                                             essential_container_name, None)

        expected_configurations = {
            "mainServiceContainer": {
                "environment": {
                    "LABEL": "Dummy",
                    "PORT": "80"
                },
                "secrets": {}
            }
        }
        self.assertDictEqual(expected_configurations, actual_configurations)
        mock_secrets_manager.get_config.assert_not_called()
    def create(self):
        log_warning(
            "Create task definition to {self.region}".format(**locals()))
        if not os.path.exists(self.env_sample_file):
            raise UnrecoverableException('env.sample not found. Exiting.')
        ecr_client = EcrClient(self.name, self.region, self.build_args)
        ecr_client.set_version(self.version)
        log_intent("name: " + self.name + " | environment: " +
                   self.environment + " | version: " + str(ecr_client.version))
        log_bold("Checking image in ECR")
        ecr_client.build_and_upload_image()
        log_bold("Creating task definition\n")
        env_config = build_config(self.environment, self.name,
                                  self.env_sample_file)
        container_definition_arguments = {
            "environment": [{
                "name": k,
                "value": v
            } for (k, v) in env_config],
            "name": pascalcase(self.name) + "Container",
            "image": _complete_image_url(ecr_client),
            "essential": True,
            "logConfiguration": self._gen_log_config(pascalcase(self.name)),
            "memoryReservation": 1024
        }

        ecs_client = EcsClient(region=self.region)
        ecs_client.register_task_definition(self._task_defn_family(),
                                            [container_definition_arguments],
                                            [], None)
        log_bold("Task definition successfully created\n")
Exemple #4
0
    def test_build_config_for_secrets_manager_from_multiple_files_already_present(
            self, mock_getcwd, m_secrets_manager):
        env_name = "staging"
        service_name = "Dummy"
        sample_env_file_path = "test-env.sample"
        essential_container_name = "mainService"
        ecs_service_name = "dummy-ecs-service"
        secrets = {
            'key1': 'actualValue1',
            'key2': 'actualValue2',
            'key3': 'actualValue3',
            'key4': 'actualValue4',
        }

        def get_config(secret_name, env):
            if secret_name == 'dummy-secrets-staging':
                return {
                    'secrets': {
                        'key1': 'actualValue1',
                        'key2': 'actualValue2',
                        'key3': 'actualValue3',
                    }
                }
            if secret_name == 'dummy-secrets-staging/app1':
                return {
                    'secrets': {
                        'key4': 'actualValue4',
                        'key5': 'actualValue5',
                    }
                }
            if secret_name == get_automated_injected_secret_name(
                    env_name, service_name, ecs_service_name):
                return {'ARN': 'injected-secret-arn', 'secrets': secrets}
            return {'secrets': {}}

        m_secrets_manager.get_config.side_effect = get_config
        mock_getcwd.return_value = os.path.join(
            os.path.dirname(__file__),
            '../env_sample_files/env_sample_files_without_duplicate_keys',
        )

        result = build_config(env_name, service_name, ecs_service_name,
                              sample_env_file_path, essential_container_name,
                              "dummy-secrets-staging")

        m_secrets_manager.set_secrets_manager_config.assert_not_called()
        self.assertEqual(
            {
                essential_container_name: {
                    'environment': {},
                    'secrets': {
                        'CLOUDLIFT_INJECTED_SECRETS': 'injected-secret-arn'
                    }
                }
            }, result)
Exemple #5
0
    def test_failure_build_config_for_if_sample_config_has_additional_keys(
            self, m_secrets_manager, m_parameter_store):
        env_name = "staging"
        service_name = "Dummy"
        sample_env_file_path = "test-env.sample"
        essential_container_name = "mainService"
        mock_store = MagicMock()
        m_parameter_store.return_value = mock_store
        mock_store.get_existing_config.return_value = ({
            'PORT':
            '80',
            "LABEL":
            "arn_for_secret_at_v1"
        }, {})
        m_secrets_manager.get_config.return_value = {}

        with pytest.raises(UnrecoverableException) as pytest_wrapped_e:
            build_config(env_name, service_name, "", sample_env_file_path,
                         essential_container_name, None)

        self.assertEqual(pytest_wrapped_e.type, UnrecoverableException)
        self.assertEqual(
            str(pytest_wrapped_e.value),
            '"There is no config value for the keys {\'ADDITIONAL_CONFIG\'}"')
 def update(self):
     log_warning(
         "Update task definition to {self.region}".format(**locals()))
     if not os.path.exists(self.env_sample_file):
         raise UnrecoverableException('env.sample not found. Exiting.')
     ecr_client = EcrClient(self.name, self.region, self.build_args)
     ecr_client.set_version(self.version)
     log_intent("name: " + self.name + " | environment: " +
                self.environment + " | version: " + str(ecr_client.version))
     log_bold("Checking image in ECR")
     ecr_client.build_and_upload_image()
     log_bold("Updating task definition\n")
     env_config = build_config(self.environment, self.name,
                               self.env_sample_file)
     ecs_client = EcsClient(region=self.region)
     deployment = DeployAction(ecs_client, self.cluster_name, None)
     task_defn = self._apply_changes_over_current_task_defn(
         env_config, ecs_client, ecr_client, deployment)
     deployment.update_task_definition(task_defn)
     log_bold("Task definition successfully updated\n")
Exemple #7
0
    def test_build_config_ignores_additional_keys_in_parameter_store(
            self, m_secrets_mgr, m_parameter_store):
        env_name = "staging"
        service_name = "Dummy"
        sample_env_file_path = "test-env.sample"
        essential_container_name = "mainService"
        secrets_name = "main"
        mock_store = MagicMock()
        m_parameter_store.return_value = mock_store
        mock_store.get_existing_config.return_value = ({
            "LABEL":
            "dummyvalue",
            'PORT':
            '80',
            'ADDITIONAL_KEY_1':
            'true'
        }, {})
        m_secrets_mgr.get_config.return_value = {
            'secrets': {},
            'ARN': "dummy_arn"
        }

        actual_configurations = build_config(env_name, service_name, "",
                                             sample_env_file_path,
                                             essential_container_name, None)

        expected_configurations = {
            "mainService": {
                "secrets": {},
                "environment": {
                    "PORT": "80",
                    "LABEL": "dummyvalue"
                }
            }
        }
        self.assertDictEqual(expected_configurations, actual_configurations)
    def _add_service(self, service_name, config):
        launch_type = self.LAUNCH_TYPE_FARGATE if 'fargate' in config else self.LAUNCH_TYPE_EC2
        env_config = build_config(
            self.env,
            self.application_name,
            self.env_sample_file_path
        )
        container_definition_arguments = {
            "Environment": [
                Environment(Name=k, Value=v) for (k, v) in env_config
            ],
            "Name": service_name + "Container",
            "Image": self.ecr_image_uri + ':' + self.current_version,
            "Essential": 'true',
            "LogConfiguration": self._gen_log_config(service_name),
            "MemoryReservation": int(config['memory_reservation']),
            "Cpu": 0
        }

        if 'http_interface' in config:
            container_definition_arguments['PortMappings'] = [
                PortMapping(
                    ContainerPort=int(
                        config['http_interface']['container_port']
                    )
                )
            ]

        if config['command'] is not None:
            container_definition_arguments['Command'] = [config['command']]

        cd = ContainerDefinition(**container_definition_arguments)

        task_role = self.template.add_resource(Role(
            service_name + "Role",
            AssumeRolePolicyDocument=PolicyDocument(
                Statement=[
                    Statement(
                        Effect=Allow,
                        Action=[AssumeRole],
                        Principal=Principal("Service", ["ecs-tasks.amazonaws.com"])
                    )
                ]
            )
        ))

        launch_type_td = {}
        if launch_type == self.LAUNCH_TYPE_FARGATE:
            launch_type_td = {
                'RequiresCompatibilities': ['FARGATE'],
                'ExecutionRoleArn': boto3.resource('iam').Role('ecsTaskExecutionRole').arn,
                'NetworkMode': 'awsvpc',
                'Cpu': str(config['fargate']['cpu']),
                'Memory': str(config['fargate']['memory'])
            }

        td = TaskDefinition(
            service_name + "TaskDefinition",
            Family=service_name + "Family",
            ContainerDefinitions=[cd],
            TaskRoleArn=Ref(task_role),
            **launch_type_td
        )

        self.template.add_resource(td)
        desired_count = self._get_desired_task_count_for_service(service_name)
        deployment_configuration = DeploymentConfiguration(
            MinimumHealthyPercent=100,
            MaximumPercent=200
        )
        if 'http_interface' in config:
            alb, lb, service_listener, alb_sg = self._add_alb(cd, service_name, config, launch_type)

            if launch_type == self.LAUNCH_TYPE_FARGATE:
                # if launch type is ec2, then services inherit the ec2 instance security group
                # otherwise, we need to specify a security group for the service
                service_security_group = SecurityGroup(
                    pascalcase("FargateService" + self.env + service_name),
                    GroupName=pascalcase("FargateService" + self.env + service_name),
                    SecurityGroupIngress=[{
                        'IpProtocol': 'TCP',
                        'SourceSecurityGroupId': Ref(alb_sg),
                        'ToPort': int(config['http_interface']['container_port']),
                        'FromPort': int(config['http_interface']['container_port']),
                    }],
                    VpcId=Ref(self.vpc),
                    GroupDescription=pascalcase("FargateService" + self.env + service_name)
                )
                self.template.add_resource(service_security_group)

                launch_type_svc = {
                    'NetworkConfiguration': NetworkConfiguration(
                        AwsvpcConfiguration=AwsvpcConfiguration(
                            Subnets=[
                                Ref(self.private_subnet1),
                                Ref(self.private_subnet2)
                            ],
                            SecurityGroups=[
                                Ref(service_security_group)
                            ]
                        )
                    )
                }
            else:
                launch_type_svc = {
                    'Role': Ref(self.ecs_service_role),
                    'PlacementStrategies': self.PLACEMENT_STRATEGIES
                }
            svc = Service(
                service_name,
                LoadBalancers=[lb],
                Cluster=self.cluster_name,
                TaskDefinition=Ref(td),
                DesiredCount=desired_count,
                DependsOn=service_listener.title,
                LaunchType=launch_type,
                **launch_type_svc,
            )
            self.template.add_output(
                Output(
                    service_name + 'EcsServiceName',
                    Description='The ECS name which needs to be entered',
                    Value=GetAtt(svc, 'Name')
                )
            )
            self.template.add_output(
                Output(
                    service_name + "URL",
                    Description="The URL at which the service is accessible",
                    Value=Sub("https://${" + alb.name + ".DNSName}")
                )
            )
            self.template.add_resource(svc)
        else:
            launch_type_svc = {}
            if launch_type == self.LAUNCH_TYPE_FARGATE:
                # if launch type is ec2, then services inherit the ec2 instance security group
                # otherwise, we need to specify a security group for the service
                service_security_group = SecurityGroup(
                    pascalcase("FargateService" + self.env + service_name),
                    GroupName=pascalcase("FargateService" + self.env + service_name),
                    SecurityGroupIngress=[],
                    VpcId=Ref(self.vpc),
                    GroupDescription=pascalcase("FargateService" + self.env + service_name)
                )
                self.template.add_resource(service_security_group)
                launch_type_svc = {
                    'NetworkConfiguration': NetworkConfiguration(
                        AwsvpcConfiguration=AwsvpcConfiguration(
                            Subnets=[
                                Ref(self.private_subnet1),
                                Ref(self.private_subnet2)
                            ],
                            SecurityGroups=[
                                Ref(service_security_group)
                            ]
                        )
                    )
                }
            else:
                launch_type_svc = {
                    'PlacementStrategies': self.PLACEMENT_STRATEGIES
                }
            svc = Service(
                service_name,
                Cluster=self.cluster_name,
                TaskDefinition=Ref(td),
                DesiredCount=desired_count,
                DeploymentConfiguration=deployment_configuration,
                LaunchType=launch_type,
                **launch_type_svc
            )
            self.template.add_output(
                Output(
                    service_name + 'EcsServiceName',
                    Description='The ECS name which needs to be entered',
                    Value=GetAtt(svc, 'Name')
                )
            )
            self.template.add_resource(svc)
        self._add_service_alarms(svc)
Exemple #9
0
    def _add_service(self, service_name, config):
        env_config = build_config(
            self.env,
            self.application_name,
            self.env_sample_file_path
        )
        container_definition_arguments = {
            "Environment": [
                Environment(Name=k, Value=v) for (k, v) in env_config
            ],
            "Name": service_name + "Container",
            "Image": self.ecr_image_uri + ':' + self.current_version,
            "Essential": 'true',
            "LogConfiguration": self._gen_log_config(service_name),
            "MemoryReservation": int(config['memory_reservation']),
            "Cpu": 0
        }

        if 'http_interface' in config:
            container_definition_arguments['PortMappings'] = [
                PortMapping(
                    ContainerPort=int(
                        config['http_interface']['container_port']
                    )
                )
            ]

        if config['command'] is not None:
            container_definition_arguments['Command'] = [config['command']]

        cd = ContainerDefinition(**container_definition_arguments)

        task_role = self.template.add_resource(Role(
            service_name + "Role",
            AssumeRolePolicyDocument=PolicyDocument(
                Statement=[
                    Statement(
                        Effect=Allow,
                        Action=[AssumeRole],
                        Principal=Principal("Service", ["ecs-tasks.amazonaws.com"])
                    )
                ]
            )
        ))

        td = TaskDefinition(
            service_name + "TaskDefinition",
            Family=service_name + "Family",
            ContainerDefinitions=[cd],
            TaskRoleArn=Ref(task_role)
        )
        self.template.add_resource(td)
        desired_count = self._get_desired_task_count_for_service(service_name)
        deployment_configuration = DeploymentConfiguration(
            MinimumHealthyPercent=100,
            MaximumPercent=200
        )
        if 'http_interface' in config:
            alb, lb, service_listener = self._add_alb(cd, service_name, config)
            svc = Service(
                service_name,
                LoadBalancers=[lb],
                Cluster=self.cluster_name,
                Role=Ref(self.ecs_service_role),
                TaskDefinition=Ref(td),
                DesiredCount=desired_count,
                DependsOn=service_listener.title,
                PlacementStrategies=self.PLACEMENT_STRATEGIES
            )
            self.template.add_output(
                Output(
                    service_name + 'EcsServiceName',
                    Description='The ECS name which needs to be entered',
                    Value=GetAtt(svc, 'Name')
                )
            )
            self.template.add_output(
                Output(
                    service_name + "URL",
                    Description="The URL at which the service is accessible",
                    Value=Sub("https://${" + alb.name + ".DNSName}")
                )
            )
            self.template.add_resource(svc)
        else:
            svc = Service(
                service_name,
                Cluster=self.cluster_name,
                TaskDefinition=Ref(td),
                DesiredCount=desired_count,
                DeploymentConfiguration=deployment_configuration,
                PlacementStrategies=self.PLACEMENT_STRATEGIES
            )
            self.template.add_output(
                Output(
                    service_name + 'EcsServiceName',
                    Description='The ECS name which needs to be entered',
                    Value=GetAtt(svc, 'Name')
                )
            )
            self.template.add_resource(svc)
        self._add_service_alarms(svc)
Exemple #10
0
    def test_build_config_for_secrets_manager_from_multiple_files(
            self, mock_getcwd, m_secrets_manager, m_parameter_store):
        env_name = "staging"
        service_name = "Dummy"
        sample_env_file_path = "test-env.sample"
        essential_container_name = "mainService"
        ecs_service_name = "dummy-ecs-service"
        injected_secret_name = get_automated_injected_secret_name(
            env_name, service_name, ecs_service_name)

        class MockSecretManager:
            injected_secret_name_call_count = 0

            @classmethod
            def get_config(cls, secret_name, env):
                if secret_name == 'dummy-secrets-staging':
                    return {
                        'secrets': {
                            'key1': 'actualValue1',
                            'key2': 'actualValue2',
                            'key3': 'actualValue3',
                        }
                    }
                if secret_name == 'dummy-secrets-staging/app1':
                    return {
                        'secrets': {
                            'key4': 'actualValue4',
                            'key5': 'actualValue5',
                        }
                    }
                if secret_name == injected_secret_name and cls.injected_secret_name_call_count == 0:
                    cls.injected_secret_name_call_count += 1
                    raise Exception('not found')
                if secret_name == injected_secret_name:
                    return {'ARN': 'injected-secret-arn'}
                return {'secrets': {}}

        m_secrets_manager.get_config.side_effect = MockSecretManager.get_config
        mock_getcwd.return_value = os.path.join(
            os.path.dirname(__file__),
            '../env_sample_files/env_sample_files_without_duplicate_keys',
        )

        result = build_config(env_name, service_name, ecs_service_name,
                              sample_env_file_path, essential_container_name,
                              "dummy-secrets-staging")

        m_secrets_manager.set_secrets_manager_config.assert_called_with(
            env_name, injected_secret_name, {
                'key1': 'actualValue1',
                'key2': 'actualValue2',
                'key3': 'actualValue3',
                'key4': 'actualValue4',
            })
        self.assertEqual(
            {
                essential_container_name: {
                    'environment': {},
                    'secrets': {
                        'CLOUDLIFT_INJECTED_SECRETS': 'injected-secret-arn'
                    }
                }
            }, result)