def test_generate_service(self, mock_region_service, mock_get_account_id, mock_build_config): environment = 'staging' application_name = 'dummy' mock_service_configuration = MagicMock(spec=ServiceConfiguration, service_name=application_name, environment=environment) mock_service_configuration.get_config.return_value = mocked_service_config() def mock_build_config_impl(env_name, service_name, dummy_ecs_service_name, sample_env_file_path, ecs_service_name, secrets_name): expected = "dummy-config" if ecs_service_name == "DummyContainer" else "dummy-sidekiq-config" self.assertEqual(secrets_name, expected) return {ecs_service_name: {"secrets": {"LABEL": 'arn_secret_label_v1'}, "environment": {"PORT": "80"}}} mock_build_config.side_effect = mock_build_config_impl mock_get_account_id.return_value = "12537612" mock_region_service.get_region_for_environment.return_value = "us-west-2" template_generator = ServiceTemplateGenerator(mock_service_configuration, self._get_env_stack(), './test/templates/test_env.sample', "12537612.dkr.ecr.us-west-2.amazonaws.com/test-service-repo:1.1.1", desired_counts={"Dummy": 100, "DummyRunSidekiqsh": 199}) generated_template = template_generator.generate_service() template_file_path = os.path.join(os.path.dirname(__file__), '../templates/expected_service_template.yml') with(open(template_file_path)) as expected_template_file: self.assert_template(to_json(''.join(expected_template_file.readlines())), to_json(generated_template))
def update(self): ''' Create and execute changeset for existing CloudFormation template for ECS service and related dependencies ''' log_bold("Starting to update service") self.service_configuration.edit_config() try: template_generator = ServiceTemplateGenerator( self.service_configuration, self.environment_stack) service_template_body = template_generator.generate_service() change_set = create_change_set(self.client, service_template_body, self.stack_name, "", self.environment) self.service_configuration.update_cloudlift_version() log_bold("Executing changeset. Checking progress...") self.client.execute_change_set( ChangeSetName=change_set['ChangeSetId']) self._print_progress() except ClientError as exc: if "No updates are to be performed." in str(exc): log_err("No updates are to be performed") else: raise exc
def create(self): ''' Create and execute CloudFormation template for ECS service and related dependencies ''' log_bold("Initiating service creation") self.service_configuration.edit_config() template_generator = ServiceTemplateGenerator( self.service_configuration, self.environment_stack) service_template_body = template_generator.generate_service() try: self.client.create_stack( StackName=self.stack_name, TemplateBody=service_template_body, Parameters=[{ 'ParameterKey': 'Environment', 'ParameterValue': self.environment, }], OnFailure='DO_NOTHING', Capabilities=['CAPABILITY_NAMED_IAM'], ) log_bold("Submitted to cloudformation. Checking progress...") self._print_progress() except ClientError as boto_client_error: error_code = boto_client_error.response['Error']['Code'] if error_code == 'AlreadyExistsException': log_err("Stack " + self.stack_name + " already exists.") exit(1) else: raise boto_client_error
def create(self, config_body=None, version=None, build_arg=None, dockerfile=None, ssh=None, cache_from=None): ''' Create and execute CloudFormation template for ECS service and related dependencies ''' log_bold("Initiating service creation") if config_body is None: self.service_configuration.edit_config() else: self.service_configuration.set_config(config_body) self.service_configuration.validate() ecr_repo_config = self.service_configuration.get_config().get( 'ecr_repo') ecr = ECR( region=get_region_for_environment(self.environment), repo_name=ecr_repo_config.get('name'), account_id=ecr_repo_config.get('account_id', None), assume_role_arn=ecr_repo_config.get('assume_role_arn', None), version=version, build_args=build_arg, dockerfile=dockerfile, ssh=ssh, cache_from=cache_from, ) ecr.upload_artefacts() template_generator = ServiceTemplateGenerator( self.service_configuration, self.environment_stack, self.env_sample_file, ecr.image_uri) service_template_body = template_generator.generate_service() try: options = prepare_stack_options_for_template( service_template_body, self.environment, self.stack_name) self.client.create_stack( StackName=self.stack_name, Parameters=[{ 'ParameterKey': 'Environment', 'ParameterValue': self.environment, }], OnFailure='DO_NOTHING', Capabilities=['CAPABILITY_NAMED_IAM'], **options, ) log_bold("Submitted to cloudformation. Checking progress...") self._print_progress() except ClientError as boto_client_error: error_code = boto_client_error.response['Error']['Code'] if error_code == 'AlreadyExistsException': raise UnrecoverableException("Stack " + self.stack_name + " already exists.") else: raise boto_client_error
def test_generate_service_with_new_alb(self, mock_region_service, mock_get_account_id, mock_build_config): environment = 'staging' application_name = 'dummy' mock_service_configuration = MagicMock(spec=ServiceConfiguration, service_name=application_name, environment=environment) mock_service_configuration.get_config.return_value = { "cloudlift_version": 'test-version', "notifications_arn": "some", "ecr_repo": {"name": "test-service-repo"}, "services": { "Dummy": { "memory_reservation": Decimal(1000), "secrets_name": "something", "command": None, "http_interface": { "internal": False, "alb": { "create_new": True, "target_5xx_error_threshold": 10 }, "container_port": Decimal(7003), "restrict_access_to": ["0.0.0.0/0"], "health_check_path": "/elb-check" }, "autoscaling": { "max_capacity": 10, "min_capacity": 5, "request_count_per_target": { "target_value": 10, "scale_in_cool_down_seconds": 120, "scale_out_cool_down_seconds": 60 } } }, } } mock_build_config.side_effect = mock_build_config_impl mock_get_account_id.return_value = "12537612" mock_region_service.get_region_for_environment.return_value = "us-west-2" mock_region_service.get_ssl_certification_for_environment.return_value = "certificateARN1234" template_generator = ServiceTemplateGenerator(mock_service_configuration, self._get_env_stack(), './test/templates/test_env.sample', "12537612.dkr.ecr.us-west-2.amazonaws.com/test-service-repo:1.1.1", desired_counts={"Dummy": 100, "DummyRunSidekiqsh": 199}) generated_template = template_generator.generate_service() template_file_path = os.path.join(os.path.dirname(__file__), '../templates/expected_service_with_new_alb_template.yml') with(open(template_file_path)) as expected_template_file: expected = load(''.join(expected_template_file.readlines())) generated = load(generated_template) diff = list(dictdiffer.diff(generated, expected)) assert diff == []
def test_initialization(self): mock_service_configuration = MagicMock(spec=ServiceConfiguration, service_name="test-service", environment="staging") generator = ServiceTemplateGenerator(mock_service_configuration, None, "env.sample", ecr_image_uri="image:v1") assert generator.env == 'staging' assert generator.application_name == 'test-service' assert generator.environment_stack is None
def test_generate_service_for_ecs_with_custom_roles(self, mock_region_service, mock_get_account_id, mock_build_config): environment = 'staging' application_name = 'dummy' mock_service_configuration = MagicMock(spec=ServiceConfiguration, service_name=application_name, environment=environment) mock_service_configuration.get_config.return_value = { "cloudlift_version": 'test-version', "notifications_arn": "some", "ecr_repo": {"name": "main-repo", "assume_role_arn": "arn1234", "account_id": "1234"}, "services": { "Dummy": { "memory_reservation": Decimal(1000), "secrets_name": "something", "command": None, "task_role_arn": "TASK_ARN", "task_execution_role_arn": "TASK_EXECUTION_ARN" }, } } mock_build_config.side_effect = mock_build_config_impl mock_get_account_id.return_value = "12537612" mock_region_service.get_region_for_environment.return_value = "us-west-2" mock_region_service.get_ssl_certification_for_environment.return_value = "certificateARN1234" template_generator = ServiceTemplateGenerator(mock_service_configuration, self._get_env_stack(), './test/templates/test_env.sample', "12537612.dkr.ecr.us-west-2.amazonaws.com/test-service-repo:1.1.1", desired_counts={"Dummy": 1}) generated_template = template_generator.generate_service() loaded_template = load(to_json(generated_template)) self.assertGreaterEqual(len(loaded_template), 1, "no template generated") generated = loaded_template[0] assert "DummyRole" in generated['Resources'] assert "DummyTaskExecutionRole" in generated['Resources'] assert "ECSServiceRole" in generated['Resources'] td = generated['Resources']['DummyTaskDefinition'] assert td['Properties']['TaskRoleArn'] == 'TASK_ARN' assert td['Properties']['ExecutionRoleArn'] == 'TASK_EXECUTION_ARN'
def update(self): ''' Create and execute changeset for existing CloudFormation template for ECS service and related dependencies ''' log_bold("Starting to update service") self.service_configuration.edit_config() self.service_configuration.validate() information_fetcher = ServiceInformationFetcher( self.service_configuration.service_name, self.environment, self.service_configuration.get_config(), ) try: current_image_uri = information_fetcher.get_current_image_uri() desired_counts = information_fetcher.fetch_current_desired_count() template_generator = ServiceTemplateGenerator( self.service_configuration, self.environment_stack, self.env_sample_file, current_image_uri, desired_counts, ) service_template_body = template_generator.generate_service() change_set = create_change_set(self.client, service_template_body, self.stack_name, None, self.environment) if change_set is None: return self.service_configuration.update_cloudlift_version() log_bold("Executing changeset. Checking progress...") self.client.execute_change_set( ChangeSetName=change_set['ChangeSetId']) self._print_progress() except ClientError as exc: if "No updates are to be performed." in str(exc): log_err("No updates are to be performed") else: raise exc
def test_generate_tcp_service(self, mock_region_service, mock_get_account_id, mock_build_config): environment = 'staging' application_name = 'dummy' mock_service_configuration = MagicMock(spec=ServiceConfiguration, service_name=application_name, environment=environment) mock_service_configuration.get_config.return_value = mocked_tcp_service_config() mock_build_config.side_effect = mock_build_config_impl mock_get_account_id.return_value = "12537612" mock_region_service.get_region_for_environment.return_value = "us-west-2" template_generator = ServiceTemplateGenerator(mock_service_configuration, self._get_env_stack(), './test/templates/test_env.sample', "12537612.dkr.ecr.us-west-2.amazonaws.com/test-service-repo:1.1.1", desired_counts={"FreeradiusServer": 100}) generated_template = template_generator.generate_service() template_file_path = os.path.join(os.path.dirname(__file__), '../templates/expected_tcp_service_template.yml') with(open(template_file_path)) as expected_template_file: self.assert_template(to_json(''.join(expected_template_file.readlines())), to_json(generated_template))
def test_generate_fargate_service_should_fail_for_udp_interface(self, mock_region_service, mock_get_account_id, mock_build_config): environment = 'staging' application_name = 'dummyFargate' mock_service_configuration = MagicMock(spec=ServiceConfiguration, service_name=application_name, environment=environment) mock_service_configuration.get_config.return_value = mocked_udp_fargate_service_config() mock_build_config.side_effect = mock_build_config_impl mock_get_account_id.return_value = "12537612" mock_region_service.get_region_for_environment.return_value = "us-west-2" template_generator = ServiceTemplateGenerator(mock_service_configuration, self._get_env_stack(), './test/templates/test_env.sample', "test-image-repo") with self.assertRaises(NotImplementedError) as context: template_generator.generate_service() self.assertTrue( 'udp interface not yet implemented in fargate type, please use ec2 type' in context.exception)
def test_initialization(self): with patch.object(ServiceConfiguration, 'get_config', new=mocked_service_config): with patch.object(ServiceInformationFetcher, 'get_current_version', new=mocked_service_information): service_config = ServiceConfiguration("test-service", "staging") generator = ServiceTemplateGenerator(service_config, None) assert generator.env == 'staging' assert generator.application_name == 'test-service' assert generator.environment_stack == None
def test_generate_service_for_ecr(self, mock_region_service, mock_get_account_id, mock_build_config): environment = 'staging' application_name = 'dummy' mock_service_configuration = MagicMock(spec=ServiceConfiguration, service_name=application_name, environment=environment) mock_service_configuration.get_config.return_value = { "cloudlift_version": 'test-version', "notifications_arn": "some", "ecr_repo": {"name": "main-repo", "assume_role_arn": "arn1234", "account_id": "1234"}, "services": { "Dummy": { "memory_reservation": Decimal(1000), "secrets_name": "something", "command": None, }, } } mock_build_config.side_effect = mock_build_config_impl mock_get_account_id.return_value = "12537612" mock_region_service.get_region_for_environment.return_value = "us-west-2" mock_region_service.get_ssl_certification_for_environment.return_value = "certificateARN1234" template_generator = ServiceTemplateGenerator(mock_service_configuration, self._get_env_stack(), './test/templates/test_env.sample', "12537612.dkr.ecr.us-west-2.amazonaws.com/test-service-repo:1.1.1", desired_counts={"Dummy": 1}) generated_template = template_generator.generate_service() loaded_template = load(to_json(generated_template)) self.assertGreaterEqual(len(loaded_template), 1, "no template generated") generated = loaded_template[0] self.check_in_outputs(generated, 'ECRRepoName', 'main-repo') self.check_in_outputs(generated, 'ECRAccountID', '1234') self.check_in_outputs(generated, 'ECRAssumeRoleARN', 'arn1234')
def test_generate_service_with_env_alb_host_based(self, mock_region_service, mock_get_account_id, mock_build_config, mock_get_environment_level_alb_listener, mock_get_client_for): environment = 'staging' application_name = 'dummy' mock_service_configuration = MagicMock(spec=ServiceConfiguration, service_name=application_name, environment=environment) mock_service_configuration.get_config.return_value = { "cloudlift_version": 'test-version', "notifications_arn": "some", "ecr_repo": {"name": "test-service-repo"}, "services": { "Dummy": { "memory_reservation": Decimal(1000), "command": None, "container_labels": {"python_version": "2"}, "secrets_name": "something", "http_interface": { "internal": False, "alb": { "create_new": False, "host": "abc.xyz.com", "priority": 4, "target_5xx_error_threshold": 10, "target_p95_latency_evaluation_periods": 3, "target_p95_latency_period_seconds": 30, "target_p95_latency_threshold_seconds": 10, "target_p99_latency_evaluation_periods": 3, "target_p99_latency_period_seconds": 30, "target_p99_latency_threshold_seconds": 10, }, "container_port": Decimal(7003), "restrict_access_to": ["0.0.0.0/0"], "health_check_path": "/elb-check" }, "autoscaling": { "max_capacity": 10, "min_capacity": 5, "request_count_per_target": { "target_value": 10, "scale_in_cool_down_seconds": 120, "scale_out_cool_down_seconds": 60, "alb_arn": "arn:aws:elasticloadbalancing:us-west-2:123456123456:loadbalancer/app/alb-name/alb-id" } } }, "DummyWithCustomListener": { "memory_reservation": Decimal(1000), "command": None, "secrets_name": "something", "http_interface": { "internal": False, "alb": { "create_new": False, "target_5xx_error_threshold": 10, "listener_arn": "arn:aws:elasticloadbalancing:us-west-2:434332696:listener/app/albname/randomalbid/randomlistenerid", "path": "/api/*", "priority": 100, }, "container_port": Decimal(7003), "restrict_access_to": ["0.0.0.0/0"], "health_check_path": "/elb-check" } }, } } mock_build_config.side_effect = mock_build_config_impl mock_get_account_id.return_value = "12537612" mock_region_service.get_region_for_environment.return_value = "us-west-2" mock_get_environment_level_alb_listener.return_value = "listenerARN1234" mock_elbv2_client = MagicMock() mock_get_client_for.return_value = mock_elbv2_client def mock_describe_rules(ListenerArn=None, Marker=None): if ListenerArn: return { 'Rules': [{'Priority': '1'}, {'Priority': '2'}], 'NextMarker': '/next/marker' } if Marker: return { 'Rules': [{'Priority': '3'}, {'Priority': '5'}] } mock_elbv2_client.describe_rules.side_effect = mock_describe_rules template_generator = ServiceTemplateGenerator(mock_service_configuration, self._get_env_stack(), './test/templates/test_env.sample', "12537612.dkr.ecr.us-west-2.amazonaws.com/test-service-repo:1.1.1", desired_counts={"Dummy": 100, "DummyRunSidekiqsh": 199}) generated_template = template_generator.generate_service() template_file_path = os.path.join(os.path.dirname(__file__), '../templates/expected_service_with_env_alb_template.yml') with(open(template_file_path)) as expected_template_file: self.assert_template(to_json(''.join(expected_template_file.readlines())), to_json(generated_template))
def test_generate_service(self): environment = 'staging' application_name = 'dummy' env_stack = { u'StackId': 'arn:aws:cloudformation:ap-south-1:725827686899:stack/cluster-staging/65410f80-d21c-11e8-913a-503a56826a2a', u'LastUpdatedTime': datetime.datetime(2018, 11, 9, 5, 22, 30, 691000), u'Parameters': [{ u'ParameterValue': 'staging-cluster-v3', u'ParameterKey': 'KeyPair' }, { u'ParameterValue': '1', u'ParameterKey': 'ClusterSize' }, { u'ParameterValue': 'arn:aws:sns:ap-south-1:725827686899:non-prod-mumbai', u'ParameterKey': 'NotificationSnsArn' }, { u'ParameterValue': 'staging', u'ParameterKey': 'Environment' }, { u'ParameterValue': 'm5.xlarge', u'ParameterKey': 'InstanceType' }, { u'ParameterValue': '50', u'ParameterKey': 'MaxSize' }], u'Tags': [], u'Outputs': [{ u'Description': 'VPC in which environment is setup', u'OutputKey': 'VPC', u'OutputValue': 'vpc-00f07c5a6b6c9abdb' }, { u'Description': 'Options used with cloudlift when building this cluster', u'OutputKey': 'CloudliftOptions', u'OutputValue': '{"min_instances": 1, "key_name": "staging-cluster-v3", "max_instances": 50, "instance_type": "m5.xlarge", "cloudlift_version": "0.9.4", "env": "staging"}' }, { u'Description': 'Maximum instances in cluster', u'OutputKey': 'MaxInstances', u'OutputValue': '50' }, { u'Description': 'Security group ID for ALB', u'OutputKey': 'SecurityGroupAlb', u'OutputValue': 'sg-095dbeb511019cfd8' }, { u'Description': 'Key Pair name for accessing the instances', u'OutputKey': 'KeyName', u'OutputValue': 'staging-cluster-v3' }, { u'Description': 'ID of the 1st subnet', u'OutputKey': 'PrivateSubnet1', u'OutputValue': 'subnet-09b6cd23af94861cc' }, { u'Description': 'ID of the 2nd subnet', u'OutputKey': 'PrivateSubnet2', u'OutputValue': 'subnet-0657bc2faa99ce5f7' }, { u'Description': 'Minimum instances in cluster', u'OutputKey': 'MinInstances', u'OutputValue': '1' }, { u'Description': 'ID of the 2nd subnet', u'OutputKey': 'PublicSubnet2', u'OutputValue': 'subnet-096377a44ccb73aca' }, { u'Description': 'EC2 instance type', u'OutputKey': 'InstanceType', u'OutputValue': 'm5.xlarge' }, { u'Description': 'ID of the 1st subnet', u'OutputKey': 'PublicSubnet1', u'OutputValue': 'subnet-0aeae8fe5e13a7ff7' }, { u'Description': 'The name of the stack', u'OutputKey': 'StackName', u'OutputValue': 'cluster-staging' }, { u'Description': 'The unique ID of the stack. To be supplied to circle CI environment variables to validate during deployment.', u'OutputKey': 'StackId', u'OutputValue': 'arn:aws:cloudformation:ap-south-1:725827686899:stack/cluster-staging/65410f80-d21c-11e8-913a-503a56826a2a' }], u'CreationTime': datetime.datetime(2018, 10, 17, 14, 53, 23, 469000), u'Capabilities': ['CAPABILITY_NAMED_IAM'], u'StackName': 'cluster-staging', u'NotificationARNs': [], u'StackStatus': 'UPDATE_COMPLETE', u'DisableRollback': True, u'ChangeSetId': 'arn:aws:cloudformation:ap-south-1:725827686899:changeSet/cg901a2f5dbf984b9e9807a21da1ac7d12/7588cd05-1e2d-4dd6-85ab-12b921baa814', u'RollbackConfiguration': {} } with patch.object(ServiceConfiguration, 'get_config', new=mocked_service_config): with patch.object(ParameterStore, 'get_existing_config', new=mocked_environment_config): with patch.object(ServiceInformationFetcher, 'get_current_version', new=mocked_service_information): service_config = ServiceConfiguration( application_name, environment) template_generator = ServiceTemplateGenerator( service_config, env_stack) template_generator.env_sample_file_path = './test/templates/test_env.sample' generated_template = template_generator.generate_service() assert to_json(''.join( open('./test/templates/expected_service_template.yml').readlines()) ) == to_json(generated_template)