class TestContainerDefinition_load_yaml(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(os.path.abspath(__file__)) state_file = os.path.join(current_dir, 'terraform.tfstate') config_yml = os.path.join(current_dir, 'interpolate.yml') with open(state_file) as f: tfstate = json.loads(f.read()) os.environ['FOOBAR_ENV'] = "hi_mom" with Replacer() as r: get_mock = r('deployfish.terraform.Terraform._get_state_file_from_s3', Mock()) get_mock.return_value = tfstate self.config = Config(filename=config_yml) def tearDown(self): del os.environ['FOOBAR_ENV'] def test_terraform_simple_interpolation(self): self.assertEqual(self.config.get_service('cit-auth-prod')['cluster'], 'foobar-proxy-prod') def test_terraform_nested_dict_interpolation(self): self.assertEqual(self.config.get_service('cit-auth-prod')['load_balancer']['load_balancer_name'], 'foobar-proxy-prod') def test_terraform_nested_list_interpolation(self): self.assertEqual(self.config.get_service('cit-auth-prod')['containers'][0]['environment'][2], 'SECRETS_BUCKET_NAME=ac-config-store') def test_environment_simple_interpolation(self): self.assertEqual(self.config.get_service('cit-auth-prod')['config'][0], 'FOOBAR=hi_mom')
class TestContainerDefinition_load_yaml_no_interpolate(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(os.path.abspath(__file__)) state_file = os.path.join(current_dir, 'terraform.tfstate') config_yml = os.path.join(current_dir, 'interpolate.yml') env_file = os.path.join(current_dir, 'env_file.env') with open(state_file) as f: tfstate = json.loads(f.read()) with Replacer() as r: get_mock = r('deployfish.terraform.Terraform._get_state_file_from_s3', Mock()) get_mock.return_value = tfstate self.config = Config(filename=config_yml, env_file=env_file, interpolate=False) def test_simple_interpolation(self): self.assertEqual(self.config.get_service('foobar-prod')['cluster'], '${terraform.cluster_name}') def test_nested_dict_interpolation(self): self.assertEqual( self.config.get_service('foobar-prod')['load_balancer']['load_balancer_name'], '${terraform.elb_id}' ) def test_nested_list_interpolation(self): self.assertEqual( self.config.get_service('foobar-prod')['containers'][0]['environment'][2], 'SECRETS_BUCKET_NAME=${terraform.secrets_bucket_name}' ) def test_environment_simple_interpolation(self): self.assertEqual(self.config.get_service('foobar-prod')['config'][0], 'FOOBAR=${env.FOOBAR_ENV}') self.assertEqual( self.config.get_service('foobar-prod')['config'][2], 'FOO_BAR_PREFIX=${env.FOO_BAR_PREFIX_ENV}/test' )
class TestContainerDefinition_load_yaml(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(os.path.abspath(__file__)) state_file = os.path.join(current_dir, 'terraform.tfstate') config_yml = os.path.join(current_dir, 'interpolate.yml') env_file = os.path.join(current_dir, 'env_file.env') with open(state_file) as f: tfstate = json.loads(f.read()) with Replacer() as r: get_mock = r( 'deployfish.terraform.Terraform._get_state_file_from_s3', Mock()) get_mock.return_value = tfstate self.config = Config(filename=config_yml, env_file=env_file) def tearDown(self): pass def test_terraform_simple_interpolation(self): self.assertEqual( self.config.get_service('foobar-prod')['cluster'], 'foobar-cluster-prod') def test_terraform_nested_dict_interpolation(self): self.assertEqual( self.config.get_service('foobar-prod')['load_balancer'] ['load_balancer_name'], 'foobar-elb-prod') def test_terraform_nested_list_interpolation(self): self.assertEqual( self.config.get_service('foobar-prod')['containers'][0] ['environment'][2], 'SECRETS_BUCKET_NAME=my-config-store') def test_terraform_list_output_interpolation(self): self.assertListEqual( self.config.get_service('foobar-prod')['vpc_configuration'] ['security_groups'], ['sg-1234567', 'sg-2345678', 'sg-3456789']) def test_terraform_map_output_interpolation(self): self.assertListEqual( self.config.get_service('output-test')['vpc_configuration'] ['subnets'], ['subnet-1234567']) self.assertListEqual( self.config.get_service('output-test')['vpc_configuration'] ['security_groups'], ['sg-1234567']) self.assertEqual( self.config.get_service('output-test')['vpc_configuration'] ['public_ip'], 'DISABLED') def test_environment_simple_interpolation(self): self.assertEqual( self.config.get_service('foobar-prod')['config'][0], 'FOOBAR=hi_mom') self.assertEqual( self.config.get_service('foobar-prod')['config'][2], 'FOO_BAR_PREFIX=oh_no/test')
class TestService_no_ec2_secrets_in_task_definition_if_no_execution_role(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(os.path.abspath(__file__)) filename = os.path.join(current_dir, 'simple.yml') self.config = Config(filename=filename, interpolate=False) del self.config.raw['services'][4]['execution_role'] client_mock = Mock() client_mock.get_parameters.return_value = {'Parameters': []} client_mock.describe_parameters.return_value = {'Parameters': []} session_mock = Mock(client=Mock(return_value=client_mock)) with Replacer() as r: r.replace('deployfish.aws.ecs.Service.from_aws', Mock()) r.replace('deployfish.aws.ecs.TaskDefinition.create', Mock()) r.replace('deployfish.aws.boto3_session', session_mock) self.service = Service('foobar-secrets-ec2', config=self.config) self.service.create() def test_sanity_check_name(self): self.assertEqual(self.service.serviceName, 'foobar-secrets-ec2') def test_sanity_check_config(self): self.assertEqual(len(self.config.get_service('foobar-secrets-ec2')['config']), 3) def test_config_with_execution_role(self): self.assertEqual(len(self.service.desired_task_definition.containers[0].secrets), 0)
def tunnel(ctx, tunnel_name): """ Tunnel through an EC2 instance in the ECS cluster. The parameters for this command should be found in a tunnels: top-level section in the yaml file, in the format: \b tunnels: - name: my_tunnel service: my_service host: config.MY_TUNNEL_DESTINATION_HOST port: 3306 local_port: 8888 where config.MY_TUNNEL_DESTINATION_HOST is the value of MY_TUNNEL_DESTINATION_HOST for this service in the AWS Parameter Store. The host value could also just be a hostname. """ config = Config(filename=ctx.obj['CONFIG_FILE'], env_file=ctx.obj['ENV_FILE'], import_env=ctx.obj['IMPORT_ENV'], tfe_token=ctx.obj['TFE_TOKEN']) yml = config.get_category_item('tunnels', tunnel_name) service_name = yml['service'] service = Service(yml=config.get_service(service_name)) host = _interpolate_tunnel_info(yml['host'], service) port = int(_interpolate_tunnel_info(yml['port'], service)) local_port = int(_interpolate_tunnel_info(yml['local_port'], service)) interim_port = random.randrange(10000, 64000, 1) service.tunnel(host, local_port, interim_port, port)
def entrypoint(ctx, command, dry_run): """ Use this as the entrypoint for your containers. It will look in the shell environment for the environment variables DEPLOYFISH_SERVICE_NAME and DEPLOYFISH_CLUSTER_NAME. If found, it will use them to: \b * download the parameters listed in "config:" section for service DEPLOYFISH_SERVICE_NAME from the AWS System Manager Parameter Store (which are prefixed by "${DEPLOYFISH_CLUSTER_NAME}.${DEPLOYFISH_SERVICE_NAME}.") * set those parameters and their values as environment variables * run COMMAND If either DEPLOYFISH_SERVICE_NAME or DEPLOYFISH_CLUSTER_NAME are not in the environment, just run COMMMAND. """ service_name = os.environ.get('DEPLOYFISH_SERVICE_NAME', None) cluster_name = os.environ.get('DEPLOYFISH_CLUSTER_NAME', None) if service_name and cluster_name: config = Config(filename=ctx.obj['CONFIG_FILE'], interpolate=False) try: service_yml = config.get_service(service_name) except KeyError: click.echo( "Our container's deployfish config file '{}' does not have service '{}'" .format(ctx.obj['CONFIG_FILE'] or 'deployfish.yml', service_name)) sys.exit(1) parameter_store = [] if 'config' in service_yml: parameter_store = ParameterStore(service_name, cluster_name, yml=service_yml['config']) parameter_store.populate() if not dry_run: for param in parameter_store: if param.exists: if param.should_exist: os.environ[param.key] = param.aws_value else: print( "event='deploy.entrypoint.parameter.ignored.not_in_deployfish_yml' service='{}' parameter='{}'" .format(service_name, param.name)) else: print( "event='deploy.entrypoint.parameter.ignored.not_in_aws' service='{}' parameter='{}'" .format(service_name, param.name)) else: exists = [] notexists = [] for param in parameter_store: if param.exists: exists.append(param) else: notexists.append(param) click.secho("Would have set these environment variables:", fg="cyan") for param in exists: click.echo(' {}={}'.format(param.key, param.aws_value)) click.secho("\nThese parameters are not in AWS:", fg="red") for param in notexists: click.echo(' {}'.format(param.key)) if dry_run: click.secho('\n\nCOMMAND: {}'.format(command)) else: subprocess.call(command)
def entrypoint(ctx, command, dry_run): """ Use this as the entrypoint for your containers. It will look in the shell environment for the environment variables DEPLOYFISH_SERVICE_NAME and DEPLOYFISH_CLUSTER_NAME. If found, it will use them to: \b * download the parameters listed in "config:" section for service DEPLOYFISH_SERVICE_NAME from the AWS System Manager Parameter Store (which are prefixed by "${DEPLOYFISH_CLUSTER_NAME}.${DEPLOYFISH_SERVICE_NAME}.") * set those parameters and their values as environment variables * run COMMAND If either DEPLOYFISH_SERVICE_NAME or DEPLOYFISH_CLUSTER_NAME are not in the environment, just run COMMMAND. \b NOTE: "deploy entrypoint" IGNORES any "aws:" section in your config file. We're assuming that you're only ever running "deploy entrypoint" inside a container in your AWS service. It should get its credentials from the container's IAM ECS Task Role. """ service_name = os.environ.get('DEPLOYFISH_SERVICE_NAME', None) cluster_name = os.environ.get('DEPLOYFISH_CLUSTER_NAME', None) if service_name and cluster_name: # The only thing we need out of Config is the names of any config: # section variables we might have. We don't need to do interpolation # in the config: section, because we retrieve the values from Parameter # Store, and we don't want to use any aws: section that might be in the # deployfish.yml to configure our boto3 session because we want to defer # to the IAM ECS Task Role. config = Config(filename=ctx.obj['CONFIG_FILE'], interpolate=False, use_aws_section=False) try: service_yml = config.get_service(service_name) except KeyError: click.echo( "Our container's deployfish config file '{}' does not have service '{}'" .format(ctx.obj['CONFIG_FILE'] or 'deployfish.yml', service_name)) sys.exit(1) parameter_store = [] if 'config' in service_yml: parameter_store = ParameterStore(service_name, cluster_name, yml=service_yml['config']) parameter_store.populate() if not dry_run: for param in parameter_store: if param.exists: if param.should_exist: os.environ[param.key] = param.aws_value else: print( "event='deploy.entrypoint.parameter.ignored.not_in_deployfish_yml' service='{}' parameter='{}'" .format(service_name, param.name)) else: print( "event='deploy.entrypoint.parameter.ignored.not_in_aws' service='{}' parameter='{}'" .format(service_name, param.name)) else: exists = [] notexists = [] for param in parameter_store: if param.exists: exists.append(param) else: notexists.append(param) click.secho("Would have set these environment variables:", fg="cyan") for param in exists: click.echo(' {}={}'.format(param.key, param.aws_value)) click.secho("\nThese parameters are not in AWS:", fg="red") for param in notexists: click.echo(' {}'.format(param.key)) if dry_run: click.secho('\n\nCOMMAND: {}'.format(command)) else: subprocess.call(command)