def main(): context = handle_args_and_set_context(sys.argv[1:]) profile = None if context.whereami == "ec2" else context.account_alias try: clients = ef_utils.create_aws_clients(EFConfig.DEFAULT_REGION, profile, "kms") except RuntimeError as error: ef_utils.fail( "Exception creating clients in region {} with profile {}".format( EFConfig.DEFAULT_REGION, profile), error) if context.secret_file: generate_secret_file(context.secret_file, context.match, context.service, context.env, clients) return if context.decrypt: decrypted = ef_utils.kms_decrypt(kms_client=clients['kms'], secret=context.decrypt) key_aliases = ef_utils.kms_key_alias(clients['kms'], decrypted.key_id) print("Decrypted Secret: {}; Key: {}".format(decrypted.plaintext, ', '.join(key_aliases))) return if context.plaintext: password = context.plaintext else: password = generate_secret(context.length) print("Generated Secret: {}".format(password)) encrypted_password = ef_utils.kms_encrypt(clients['kms'], context.service, context.env, password) print(format_secret(encrypted_password)) return
def test_create_aws_clients_no_profile(self, mock_session_constructor): """ Test create_aws_clients with all the parameters except profile and mocking the boto3 Session constructor. Verifies that all the keys show up in the dict object returned. Args: mock_session_constructor: MagicMock, returns Mock object representing a boto3.Session object Returns: None Raises: AssertionError if any of the assert checks fail """ mock_session = Mock(name="mock-boto3-session") mock_session.client.return_value = Mock(name="mock-client") mock_session_constructor.return_value = mock_session amazon_services = ["acm", "batch", "ec2", "sqs"] client_dict = ef_utils.create_aws_clients("us-west-2d", None, *amazon_services) self.assertTrue("acm" in client_dict) self.assertTrue("batch" in client_dict) self.assertTrue("ec2" in client_dict) self.assertTrue("sqs" in client_dict) self.assertTrue("SESSION" in client_dict)
def test_create_aws_clients_cache_same_client(self, mock_session_constructor): """ Test create_aws_clients with same parameters and mocking the boto3 Session constructor. Check that we get the same clients every time. Args: mock_session_constructor: MagicMock, returns Mock object representing a boto3.Session object Returns: None Raises: AssertionError if any of the assert checks fail """ mock_session = Mock(name="mock-boto3-session") # make sure we get different clients on every call mock_session.client.side_effect = lambda *args, **kwargs: Mock( name="mock-boto3-session") mock_session_constructor.return_value = mock_session amazon_services = ["acm", "batch", "ec2", "sqs"] cases = [ ("us-west-2d", None), ("us-west-3d", None), ("us-west-2d", "codemobs"), ("us-west-2d", "ellationeng"), ("", None), ] for region, profile in cases: clients1 = ef_utils.create_aws_clients(region, profile, *amazon_services) clients2 = ef_utils.create_aws_clients(region, profile, *amazon_services) self.assertEquals( clients1, clients2, msg= "Should get the same clients for the same region/profile pair")
def test_create_aws_clients_cache_new_clients(self, mock_session_constructor): """ Test create_aws_clients with same parameters and mocking the boto3 Session constructor. Check that we get the same clients every time. Args: mock_session_constructor: MagicMock, returns Mock object representing a boto3.Session object Returns: None Raises: AssertionError if any of the assert checks fail """ mock_session = Mock(name="mock-boto3-session") # make sure we get different clients on every call mock_session.client.side_effect = lambda *args, **kwargs: Mock( name="mock-boto3-session") mock_session_constructor.return_value = mock_session amazon_services = ["acm", "batch", "ec2", "sqs"] new_amazon_services = amazon_services + ["cloudfront"] region, profile = "us-west-2", "testing" clients = ef_utils.create_aws_clients(region, profile, *amazon_services) # copy the old clients, so they're not overwritten built_clients = {k: v for k, v in clients.items()} new_clients = ef_utils.create_aws_clients(region, profile, *new_amazon_services) for service in new_amazon_services: self.assertIn(service, new_clients) for service, client in built_clients.items(): self.assertEquals(new_clients.get(service), client)
def test_create_aws_clients_cache_multiple_configs( self, mock_session_constructor): """ Test create_aws_clients with multiple parameters and mocking the boto3 Session constructor. Check that every (region, profile) pair gets its own set of clients. Args: mock_session_constructor: MagicMock, returns Mock object representing a boto3.Session object Returns: None Raises: AssertionError if any of the assert checks fail """ mock_session = Mock(name="mock-boto3-session") # make sure we get different clients on every call mock_session.client.side_effect = lambda *args, **kwargs: Mock( name="mock-boto3-session") mock_session_constructor.return_value = mock_session amazon_services = ["acm", "batch", "ec2", "sqs"] cases = [ ("us-west-2d", None), ("us-west-3d", None), ("us-west-2d", "codemobs"), ("us-west-2d", "ellationeng"), ("", None), ] built_clients = {} for region, profile in cases: client_dict = ef_utils.create_aws_clients(region, profile, *amazon_services) for key, clients in built_clients.items(): # check if the new clients are unique self.assertNotEquals( client_dict, clients, msg="Duplicate clients for {} vs {}".format( key, (region, profile))) built_clients[(region, profile)] = client_dict
def main(): # Fetch args and load context context = handle_args_and_set_context(sys.argv[1:]) # Refresh from repo if necessary and possible (gets don't need service registry, sets do) if (context.rollback or context.value) and not (context.devel or getenv( "JENKINS_URL", False)): print("Refreshing repo") try: pull_repo() except RuntimeError as error: fail("Error checking or pulling repo", error) # Sign on to AWS and create clients if context.whereami in ["ec2"]: # Always use instance credentials in EC2. One day we'll have "lambda" in there too, so use "in" w/ list aws_session_alias = None else: # Otherwise use local user credential matching the account alias aws_session_alias = context.account_alias # Make AWS clients try: context.set_aws_clients( create_aws_clients(EFConfig.DEFAULT_REGION, aws_session_alias, "ec2", "s3", "sts")) except RuntimeError: fail( "Exception creating AWS client in region {} with aws account alias {} (None=instance credentials)" .format(EFConfig.DEFAULT_REGION, aws_session_alias)) # Instantiate a versionresolver - we'll use some of its methods context._versionresolver = EFVersionResolver(context.aws_client()) # Carry out the requested action if context.get: cmd_get(context) elif context.history: cmd_history(context) elif context.rollback: cmd_rollback(context) elif context.rollback_to: cmd_rollback_to(context) elif context.show: cmd_show(context) elif context.value: cmd_set(context)
def __init__(self, profile=None, region=None, # set both for user access mode lambda_context=None, # set if target is 'self' and this is a lambda target_other=False, env=None, service=None, # set env & service if target_other=True verbose=False ): """ Depending on how this is called, access mode (how it logs into AWS) and target (what the various context vars report) will vary ACCESS MODE - how this logs in to AWS user: "******" both 'profile' and 'region' are required is always "operating on something else" (TARGET is never "self") role: "running in AWS EC2 or Lambda with a role credential" do not set profile TARGET - what context is reported self: "this ec2 instance or lambda is initializing itself" assumed for ec2 and lambda, unless target_other=True in the constructor never an option for "user" access mode other: "this local user, ec2 instance, or lambda is configuring something else" always "other" if access mode is "user" if access mode is "role", set target_other=True in the constructor Constructor must also set 'env' and 'service' self: lambda_context must be provided if this is a lambda; leave it unset for ec2 INSTANCE_ID = instance ID if EC2, else None FUNCTION_NAME = function name if lambda, else None ACCOUNT = numeric account this is running in ACCOUNT_ALIAS = named alias of the account this is running in ROLE = role this is running as ENV = the environment this is running in, from role name ENV_SHORT = derived from ENV ENV_FULL = fully qualified environment, same as ENV unless env is a global env (mgmt.* or global.*) SERVICE = the service this is, from role name REGION = region this is running in something else: INSTANCE_ID = None FUNCTION_NAME = None ACCOUNT = the numeric account I'm logged into (look up) ACCOUNT_ALIAS = the alias of the account i'm logged into (look up) ROLE = None ENV = the target's environment, passed in from the constructor ENV_SHORT = derived from ENV ENV_FULL = ENV, with ".<ACCOUNT_ALIAS>" as appropriate SERVICE = the service name, passed in from the constructor REGION = the region I am in (ec2, lambda) or explicitly set (region= in constructor) Collects instance's environment for use in templates: {{ACCOUNT}} - AWS account number CloudFormation can use this or the AWS::AccountID pseudo param {{ACCOUNT_ALIAS}} - AWS account alias {{ENV}} - environment: mgmt, prod, staging, proto<N>, etc. {{ENV_SHORT}} - env with <N> or account trimmed: mgmt, prod, staging, proto, etc. {{ENV_FULL}} - env fully qualified: prod, staging, proto<N>, mgmt.<account_alias>, etc. {{FUNCTION_NAME}} - only for lambdas {{INSTANCE_ID}} - only for ec2 {{REGION}} - the region currently being worked in CloudFormation can use this or the AWS::Region pseudo param {{ROLE}} - the role bound to the ec2 instance or lambda; only for ec2 and lambda CloudFormation: compose role name in template by joining other strings """ # instance vars self.verbose = False # print noisy status if True # resolved tokens - only look up symbols once per session. Protect internal names by declaring self.resolved = { "ACCOUNT": None, "ACCOUNT_ALIAS": None, "ENV": None, "ENV_SHORT": None, "ENV_FULL": None, "FUNCTION_NAME": None, "INSTANCE_ID": None, "REGION": None, "ROLE": None } # template and parameters are populated by the load() method as each template is processed self.template = None # parameters that accompany this template, if any self.parameters = {} # Sets of symbols found in the current template (only) # read back with self.symbols() and self.unresolved_symbols() self.symbols = set() # capture verbosity pref from constructor self.verbose = verbose # determine ACCESS MODE if profile: # accessing as a user target_other = True if not region: fail("'region' is required with 'profile' for user-mode access") where = whereami() # require env and service params init() when target is 'other' if (target_other or where == "virtualbox-kvm") and (env is None or service is None): fail("'env' and 'service' must be set when target is 'other' or running in " + where) if target_other or profile: self.resolved["REGION"] = region # lambda initializing self elif lambda_context: self.resolved["REGION"] = lambda_context.invoked_function_arn.split(":")[3] # ec2 initializing self else: self.resolved["REGION"] = get_metadata_or_fail("placement/availability-zone/")[:-1] # Create clients - if accessing by role, profile should be None clients = [ "cloudformation", "cloudfront", "cognito-identity", "cognito-idp", "ec2", "elbv2", "iam", "kms", "lambda", "route53", "s3", "sts", "waf" ] try: EFTemplateResolver.__CLIENTS = create_aws_clients(self.resolved["REGION"], profile, *clients) except RuntimeError as error: fail("Exception logging in with Session()", error) # Create EFAwsResolver object for interactive lookups EFTemplateResolver.__AWSR = EFAwsResolver(EFTemplateResolver.__CLIENTS) # Create EFConfigResolver object for ef tooling config lookups EFTemplateResolver.__EFCR = EFConfigResolver() # Create EFVersionResolver object for version lookups EFTemplateResolver.__VR = EFVersionResolver(EFTemplateResolver.__CLIENTS) # Set the internal parameter values for aws # self-configuring lambda if (not target_other) and lambda_context: arn_split = lambda_context.invoked_function_arn.split(":") self.resolved["ACCOUNT"] = arn_split[4] self.resolved["FUNCTION_NAME"] = arn_split[6] try: lambda_desc = EFTemplateResolver.__CLIENTS["lambda"].get_function() except: fail("Exception in get_function: ", sys.exc_info()) self.resolved["ROLE"] = lambda_desc["Configuration"]["Role"] env = re.search("^({})-".format(EFConfig.VALID_ENV_REGEX), self.resolved["ROLE"]) if not env: fail("Did not find environment in lambda function name.") self.resolved["ENV"] = env.group(1) parsed_service = re.search(self.resolved["ENV"] + "-(.*?)-lambda", self.resolved["ROLE"]) if parsed_service: self.resolved["SERVICE"] = parsed_service.group(1) # self-configuring EC2 elif (not target_other) and (not lambda_context): self.resolved["INSTANCE_ID"] = get_metadata_or_fail('instance-id') try: instance_desc = EFTemplateResolver.__CLIENTS["ec2"].describe_instances(InstanceIds=[self.resolved["INSTANCE_ID"]]) except: fail("Exception in describe_instances: ", sys.exc_info()) self.resolved["ACCOUNT"] = instance_desc["Reservations"][0]["OwnerId"] arn = instance_desc["Reservations"][0]["Instances"][0]["IamInstanceProfile"]["Arn"] self.resolved["ROLE"] = arn.split(":")[5].split("/")[1] env = re.search("^({})-".format(EFConfig.VALID_ENV_REGEX), self.resolved["ROLE"]) if not env: fail("Did not find environment in role name") self.resolved["ENV"] = env.group(1) self.resolved["SERVICE"] = "-".join(self.resolved["ROLE"].split("-")[1:]) # target is "other" else: try: if whereami() == "ec2": self.resolved["ACCOUNT"] = str(json.loads(http_get_metadata('iam/info'))["InstanceProfileArn"].split(":")[4]) else: self.resolved["ACCOUNT"] = get_account_id(EFTemplateResolver.__CLIENTS["sts"]) except botocore.exceptions.ClientError as error: fail("Exception in get_user()", error) self.resolved["ENV"] = env self.resolved["SERVICE"] = service # ACCOUNT_ALIAS is resolved consistently for access modes and targets other than virtualbox try: self.resolved["ACCOUNT_ALIAS"] = EFTemplateResolver.__CLIENTS["iam"].list_account_aliases()["AccountAliases"][0] except botocore.exceptions.ClientError as error: fail("Exception in list_account_aliases", error) # ENV_SHORT is resolved the same way for all access modes and targets self.resolved["ENV_SHORT"] = self.resolved["ENV"].strip(".0123456789") # ENV_FULL is resolved the same way for all access modes and targets, depending on previously-resolved values if self.resolved["ENV"] in EFConfig.ACCOUNT_SCOPED_ENVS: self.resolved["ENV_FULL"] = "{}.{}".format(self.resolved["ENV"], self.resolved["ACCOUNT_ALIAS"]) else: self.resolved["ENV_FULL"] = self.resolved["ENV"] if self.verbose: print(repr(self.resolved), file=sys.stderr)
def main(): global CONTEXT, CLIENTS, AWS_RESOLVER CONTEXT = handle_args_and_set_context(sys.argv[1:]) if not (CONTEXT.devel or getenv("JENKINS_URL", False)): try: pull_repo() except RuntimeError as error: fail("Error checking or pulling repo", error) else: print("Not refreshing repo because --devel was set or running on Jenkins") # sign on to AWS and create clients and get account ID try: # If running in EC2, always use instance credentials. One day we'll have "lambda" in there too, so use "in" w/ list if CONTEXT.whereami == "ec2": CLIENTS = create_aws_clients(EFConfig.DEFAULT_REGION, None, "ec2", "iam", "kms") CONTEXT.account_id = str(json.loads(http_get_metadata('iam/info'))["InstanceProfileArn"].split(":")[4]) else: # Otherwise, we use local user creds based on the account alias CLIENTS = create_aws_clients(EFConfig.DEFAULT_REGION, CONTEXT.account_alias, "ec2", "iam", "kms", "sts") CONTEXT.account_id = get_account_id(CLIENTS["sts"]) except RuntimeError: fail("Exception creating AWS clients in region {} with profile {}".format( EFConfig.DEFAULT_REGION, CONTEXT.account_alias)) # Instantiate an AWSResolver to lookup AWS resources AWS_RESOLVER = EFAwsResolver(CLIENTS) # Show where we're working if not CONTEXT.commit: print("=== DRY RUN ===\nUse --commit to create roles and security groups\n=== DRY RUN ===") print("env: {}".format(CONTEXT.env)) print("env_full: {}".format(CONTEXT.env_full)) print("env_short: {}".format(CONTEXT.env_short)) print("aws account profile: {}".format(CONTEXT.account_alias)) print("aws account number: {}".format(CONTEXT.account_id)) # Step through all services in the service registry for CONTEXT.service in CONTEXT.service_registry.iter_services(): service_name = CONTEXT.service[0] target_name = "{}-{}".format(CONTEXT.env, service_name) sr_entry = CONTEXT.service[1] service_type = sr_entry['type'] print_if_verbose("service: {} in env: {}".format(service_name, CONTEXT.env)) # Is this service_type handled by this tool? if service_type not in SUPPORTED_SERVICE_TYPES: print_if_verbose("unsupported service type: {}".format(service_type)) continue # Is the env valid for this service? if CONTEXT.env_full not in CONTEXT.service_registry.valid_envs(service_name): print_if_verbose("env: {} not valid for service {}".format(CONTEXT.env_full, service_name)) continue # Is the service_type allowed in 'global'? if CONTEXT.env == "global" and service_type not in GLOBAL_SERVICE_TYPES: print_if_verbose("env: {} not valid for service type {}".format(CONTEXT.env, service_type)) continue # 1. CONDITIONALLY MAKE ROLE AND/OR INSTANCE PROFILE FOR THE SERVICE # If service gets a role, create with either a custom or default AssumeRole policy document conditionally_create_role(target_name, sr_entry) # Instance profiles and security groups are not allowed in the global scope if CONTEXT.env != "global": conditionally_create_profile(target_name, service_type) # 2. SECURITY GROUP(S) FOR THE SERVICE : only some types of services get security groups conditionally_create_security_groups(CONTEXT.env, service_name, service_type) # 3. KMS KEY FOR THE SERVICE : only some types of services get kms keys conditionally_create_kms_key(target_name, service_type) # 4. ATTACH AWS MANAGED POLICIES TO ROLE conditionally_attach_aws_managed_policies(target_name, sr_entry) # 5. ATTACH CUSTOMER MANAGED POLICIES TO ROLE conditionally_attach_customer_managed_policies(target_name, sr_entry) # 6. INLINE SERVICE'S POLICIES INTO ROLE # only eligible service types with "policies" sections in the service registry get policies conditionally_inline_policies(target_name, sr_entry) print("Exit: success")
def main(): context = handle_args_and_set_context(sys.argv[1:]) # argument sanity checks and contextual messages if context.commit and context.changeset: fail("Cannot use --changeset and --commit together") if context.changeset: print( "=== CHANGESET ===\nCreating changeset only. See AWS GUI for changeset\n=== CHANGESET ===" ) elif not context.commit: print( "=== DRY RUN ===\nValidation only. Use --commit to push template to CF\n=== DRY RUN ===" ) service_name = basename(splitext(context.template_file)[0]) template_file_dir = dirname(context.template_file) # parameter file may not exist, but compute the name it would have if it did parameter_file_dir = template_file_dir + "/../parameters" parameter_file = parameter_file_dir + "/" + service_name + ".parameters." + context.env_full + ".json" # If running in EC2, use instance credentials (i.e. profile = None) # otherwise, use local credentials with profile name in .aws/credentials == account alias name if context.whereami == "ec2": profile = None else: profile = context.account_alias # Get service registry and refresh repo if appropriate try: if not (context.devel or getenv("JENKINS_URL", False)): pull_repo() else: print( "not refreshing repo because --devel was set or running on Jenkins" ) except Exception as error: fail("Error: ", error) # Service must exist in service registry if context.service_registry.service_record(service_name) is None: fail("service: {} not found in service registry: {}".format( service_name, context.service_registry.filespec)) if not context.env_full in context.service_registry.valid_envs( service_name): fail("Invalid environment: {} for service_name: {}\nValid environments are: {}" \ .format(context.env_full, service_name, ", ".join(context.service_registry.valid_envs(service_name)))) if context.verbose: print("service_name: {}".format(service_name)) print("env: {}".format(context.env)) print("env_full: {}".format(context.env_full)) print("env_short: {}".format(context.env_short)) print("template_file: {}".format(context.template_file)) print("parameter_file: {}".format(parameter_file)) if profile: print("profile: {}".format(profile)) print("whereami: {}".format(context.whereami)) print("service type: {}".format( context.service_registry.service_record(service_name)["type"])) template = resolve_template(template=context.template_file, profile=profile, env=context.env, region=EFConfig.DEFAULT_REGION, service=service_name, verbose=context.verbose) # Create clients - if accessing by role, profile should be None try: clients = create_aws_clients(EFConfig.DEFAULT_REGION, profile, "cloudformation") except RuntimeError as error: fail( "Exception creating clients in region {} with profile {}".format( EFConfig.DEFAULT_REGION, profile), error) stack_name = context.env + "-" + service_name try: stack_exists = clients["cloudformation"].describe_stacks( StackName=stack_name) except botocore.exceptions.ClientError: stack_exists = False # Load parameters from file if isfile(parameter_file): parameters_template = resolve_template(template=parameter_file, profile=profile, env=context.env, region=EFConfig.DEFAULT_REGION, service=service_name, verbose=context.verbose) try: parameters = json.loads(parameters_template) except ValueError as error: fail("JSON error in parameter file: {}".format( parameter_file, error)) else: parameters = [] # Validate rendered template before trying the stack operation if context.verbose: print("Validating template") try: clients["cloudformation"].validate_template(TemplateBody=template) except botocore.exceptions.ClientError as error: fail("Template did not pass validation", error) print("Template passed validation") # DO IT try: if context.changeset: print("Creating changeset: {}".format(stack_name)) clients["cloudformation"].create_change_set( StackName=stack_name, TemplateBody=template, Parameters=parameters, Capabilities=['CAPABILITY_IAM'], ChangeSetName=stack_name, ClientToken=stack_name) elif context.commit: if stack_exists: print("Updating stack: {}".format(stack_name)) clients["cloudformation"].update_stack( StackName=stack_name, TemplateBody=template, Parameters=parameters, Capabilities=['CAPABILITY_IAM']) else: print("Creating stack: {}".format(stack_name)) clients["cloudformation"].create_stack( StackName=stack_name, TemplateBody=template, Parameters=parameters, Capabilities=['CAPABILITY_IAM']) if context.poll_status: while True: stack_status = clients["cloudformation"].describe_stacks( StackName=stack_name)["Stacks"][0]["StackStatus"] if context.verbose: print("{}".format(stack_status)) if stack_status.endswith('ROLLBACK_COMPLETE'): print( "Stack went into rollback with status: {}".format( stack_status)) sys.exit(1) elif re.match(r".*_COMPLETE(?!.)", stack_status) is not None: break elif re.match(r".*_FAILED(?!.)", stack_status) is not None: print("Stack failed with status: {}".format( stack_status)) sys.exit(1) elif re.match(r".*_IN_PROGRESS(?!.)", stack_status) is not None: time.sleep(EFConfig.EF_CF_POLL_PERIOD) run_plugins(context, clients) except botocore.exceptions.ClientError as error: if error.response["Error"][ "Message"] in "No updates are to be performed.": # Don't fail when there is no update to the stack print("No updates are to be performed.") else: fail("Error occurred when creating or updating stack", error)
def main(): context = handle_args_and_set_context(sys.argv[1:]) if context.changeset: print( "=== CHANGESET ===\nCreating changeset only. See AWS GUI for changeset\n=== CHANGESET ===" ) elif not context.commit: print( "=== DRY RUN ===\nValidation only. Use --commit to push template to CF\n=== DRY RUN ===" ) service_name = os.path.basename(os.path.splitext(context.template_file)[0]) template_file_dir = os.path.dirname(context.template_file) # parameter file may not exist, but compute the name it would have if it did parameter_file_dir = template_file_dir + "/../parameters" parameter_file = parameter_file_dir + "/" + service_name + ".parameters." + context.env_full + ".json" # If running in EC2, use instance credentials (i.e. profile = None) # otherwise, use local credentials with profile name in .aws/credentials == account alias name if context.whereami == "ec2" and not os.getenv("JENKINS_URL", False): profile = None else: profile = context.account_alias # Get service registry and refresh repo if appropriate try: if not (context.devel or os.getenv("JENKINS_URL", False)): pull_repo() else: print( "not refreshing repo because --devel was set or running on Jenkins" ) except Exception as error: fail("Error: ", error) # Service must exist in service registry if context.service_registry.service_record(service_name) is None: fail("service: {} not found in service registry: {}".format( service_name, context.service_registry.filespec)) if not context.env_full in context.service_registry.valid_envs( service_name): fail("Invalid environment: {} for service_name: {}\nValid environments are: {}" \ .format(context.env_full, service_name, ", ".join(context.service_registry.valid_envs(service_name)))) if context.percent and (context.percent <= 0 or context.percent > 100): fail( "Percent value cannot be less than or equal to 0 and greater than 100" ) # Set the region found in the service_registry. Default is EFConfig.DEFAULT_REGION if region key not found region = context.service_registry.service_region(service_name) if context.verbose: print("service_name: {}".format(service_name)) print("env: {}".format(context.env)) print("env_full: {}".format(context.env_full)) print("env_short: {}".format(context.env_short)) print("region: {}".format(region)) print("template_file: {}".format(context.template_file)) print("parameter_file: {}".format(parameter_file)) if profile: print("profile: {}".format(profile)) print("whereami: {}".format(context.whereami)) print("service type: {}".format( context.service_registry.service_record(service_name)["type"])) template = resolve_template(template=context.template_file, profile=profile, env=context.env, region=region, service=service_name, verbose=context.verbose) # Create clients - if accessing by role, profile should be None try: clients = create_aws_clients(region, profile, "cloudformation", "autoscaling") except RuntimeError as error: fail( "Exception creating clients in region {} with profile {}".format( region, profile), error) stack_name = context.env + "-" + service_name try: stack_exists = clients["cloudformation"].describe_stacks( StackName=stack_name) except botocore.exceptions.ClientError: stack_exists = False # Load parameters from file if os.path.isfile(parameter_file): parameters_template = resolve_template(template=parameter_file, profile=profile, env=context.env, region=region, service=service_name, verbose=context.verbose) try: parameters = json.loads(parameters_template) except ValueError as error: fail("JSON error in parameter file: {}".format( parameter_file, error)) else: parameters = [] if context.percent: print("Modifying deploy rate to {}%".format(context.percent)) modify_template = json.loads(template) for key in modify_template["Resources"]: if modify_template["Resources"][key][ "Type"] == "AWS::AutoScaling::AutoScalingGroup": if modify_template["Resources"][key]["UpdatePolicy"]: autoscaling_group = modify_template["Resources"][key][ "Properties"] service = autoscaling_group["Tags"][0]["Value"] autoscaling_group_properties = get_autoscaling_group_properties( clients["autoscaling"], service.split("-")[0], "-".join(service.split("-")[1:])) new_max_batch_size = calculate_max_batch_size( clients["autoscaling"], service, context.percent) modify_template["Resources"][key]["UpdatePolicy"][ "AutoScalingRollingUpdate"][ "MaxBatchSize"] = new_max_batch_size current_desired = autoscaling_group_properties[0][ "DesiredCapacity"] if autoscaling_group_properties else "missing" print( "Service {} [current desired: {}, calculated max batch size: {}]" .format(service, current_desired, new_max_batch_size)) template = json.dumps(modify_template) # Detect if the template exceeds the maximum size that is allowed by Cloudformation if len(template) > CLOUDFORMATION_SIZE_LIMIT: # Compress the generated template by removing whitespaces print( "Template exceeds the max allowed length that Cloudformation will accept. Compressing template..." ) print("Uncompressed size of template: {}".format(len(template))) unpacked = json.loads(template) template = json.dumps(unpacked, separators=(",", ":")) print("Compressed size of template: {}".format(len(template))) # Validate rendered template before trying the stack operation if context.verbose: print("Validating template") try: clients["cloudformation"].validate_template(TemplateBody=template) json.loads( template) # Tests for valid JSON syntax, oddly not handled above except botocore.exceptions.ClientError as error: fail("Template did not pass validation", error) except ValueError as e: # includes simplejson.decoder.JSONDecodeError fail('Failed to decode JSON', e) print("Template passed validation") # DO IT try: if context.changeset: print("Creating changeset: {}".format(stack_name)) results = clients["cloudformation"].create_change_set( StackName=stack_name, TemplateBody=template, Parameters=parameters, Capabilities=[ 'CAPABILITY_AUTO_EXPAND', 'CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM' ], ChangeSetName=stack_name, ClientToken=stack_name) if is_stack_termination_protected_env(context.env): enable_stack_termination_protection(clients, stack_name) results_ids = { key: value for key, value in results.iteritems() if key in ('Id', 'StackId') } print("Changeset Info: {}".format(json.dumps(results_ids))) elif context.commit: if stack_exists: print("Updating stack: {}".format(stack_name)) clients["cloudformation"].update_stack( StackName=stack_name, TemplateBody=template, Parameters=parameters, Capabilities=[ 'CAPABILITY_AUTO_EXPAND', 'CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM' ]) if is_stack_termination_protected_env(context.env): enable_stack_termination_protection(clients, stack_name) else: print("Creating stack: {}".format(stack_name)) clients["cloudformation"].create_stack( StackName=stack_name, TemplateBody=template, Parameters=parameters, Capabilities=[ 'CAPABILITY_AUTO_EXPAND', 'CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM' ]) if is_stack_termination_protected_env(context.env): enable_stack_termination_protection(clients, stack_name) if context.poll_status: while True: stack_status = clients["cloudformation"].describe_stacks( StackName=stack_name)["Stacks"][0]["StackStatus"] if context.verbose: print("{}".format(stack_status)) if stack_status.endswith('ROLLBACK_COMPLETE'): print( "Stack went into rollback with status: {}".format( stack_status)) sys.exit(1) elif re.match(r".*_COMPLETE(?!.)", stack_status) is not None: break elif re.match(r".*_FAILED(?!.)", stack_status) is not None: print("Stack failed with status: {}".format( stack_status)) sys.exit(1) elif re.match(r".*_IN_PROGRESS(?!.)", stack_status) is not None: time.sleep(EFConfig.EF_CF_POLL_PERIOD) elif context.lint: tester = CFTemplateLinter(template) tester.run_tests() exit(tester.exit_code) except botocore.exceptions.ClientError as error: if error.response["Error"][ "Message"] in "No updates are to be performed.": # Don't fail when there is no update to the stack print("No updates are to be performed.") else: fail("Error occurred when creating or updating stack", error)