def output(layer, config, layer_name=None, output_name=None, environment_name=None, stage=None): """Gets the value of an output produced by an already deployed layer. :param layer: The Layer object for the layer declaring the reference. :param config: An object holding humilis configuration options. :param layer_name: The logical name of the layer that produced the output. :param output_name: The logical name of the output parameter. """ if not environment_name or not stage: environment_name = layer.env_name stage = layer.env_stage stack_name = utils.get_cf_name(environment_name, layer_name, stage=stage) cf = Cloudformation(config) try: output = cf.get_stack_output(stack_name, output_name) except AttributeError: msg = "No output '{}' in CF stack '{}'".format(output_name, stack_name) ref = "output/{}/{}/{}/{}".format(environment_name, layer_name, stage, output_name) raise ReferenceError(ref, msg) if len(output) < 1: all_stack_outputs = [x['OutputKey'] for x in cf.stack_outputs[stack_name]] msg = ("{} output does not exist for stack {} " "(with outputs {}).").format(output_name, stack_name, all_stack_outputs) ref = "output ({}/{})".format(layer_name, output_name) raise ReferenceError(ref, msg, logger=layer.logger) return output[0]
def _get_stack_resource(layer, config, stack_name, resource_name): """Gets the physical ID of a resource in a CF Stack. :param stack_name: The name of the CF stack. :param resource_name: The logical name of the CF resource. :returns: The physical ID of the resource. """ cf = Cloudformation(config) resource = cf.get_stack_resource(stack_name, resource_name) if len(resource) < 1: all_stack_resources = [x.logical_resource_id for x in cf.get_stack_resources(stack_name)] msg = "{} does not exist in stack {} (with resources {}).".format( resource_name, stack_name, all_stack_resources) raise ReferenceError(resource_name, msg, logger=layer.logger) return resource[0].physical_resource_id
def _get_stack_resource(config, stack_name, resource_name): """Gets the physical ID of a resource in a CF Stack. :param stack_name: The name of the CF stack. :param resource_name: The logical name of the CF resource. :returns: The physical ID of the resource. """ cf = Cloudformation(config) resource = cf.get_stack_resource(stack_name, resource_name) if len(resource) < 1: all_stack_resources = [x.logical_resource_id for x in cf.get_stack_resources(stack_name)] msg = "{} does not exist in stack {} (with resources {}).".format( resource_name, stack_name, all_stack_resources) raise ReferenceError(resource_name, msg, logger=layer.logger) return resource[0].physical_resource_id
def output(layer, config, layer_name=None, output_name=None): """Gets the value of an output produced by an already deployed layer. :param layer: The Layer object for the layer declaring the reference. :param config: An object holding humilis configuration options. :param layer_name: The logical name of the layer that produced the output. :param output_name: The logical name of the output parameter. """ stack_name = utils.get_cf_name(layer.env_name, layer_name, stage=layer.env_stage) cf = Cloudformation(config) output = cf.get_stack_output(stack_name, output_name) if len(output) < 1: all_stack_outputs = [x['OutputKey'] for x in cf.stack_outputs[stack_name]] msg = ("{} output does not exist for stack {} " "(with outputs {}).").format(output_name, stack_name, all_stack_outputs) ref = "output ({}/{})".format(layer_name, output_name) raise ReferenceError(ref, msg, logger=layer.logger) return output[0]
def __init__(self, __env, __name, layer_type=None, logger=None, loader=None, humilis_profile=None, **user_params): self.__environment_repr = repr(__env) self.environment = __env if not humilis_profile: self.cf = self.environment.cf else: config.boto_config.activate_profile(humilis_profile) self.cf = Cloudformation(config.boto_config) if logger is None: self.logger = logging.getLogger(__name__) # To prevent warnings self.logger.addHandler(logging.NullHandler()) else: self.logger = logger self.name = __name self.env_name = self.environment.name self.env_stage = self.environment.stage self.env_basedir = self.environment.basedir self.depends_on = [] self.section = {} self.type = layer_type self.s3_prefix = "{base}{env}/{stage}/{layer}/".format( base=config.boto_config.profile.get("s3prefix"), env=self.environment.name, stage=self.environment.stage, layer=__name) if layer_type is not None: basedir = config.layers.get(layer_type) if not basedir: msg = ("The plugin providing the layer type '{}' is not " "installed in this system. Please install it and " "try again.").format(layer_type) raise MissingPluginError(msg) else: basedir = None if basedir is None: basedir = os.path.join(self.env_basedir, 'layers', self.name) self.basedir = basedir if loader is None: loader = DirTreeBackedObject(basedir, self.logger) self.loader = loader # These param set will be sent to the template compiler and will be # populated once the layers this layer depend on have been created. self.params = {} # the parameters that will be used to compile meta.yaml self.meta = {} meta_params = { p[0]: p[1] for p in itertools.chain(self.loader_params.items(), user_params.items()) } self.meta = self.loader.load_section('meta', params=meta_params) self.sns_topic_arn = self.environment.sns_topic_arn self.tags = { 'humilis:environment': self.env_name, 'humilis:layer': self.name, 'humilis:stage': self.env_stage, 'humilis:created': str(datetime.datetime.now()) } for tagname, tagvalue in self.environment.tags.items(): self.tags[tagname] = tagvalue for tagname, tagvalue in self.meta.get('tags', {}).items(): self.tags[tagname] = tagvalue self.yaml_params = self.meta.get('parameters', {}) for k, v in self.yaml_params.items(): # Set 1 as default priority for all parameters v['priority'] = v.get('priority', 1) # User params override what is in the layer definition file self.user_params = user_params for pname, pvalue in user_params.items(): if pname in self.yaml_params: self.yaml_params[pname]['value'] = pvalue self.__ec2 = None self.__s3 = None
class Layer: """A layer of infrastructure that translates into a single CF stack""" def __init__(self, __env, __name, layer_type=None, logger=None, loader=None, humilis_profile=None, **user_params): self.__environment_repr = repr(__env) self.environment = __env if not humilis_profile: self.cf = self.environment.cf else: config.boto_config.activate_profile(humilis_profile) self.cf = Cloudformation(config.boto_config) if logger is None: self.logger = logging.getLogger(__name__) # To prevent warnings self.logger.addHandler(logging.NullHandler()) else: self.logger = logger self.name = __name self.env_name = self.environment.name self.env_stage = self.environment.stage self.env_basedir = self.environment.basedir self.depends_on = [] self.section = {} self.type = layer_type self.s3_prefix = "{base}{env}/{stage}/{layer}/".format( base=config.boto_config.profile.get("s3prefix"), env=self.environment.name, stage=self.environment.stage, layer=__name) if layer_type is not None: basedir = config.layers.get(layer_type) if not basedir: msg = ("The plugin providing the layer type '{}' is not " "installed in this system. Please install it and " "try again.").format(layer_type) raise MissingPluginError(msg) else: basedir = None if basedir is None: basedir = os.path.join(self.env_basedir, 'layers', self.name) self.basedir = basedir if loader is None: loader = DirTreeBackedObject(basedir, self.logger) self.loader = loader # These param set will be sent to the template compiler and will be # populated once the layers this layer depend on have been created. self.params = {} # the parameters that will be used to compile meta.yaml self.meta = {} meta_params = { p[0]: p[1] for p in itertools.chain(self.loader_params.items(), user_params.items()) } self.meta = self.loader.load_section('meta', params=meta_params) self.sns_topic_arn = self.environment.sns_topic_arn self.tags = { 'humilis:environment': self.env_name, 'humilis:layer': self.name, 'humilis:stage': self.env_stage, 'humilis:created': str(datetime.datetime.now()) } for tagname, tagvalue in self.environment.tags.items(): self.tags[tagname] = tagvalue for tagname, tagvalue in self.meta.get('tags', {}).items(): self.tags[tagname] = tagvalue self.yaml_params = self.meta.get('parameters', {}) for k, v in self.yaml_params.items(): # Set 1 as default priority for all parameters v['priority'] = v.get('priority', 1) # User params override what is in the layer definition file self.user_params = user_params for pname, pvalue in user_params.items(): if pname in self.yaml_params: self.yaml_params[pname]['value'] = pvalue self.__ec2 = None self.__s3 = None @property def termination_protection(self): """Is termination protection set for this layer?.""" return self.meta.get('parameters', {}).get('termination_protection', {}).get('value', False) @property def cf_name(self): """The name of the CF stack associated to this layer.""" return get_cf_name(self.env_name, self.name, stage=self.env_stage) @property def loader_params(self): """Produces a dictionary of parameters to pass to a section loader.""" # User parameters in the layer meta.yaml # Not that some param values may not have been populated when this # property is accessed since that may happen during the parsing of # some references in the parameter list: thus the if 'value' in v # For backwards compatibility, to be deprecated params = { k: v['value'] for k, v in self.params.items() if 'value' in v } params["__vars"] = dict(params) # For backwards compatibility, to be deprecated params['_env'] = { 'stage': self.env_stage, 'name': self.env_name, 'basedir': self.env_basedir } params['_os_env'] = os.environ params['_layer'] = {'name': self.name} params['env'] = os.environ # The new format: params['__env'] = os.environ params['__context'] = { 'environment': { 'name': self.env_name, 'basedir': self.env_basedir, 'tags': self.environment.tags }, 'stage': self.env_stage, 'layer': { 'name': self.name, 'basedir': self.basedir }, 'aws': { 'account_id': boto3.client('sts').get_caller_identity().get('Account') } } # For backwards compatibility params["context"] = params["__context"] return params @property def in_cf(self): """Returns true if the layer has been already deployed to CF.""" return self.cf_name in {stk['StackName'] for stk in self.cf.stacks} @property def ec2(self): """Connection to AWS EC2 service.""" if self.__ec2 is None: self.__ec2 = Ec2(config.boto_config) return self.__ec2 @property def s3(self): """Connection to AWS S3.""" if self.__s3 is None: self.__s3 = S3(config.boto_config) return self.__s3 @property def ok(self): """Layer is fully deployed in CF and ready for use""" return self.cf.stack_ok(self.cf_name) @property def outputs(self): """Layer CF outputs.""" ly = self.cf.stack_outputs.get(self.cf_name) if ly: ly = {o['OutputKey']: o['OutputValue'] for o in ly} return ly @property def resources(self): """Layer CF resources.""" ly = self.cf.get_stack_resources(self.cf_name) if ly: ly = {o.logical_id: o.physical_resource_id for o in ly} return ly def compile(self): """Loads all files associated to a layer.""" # Some templates may refer to params, so populate them first self.populate_params() # Load all files with layer contents for section in config.LAYER_SECTIONS: self.section[section] = self.loader.load_section( section, params=self.loader_params) # Package the layer as a CF template default_description = "{}-{} ({})".format(self.environment.name, self.name, self.environment.stage) description = self.params.get('description', {}).get('value') or \ self.environment.meta['description'] or \ self.meta.get('description') or \ default_description cf_template = { 'AWSTemplateFormatVersion': str(config.CF_TEMPLATE_VERSION), 'Description': description, 'Mappings': self.section.get('mappings', {}), 'Parameters': self.section.get('parameters', {}), 'Resources': self.section.get('resources', {}), 'Outputs': self.section.get('outputs', {}) } if self.section.get('transform', {}).get('value', {}): cf_template['Transform'] = self.section['transform']['value'] return cf_template def populate_params(self): """Populates parameters in a layer by resolving references.""" if len(self.yaml_params) < 1: return for pname, param in sorted(self.yaml_params.items(), key=lambda t: t[1].get('priority', '1')): self.params[pname] = {} self.params[pname]['description'] = param.get('description', None) try: self.params[pname]['value'] = self._parse_param_value( param['value']) except: self.logger.error("Error parsing layer '{}'".format(self.name)) raise def print_params(self): """Prints the params used during layer creation.""" if len(self.params) < 1: print("No parameters. Did you forget to run populate_params()?") return print("Parameters for layer {}:".format(self.cf_name)) for pname, param in self.params.items(): pval = param.get('value', None) if len(pval) > 30: pval = pval[0:30] print("{pname:<15}: {pval:>30}".format(pname=pname, pval=pval)) def _parse_param_value(self, pval): """Parses layer parameter values.""" if isinstance(pval, list): return [self._parse_param_value(_) for _ in pval] elif _is_reference(pval): return self._resolve_ref( list(pval.keys())[0][1:], list(pval.values())[0]) elif _is_legacy_reference(pval): return self._resolve_ref(pval['ref']['parser'], pval['ref'].get('parameters', {})) elif isinstance(pval, dict): return {k: self._parse_param_value(v) for k, v in pval.items()} else: return pval def _resolve_ref(self, parsername, parameters): """Resolves references.""" parser = config.reference_parsers.get(parsername) if not parser: msg = "Invalid reference parser '{}' in layer '{}'".format( parsername, self.cf_name) raise ReferenceError(ref, msg, logger=self.logger) result = parser(self, config.boto_config, **parameters) return result def delete(self): """Deletes a stack in CF.""" msg = "Deleting stack {} from CF".format(self.cf_name) self.logger.info(msg) self.cf.delete_stack(self.cf_name) def create(self, update=False, debug=False): """Deploys a layer as a CF stack.""" msg = "Starting checks for layer {}".format(self.name) self.logger.info(msg) cf_template = None # CAPABILITY_IAM is needed only for layers that contain certain # resources, but we add it always for simplicity. if not self.in_cf: self.logger.info("Creating layer '{}' (CF stack '{}')".format( self.name, self.cf_name)) cf_template = self.compile() try: self.create_with_changeset(cf_template) except Exception: self.logger.error("Error deploying stack '{}'".format( self.cf_name)) self.logger.error("Stack template: {}".format( json.dumps(cf_template, indent=4))) raise elif update: cf_template = self.compile() try: self.create_with_changeset(cf_template, update) except NoUpdatesError: msg = "Nothing to update on stack '{}'".format(self.cf_name) self.logger.warning(msg) except Exception: self.logger.error("Error deploying stack '{}'".format( self.cf_name)) self.logger.error("Stack template: {}".format( json.dumps(cf_template, indent=4))) raise else: msg = "Layer '{}' already in CF: not creating".format(self.name) self.logger.info(msg) if debug and cf_template: directory = os.path.join(self.env_basedir, "debug_output") if not os.path.exists(directory): os.makedirs(directory) with open(os.path.join(directory, self.name + ".yaml"), "w") as f: yaml.dump(cf_template, f, default_flow_style=False) return self.outputs def _upload_cf_template(self, cf_template): """Upload CF template to S3.""" bucket = config.boto_config.profile.get('bucket') key = "{}{}-{}.json".format(self.s3_prefix, round(time.time()), str(uuid.uuid4())) cf_template = json.dumps(cf_template).encode() S3().resource.Bucket(bucket).put_object(Key=key, Body=cf_template) return "https://s3-{}.amazonaws.com/{}/{}".format( config.boto_config.profile['aws_region'], bucket, key) def create_with_changeset(self, cf_template, update=False): """Use a changeset to create a stack.""" changeset_type = "CREATE" if update: changeset_type = "UPDATE" changeset_name = self.cf_name + str(uuid4()) template_url = self._upload_cf_template(cf_template) self.cf.client.create_change_set( StackName=self.cf_name, TemplateURL=template_url, Capabilities=["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"], NotificationARNs=self.sns_topic_arn, Tags=[{ "Key": k, "Value": v } for k, v in self.tags.items()], ChangeSetName=changeset_name, ChangeSetType=changeset_type) self.wait_for_status_change() self.wait_changeset_creation(changeset_name) if update: changeset = self.cf.client.describe_change_set( ChangeSetName=changeset_name, StackName=self.cf_name) if not changeset["Changes"]: raise NoUpdatesError("Nothing to update") self.cf.client.execute_change_set(ChangeSetName=changeset_name, StackName=self.cf_name) self.wait_for_status_change() @staticmethod def _is_bad_status(status): """True if a stack status is not healthy.""" return status is None \ or status not in {"CREATE_COMPLETE", "UPDATE_COMPLETE", "REVIEW_IN_PROGRESS", "UPDATE_ROLLBACK_COMPLETE"} def _print_events(self, already_seen=None): """Prints the events reported by AWS.""" if already_seen is None: already_seen = set() events = self.cf.get_stack_events(self.cf_name) new_events = [ev for ev in events if ev.id not in already_seen] cm = config.EVENT_STATUS_COLOR_MAP for event in new_events: self.logger.info("{color}{status}\033[0m {restype} {logid} " "{reason}".format( color=cm.get(event.resource_status, ''), status=event.resource_status, restype=event.resource_type, logid=event.logical_resource_id, reason=event.resource_status_reason or "", )) already_seen.add(event.id) return already_seen def wait_for_status_change(self): """Wait for the status deployment state to change.""" status, seen_events = self.watch_events() if self._is_bad_status(status): # One retry, also to flush all events status, seen_events = self.watch_events(already_seen=seen_events) if self._is_bad_status(status): # One retry msg = "Unable to deploy layer '{}': status is {}".format( self.name, status) raise CloudformationError(msg, logger=self.logger) return status def watch_events( self, progress_status={ 'CREATE_IN_PROGRESS', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS' }, already_seen=None): """Watches CF events during stack creation.""" stack_status = self.cf.get_stack_status(self.cf_name) if already_seen is None: already_seen = set() while (stack_status is None) or (stack_status in progress_status): already_seen = self._print_events(already_seen) time.sleep(5) stack_status = self.cf.get_stack_status(self.cf_name) return stack_status, already_seen def wait_changeset_creation( self, changeset_name, progress_status={"CREATE_PENDING", "CREATE_IN_PROGRESS"}): """Wait for a changeset to be in the right status to be executed.""" status = self.cf.client.describe_change_set( ChangeSetName=changeset_name, StackName=self.cf_name)["Status"] while (status is None) or (status in progress_status): status = self.cf.client.describe_change_set( ChangeSetName=changeset_name, StackName=self.cf_name)["Status"] time.sleep(5) if status != "CREATE_COMPLETE": msg = "Unable to deploy layer '{}': changeset status is {}".format( self.name, status) raise CloudformationError(msg, logger=self.logger) return status def __repr__(self): return str(self) def __str__(self): args = re.sub(r'\'(\w+)\'\s*:\s*', r'\1=', str(self.user_params))[1:-1] if len(args) > 0: basestr = "Layer({env}, '{name}', {args})" else: basestr = "Layer({env}, '{name}')" return basestr.format(env=self.__environment_repr, name=self.name, args=args)
def __init__(self, yml_path, logger=None, stage=None, vault_layer=None, parameters=None): if logger is None: self.logger = logging.getLogger(__name__) # To prevent warnings self.logger.addHandler(logging.NullHandler()) else: self.logger = logger self.__yml_path = yml_path self.stage = stage and stage.upper() basedir, envfile = os.path.split(yml_path) self.basedir = os.path.abspath(basedir) self._j2_env = j2.Environment(extensions=["jinja2.ext.with_"], loader=j2.FileSystemLoader(self.basedir)) # Add custom functions and filters utils.update_jinja2_env(self._j2_env) if parameters is None: parameters = {} if "_default" in parameters: def_params = parameters.get("_default", {}) del parameters["_default"] parameters.update(def_params) parameters.update(os.environ) with open(yml_path, 'r') as f: if os.path.splitext(yml_path)[1] == ".j2": template = self._j2_env.get_template(envfile) meta = yaml.load(template.render(stage=stage, **parameters)) else: meta = yaml.load(f) self.name = list(meta.keys())[0] self.meta = meta.get(self.name) if len(self.meta) == 0: raise FileFormatError(yml_path, "Error getting environment name ", logger=self.logger) self.cf = Cloudformation(config.boto_config) self.sns_topic_arn = self.meta.get('sns-topic-arn', []) self.tags = self.meta.get('tags', {}) self.tags['humilis:environment'] = self.name self.layers = [] for layer in self.meta.get('layers', []): layer_name = layer.get('layer', None) if layer_name is None: msg = "Wrongly formatted layer: {}".format(layer) raise FileFormatError(yml_path, msg) if layer.get('disable', False): message = ("Layer {} is disabled by configuration. " "Skipping.".format(layer.get('layer'))) self.logger.warning(message) continue # Get the layer params provided in the environment spec layer_params = {k: v for k, v in layer.items() if k != 'layer'} layer_obj = Layer(self, layer_name, **layer_params) self.layers.append(layer_obj) self.vault_layer = self.get_layer(vault_layer or 'secrets-vault') self.__secrets_table_name = "{}-{}-secrets".format( self.name, self.stage) if self.stage: self.__keychain_namespace = "{}:{}".format(self.name, self.stage.lower()) else: self.__keychain_namespace = self.name self.__dynamodb = None
class Environment(): """Manages the deployment of a collection of humilis layers.""" def __init__(self, yml_path, logger=None, stage=None, vault_layer=None, parameters=None): if logger is None: self.logger = logging.getLogger(__name__) # To prevent warnings self.logger.addHandler(logging.NullHandler()) else: self.logger = logger self.__yml_path = yml_path self.stage = stage and stage.upper() basedir, envfile = os.path.split(yml_path) self.basedir = os.path.abspath(basedir) self._j2_env = j2.Environment(extensions=["jinja2.ext.with_"], loader=j2.FileSystemLoader(self.basedir)) # Add custom functions and filters utils.update_jinja2_env(self._j2_env) if parameters is None: parameters = {} if "_default" in parameters: def_params = parameters.get("_default", {}) del parameters["_default"] parameters.update(def_params) parameters.update(os.environ) with open(yml_path, 'r') as f: if os.path.splitext(yml_path)[1] == ".j2": template = self._j2_env.get_template(envfile) meta = yaml.load(template.render(stage=stage, **parameters)) else: meta = yaml.load(f) self.name = list(meta.keys())[0] self.meta = meta.get(self.name) if len(self.meta) == 0: raise FileFormatError(yml_path, "Error getting environment name ", logger=self.logger) self.cf = Cloudformation(config.boto_config) self.sns_topic_arn = self.meta.get('sns-topic-arn', []) self.tags = self.meta.get('tags', {}) self.tags['humilis:environment'] = self.name self.layers = [] for layer in self.meta.get('layers', []): layer_name = layer.get('layer', None) if layer_name is None: msg = "Wrongly formatted layer: {}".format(layer) raise FileFormatError(yml_path, msg) if layer.get('disable', False): message = ("Layer {} is disabled by configuration. " "Skipping.".format(layer.get('layer'))) self.logger.warning(message) continue # Get the layer params provided in the environment spec layer_params = {k: v for k, v in layer.items() if k != 'layer'} layer_obj = Layer(self, layer_name, **layer_params) self.layers.append(layer_obj) self.vault_layer = self.get_layer(vault_layer or 'secrets-vault') self.__secrets_table_name = "{}-{}-secrets".format( self.name, self.stage) if self.stage: self.__keychain_namespace = "{}:{}".format(self.name, self.stage.lower()) else: self.__keychain_namespace = self.name self.__dynamodb = None @property def outputs(self): """Outputs produced by each environment layer.""" outputs = {} self.cf.flush_cache() for layer in self.layers: try: ly = layer.outputs except CloudformationError: self.logger.error("Could not retrieve outputs for layer" " '{}'".format(layer.name)) ly = None if ly is not None: outputs[layer.name] = ly return outputs @property def resources(self): """Resources produced by each environment layer.""" resources = {} for layer in self.layers: try: ly = layer.resources except CloudformationError: self.logger.error("Could not retrieve resources for layer" " '{}'".format(layer.name)) ly = None if ly is not None: resources[layer.name] = ly return resources @property def kms_key_id(self): """The ID of the KMS Key associated to the environment vault.""" if not self.vault_layer: raise RequiresVaultError("Requires a secrets-vault layer") if self.vault_layer: return self.outputs[self.vault_layer.name]['KmsKeyId'] @property def dynamodb(self): """Connection to AWS DynamoDB.""" if self.__dynamodb is None: self.__dynamodb = Dynamodb(config.boto_config) return self.__dynamodb def set_secret(self, key, plaintext): """Sets and environment secret.""" if not self.vault_layer: msg = "No secrets-vault layer in this environment" self.logger.error(msg) raise RequiresVaultError(msg) else: client = Kms(config.boto_config).client encrypted = client.encrypt(KeyId=self.kms_key_id, Plaintext=plaintext)['CiphertextBlob'] resp = self.dynamodb.client.put_item( TableName=self.__secrets_table_name, Item={ 'id': { 'S': key }, 'value': { 'B': encrypted } }) return resp def get_secret(self, key): """Retrieves a secret.""" if not self.vault_layer: msg = "No secrets-vault layer in this environment" self.logger.error(msg) raise RequiresVaultError(msg) else: client = Dynamodb(config.boto_config).client encrypted = client.get_item(TableName=self.__secrets_table_name, Key={'id': { 'S': key }})['Item']['value']['B'] # Decrypt using KMS (assuming the secret value is a string) client = boto3.client('kms') plaintext = client.decrypt(CiphertextBlob=encrypted)['Plaintext'] return plaintext.decode() def delete_secret(self, key): """Deletes a secret.""" if not self.vault_layer: msg = "No secrets-vault layer in this environment" self.logger.error(msg) raise RequiresVaultError(msg) else: client = Dynamodb(config.boto_config).client resp = client.delete_item(TableName=self.__secrets_table_name, Key={'id': { 'S': key }})['Item']['value']['B'] return resp def create(self, output_file=None, update=False): """Creates or updates an environment.""" for layer in self.layers: layer.create(update=update) self.logger.info({"outputs": self.outputs}) if output_file is not None: self.write_outputs(output_file) def write_outputs(self, output_file=None): """Writes layer outputs to a YAML file.""" if output_file is None: output_file = "{environment}-{stage}.outputs.yaml" output_file = output_file.format(environment=self.name, stage=self.stage) with open(output_file, "w") as f: f.write(yaml.dump(self.outputs, indent=4, default_flow_style=False)) def get_layer(self, layer_name): """Gets a layer by name""" sel_layer = [ layer for layer in self.layers if layer.cf_name == layer_name or layer.name == layer_name ] if len(sel_layer) > 0: return sel_layer[0] def delete(self): """Deletes the complete environment from CF.""" for layer in reversed(self.layers): if layer.termination_protection: self.logger.warning( "Layer '%s' has termination protection set: " "will not be deleted", layer.name) else: layer.delete() @property def in_cf(self): """Returns true if the environment has been deployed to CF.""" return self.name in { utils.unroll_tags(stk['Tags']).get('humilis:environment') for stk in self.cf.stacks } def __repr__(self): return str(self) def __str__(self): return "Environment('{}')".format(self.__yml_path)
def cf(): """Create a Cloudformation facade object""" yield Cloudformation(config.boto_config)
def __init__(self, yml_path, logger=None, stage=None, vault_layer=None, parameters=None): if logger is None: self.logger = logging.getLogger(__name__) # To prevent warnings self.logger.addHandler(logging.NullHandler()) else: self.logger = logger if stage is None: raise ValueError("stage can't be None") self.__yml_path = yml_path self.stage = stage and stage.upper() basedir, envfile = os.path.split(yml_path) self.basedir = os.path.abspath(basedir) self._j2_env = j2.Environment(loader=j2.FileSystemLoader(self.basedir)) # Add custom functions and filters utils.update_jinja2_env(self._j2_env) parameters = self._preprocess_parameters(parameters) with open(yml_path, 'r') as f: if os.path.splitext(yml_path)[1] == ".j2": template = self._j2_env.get_template(envfile) meta = yaml.load( template.render( stage=stage, # Backwards compatibility __context={ 'stage': stage, 'aws': { 'account_id': boto3.client('sts').get_caller_identity().get( 'Account') } }, __env=os.environ, **parameters), Loader=yaml.FullLoader) else: meta = yaml.load(f, Loader=yaml.FullLoader) self.name = list(meta.keys())[0] self.meta = meta.get(self.name) if len(self.meta) == 0: raise FileFormatError(yml_path, "Error getting environment name ", logger=self.logger) self.cf = Cloudformation(config.boto_config) self.sns_topic_arn = self.meta.get('sns-topic-arn', []) self.tags = self.meta.get('tags', {}) self.tags['humilis:environment'] = self.name self.layers = [] for layer in self.meta.get('layers', []): layer_name = layer.get('layer', None) if layer_name is None: msg = "Wrongly formatted layer: {}".format(layer) raise FileFormatError(yml_path, msg) if layer.get('disable', False): message = ("Layer {} is disabled by configuration. " "Skipping.".format(layer.get('layer'))) self.logger.warning(message) continue # Get the layer params provided in the environment spec layer_params = {k: v for k, v in layer.items() if k != 'layer'} layer_obj = Layer(self, layer_name, **layer_params) self.layers.append(layer_obj) self.vault_layer = self.get_layer(vault_layer or 'secrets-vault') self.__secrets_table_name = "{}-{}-secrets".format( self.name, self.stage) self.__keychain_namespace = "{}:{}".format(self.name, self.stage.lower()) self.__dynamodb = None