def output(layer, config, layer_name=None, output_name=None,
environment_name=None, stage=None):
"""Gets the value of an output produced by an already deployed layer.
:param layer: The Layer object for the layer declaring the reference.
:param config: An object holding humilis configuration options.
:param layer_name: The logical name of the layer that produced the output.
:param output_name: The logical name of the output parameter.
"""
if not environment_name or not stage:
environment_name = layer.env_name
stage = layer.env_stage
stack_name = utils.get_cf_name(environment_name, layer_name, stage=stage)
cf = Cloudformation(config)
try:
output = cf.get_stack_output(stack_name, output_name)
except AttributeError:
msg = "No output '{}' in CF stack '{}'".format(output_name, stack_name)
ref = "output/{}/{}/{}/{}".format(environment_name, layer_name, stage,
output_name)
raise ReferenceError(ref, msg)
if len(output) < 1:
all_stack_outputs = [x['OutputKey'] for x
in cf.stack_outputs[stack_name]]
msg = ("{} output does not exist for stack {} "
"(with outputs {}).").format(output_name,
stack_name,
all_stack_outputs)
ref = "output ({}/{})".format(layer_name, output_name)
raise ReferenceError(ref, msg, logger=layer.logger)
return output[0]
def _get_stack_resource(layer, config, stack_name, resource_name):
"""Gets the physical ID of a resource in a CF Stack.
:param stack_name: The name of the CF stack.
:param resource_name: The logical name of the CF resource.
:returns: The physical ID of the resource.
"""
cf = Cloudformation(config)
resource = cf.get_stack_resource(stack_name, resource_name)
if len(resource) < 1:
all_stack_resources = [x.logical_resource_id for x
in cf.get_stack_resources(stack_name)]
msg = "{} does not exist in stack {} (with resources {}).".format(
resource_name, stack_name, all_stack_resources)
raise ReferenceError(resource_name, msg, logger=layer.logger)
return resource[0].physical_resource_id
def _get_stack_resource(config, stack_name, resource_name):
"""Gets the physical ID of a resource in a CF Stack.
:param stack_name: The name of the CF stack.
:param resource_name: The logical name of the CF resource.
:returns: The physical ID of the resource.
"""
cf = Cloudformation(config)
resource = cf.get_stack_resource(stack_name, resource_name)
if len(resource) < 1:
all_stack_resources = [x.logical_resource_id for x
in cf.get_stack_resources(stack_name)]
msg = "{} does not exist in stack {} (with resources {}).".format(
resource_name, stack_name, all_stack_resources)
raise ReferenceError(resource_name, msg, logger=layer.logger)
return resource[0].physical_resource_id
def output(layer, config, layer_name=None, output_name=None):
"""Gets the value of an output produced by an already deployed layer.
:param layer: The Layer object for the layer declaring the reference.
:param config: An object holding humilis configuration options.
:param layer_name: The logical name of the layer that produced the output.
:param output_name: The logical name of the output parameter.
"""
stack_name = utils.get_cf_name(layer.env_name, layer_name,
stage=layer.env_stage)
cf = Cloudformation(config)
output = cf.get_stack_output(stack_name, output_name)
if len(output) < 1:
all_stack_outputs = [x['OutputKey'] for x
in cf.stack_outputs[stack_name]]
msg = ("{} output does not exist for stack {} "
"(with outputs {}).").format(output_name,
stack_name,
all_stack_outputs)
ref = "output ({}/{})".format(layer_name, output_name)
raise ReferenceError(ref, msg, logger=layer.logger)
return output[0]
def __init__(self,
__env,
__name,
layer_type=None,
logger=None,
loader=None,
humilis_profile=None,
**user_params):
self.__environment_repr = repr(__env)
self.environment = __env
if not humilis_profile:
self.cf = self.environment.cf
else:
config.boto_config.activate_profile(humilis_profile)
self.cf = Cloudformation(config.boto_config)
if logger is None:
self.logger = logging.getLogger(__name__)
# To prevent warnings
self.logger.addHandler(logging.NullHandler())
else:
self.logger = logger
self.name = __name
self.env_name = self.environment.name
self.env_stage = self.environment.stage
self.env_basedir = self.environment.basedir
self.depends_on = []
self.section = {}
self.type = layer_type
self.s3_prefix = "{base}{env}/{stage}/{layer}/".format(
base=config.boto_config.profile.get("s3prefix"),
env=self.environment.name,
stage=self.environment.stage,
layer=__name)
if layer_type is not None:
basedir = config.layers.get(layer_type)
if not basedir:
msg = ("The plugin providing the layer type '{}' is not "
"installed in this system. Please install it and "
"try again.").format(layer_type)
raise MissingPluginError(msg)
else:
basedir = None
if basedir is None:
basedir = os.path.join(self.env_basedir, 'layers', self.name)
self.basedir = basedir
if loader is None:
loader = DirTreeBackedObject(basedir, self.logger)
self.loader = loader
# These param set will be sent to the template compiler and will be
# populated once the layers this layer depend on have been created.
self.params = {}
# the parameters that will be used to compile meta.yaml
self.meta = {}
meta_params = {
p[0]: p[1]
for p in itertools.chain(self.loader_params.items(),
user_params.items())
}
self.meta = self.loader.load_section('meta', params=meta_params)
self.sns_topic_arn = self.environment.sns_topic_arn
self.tags = {
'humilis:environment': self.env_name,
'humilis:layer': self.name,
'humilis:stage': self.env_stage,
'humilis:created': str(datetime.datetime.now())
}
for tagname, tagvalue in self.environment.tags.items():
self.tags[tagname] = tagvalue
for tagname, tagvalue in self.meta.get('tags', {}).items():
self.tags[tagname] = tagvalue
self.yaml_params = self.meta.get('parameters', {})
for k, v in self.yaml_params.items():
# Set 1 as default priority for all parameters
v['priority'] = v.get('priority', 1)
# User params override what is in the layer definition file
self.user_params = user_params
for pname, pvalue in user_params.items():
if pname in self.yaml_params:
self.yaml_params[pname]['value'] = pvalue
self.__ec2 = None
self.__s3 = None
class Layer:
"""A layer of infrastructure that translates into a single CF stack"""
def __init__(self,
__env,
__name,
layer_type=None,
logger=None,
loader=None,
humilis_profile=None,
**user_params):
self.__environment_repr = repr(__env)
self.environment = __env
if not humilis_profile:
self.cf = self.environment.cf
else:
config.boto_config.activate_profile(humilis_profile)
self.cf = Cloudformation(config.boto_config)
if logger is None:
self.logger = logging.getLogger(__name__)
# To prevent warnings
self.logger.addHandler(logging.NullHandler())
else:
self.logger = logger
self.name = __name
self.env_name = self.environment.name
self.env_stage = self.environment.stage
self.env_basedir = self.environment.basedir
self.depends_on = []
self.section = {}
self.type = layer_type
self.s3_prefix = "{base}{env}/{stage}/{layer}/".format(
base=config.boto_config.profile.get("s3prefix"),
env=self.environment.name,
stage=self.environment.stage,
layer=__name)
if layer_type is not None:
basedir = config.layers.get(layer_type)
if not basedir:
msg = ("The plugin providing the layer type '{}' is not "
"installed in this system. Please install it and "
"try again.").format(layer_type)
raise MissingPluginError(msg)
else:
basedir = None
if basedir is None:
basedir = os.path.join(self.env_basedir, 'layers', self.name)
self.basedir = basedir
if loader is None:
loader = DirTreeBackedObject(basedir, self.logger)
self.loader = loader
# These param set will be sent to the template compiler and will be
# populated once the layers this layer depend on have been created.
self.params = {}
# the parameters that will be used to compile meta.yaml
self.meta = {}
meta_params = {
p[0]: p[1]
for p in itertools.chain(self.loader_params.items(),
user_params.items())
}
self.meta = self.loader.load_section('meta', params=meta_params)
self.sns_topic_arn = self.environment.sns_topic_arn
self.tags = {
'humilis:environment': self.env_name,
'humilis:layer': self.name,
'humilis:stage': self.env_stage,
'humilis:created': str(datetime.datetime.now())
}
for tagname, tagvalue in self.environment.tags.items():
self.tags[tagname] = tagvalue
for tagname, tagvalue in self.meta.get('tags', {}).items():
self.tags[tagname] = tagvalue
self.yaml_params = self.meta.get('parameters', {})
for k, v in self.yaml_params.items():
# Set 1 as default priority for all parameters
v['priority'] = v.get('priority', 1)
# User params override what is in the layer definition file
self.user_params = user_params
for pname, pvalue in user_params.items():
if pname in self.yaml_params:
self.yaml_params[pname]['value'] = pvalue
self.__ec2 = None
self.__s3 = None
@property
def termination_protection(self):
"""Is termination protection set for this layer?."""
return self.meta.get('parameters', {}).get('termination_protection',
{}).get('value', False)
@property
def cf_name(self):
"""The name of the CF stack associated to this layer."""
return get_cf_name(self.env_name, self.name, stage=self.env_stage)
@property
def loader_params(self):
"""Produces a dictionary of parameters to pass to a section loader."""
# User parameters in the layer meta.yaml
# Not that some param values may not have been populated when this
# property is accessed since that may happen during the parsing of
# some references in the parameter list: thus the if 'value' in v
# For backwards compatibility, to be deprecated
params = {
k: v['value']
for k, v in self.params.items() if 'value' in v
}
params["__vars"] = dict(params)
# For backwards compatibility, to be deprecated
params['_env'] = {
'stage': self.env_stage,
'name': self.env_name,
'basedir': self.env_basedir
}
params['_os_env'] = os.environ
params['_layer'] = {'name': self.name}
params['env'] = os.environ
# The new format:
params['__env'] = os.environ
params['__context'] = {
'environment': {
'name': self.env_name,
'basedir': self.env_basedir,
'tags': self.environment.tags
},
'stage': self.env_stage,
'layer': {
'name': self.name,
'basedir': self.basedir
},
'aws': {
'account_id':
boto3.client('sts').get_caller_identity().get('Account')
}
}
# For backwards compatibility
params["context"] = params["__context"]
return params
@property
def in_cf(self):
"""Returns true if the layer has been already deployed to CF."""
return self.cf_name in {stk['StackName'] for stk in self.cf.stacks}
@property
def ec2(self):
"""Connection to AWS EC2 service."""
if self.__ec2 is None:
self.__ec2 = Ec2(config.boto_config)
return self.__ec2
@property
def s3(self):
"""Connection to AWS S3."""
if self.__s3 is None:
self.__s3 = S3(config.boto_config)
return self.__s3
@property
def ok(self):
"""Layer is fully deployed in CF and ready for use"""
return self.cf.stack_ok(self.cf_name)
@property
def outputs(self):
"""Layer CF outputs."""
ly = self.cf.stack_outputs.get(self.cf_name)
if ly:
ly = {o['OutputKey']: o['OutputValue'] for o in ly}
return ly
@property
def resources(self):
"""Layer CF resources."""
ly = self.cf.get_stack_resources(self.cf_name)
if ly:
ly = {o.logical_id: o.physical_resource_id for o in ly}
return ly
def compile(self):
"""Loads all files associated to a layer."""
# Some templates may refer to params, so populate them first
self.populate_params()
# Load all files with layer contents
for section in config.LAYER_SECTIONS:
self.section[section] = self.loader.load_section(
section, params=self.loader_params)
# Package the layer as a CF template
default_description = "{}-{} ({})".format(self.environment.name,
self.name,
self.environment.stage)
description = self.params.get('description', {}).get('value') or \
self.environment.meta['description'] or \
self.meta.get('description') or \
default_description
cf_template = {
'AWSTemplateFormatVersion': str(config.CF_TEMPLATE_VERSION),
'Description': description,
'Mappings': self.section.get('mappings', {}),
'Parameters': self.section.get('parameters', {}),
'Resources': self.section.get('resources', {}),
'Outputs': self.section.get('outputs', {})
}
if self.section.get('transform', {}).get('value', {}):
cf_template['Transform'] = self.section['transform']['value']
return cf_template
def populate_params(self):
"""Populates parameters in a layer by resolving references."""
if len(self.yaml_params) < 1:
return
for pname, param in sorted(self.yaml_params.items(),
key=lambda t: t[1].get('priority', '1')):
self.params[pname] = {}
self.params[pname]['description'] = param.get('description', None)
try:
self.params[pname]['value'] = self._parse_param_value(
param['value'])
except:
self.logger.error("Error parsing layer '{}'".format(self.name))
raise
def print_params(self):
"""Prints the params used during layer creation."""
if len(self.params) < 1:
print("No parameters. Did you forget to run populate_params()?")
return
print("Parameters for layer {}:".format(self.cf_name))
for pname, param in self.params.items():
pval = param.get('value', None)
if len(pval) > 30:
pval = pval[0:30]
print("{pname:<15}: {pval:>30}".format(pname=pname, pval=pval))
def _parse_param_value(self, pval):
"""Parses layer parameter values."""
if isinstance(pval, list):
return [self._parse_param_value(_) for _ in pval]
elif _is_reference(pval):
return self._resolve_ref(
list(pval.keys())[0][1:],
list(pval.values())[0])
elif _is_legacy_reference(pval):
return self._resolve_ref(pval['ref']['parser'],
pval['ref'].get('parameters', {}))
elif isinstance(pval, dict):
return {k: self._parse_param_value(v) for k, v in pval.items()}
else:
return pval
def _resolve_ref(self, parsername, parameters):
"""Resolves references."""
parser = config.reference_parsers.get(parsername)
if not parser:
msg = "Invalid reference parser '{}' in layer '{}'".format(
parsername, self.cf_name)
raise ReferenceError(ref, msg, logger=self.logger)
result = parser(self, config.boto_config, **parameters)
return result
def delete(self):
"""Deletes a stack in CF."""
msg = "Deleting stack {} from CF".format(self.cf_name)
self.logger.info(msg)
self.cf.delete_stack(self.cf_name)
def create(self, update=False, debug=False):
"""Deploys a layer as a CF stack."""
msg = "Starting checks for layer {}".format(self.name)
self.logger.info(msg)
cf_template = None
# CAPABILITY_IAM is needed only for layers that contain certain
# resources, but we add it always for simplicity.
if not self.in_cf:
self.logger.info("Creating layer '{}' (CF stack '{}')".format(
self.name, self.cf_name))
cf_template = self.compile()
try:
self.create_with_changeset(cf_template)
except Exception:
self.logger.error("Error deploying stack '{}'".format(
self.cf_name))
self.logger.error("Stack template: {}".format(
json.dumps(cf_template, indent=4)))
raise
elif update:
cf_template = self.compile()
try:
self.create_with_changeset(cf_template, update)
except NoUpdatesError:
msg = "Nothing to update on stack '{}'".format(self.cf_name)
self.logger.warning(msg)
except Exception:
self.logger.error("Error deploying stack '{}'".format(
self.cf_name))
self.logger.error("Stack template: {}".format(
json.dumps(cf_template, indent=4)))
raise
else:
msg = "Layer '{}' already in CF: not creating".format(self.name)
self.logger.info(msg)
if debug and cf_template:
directory = os.path.join(self.env_basedir, "debug_output")
if not os.path.exists(directory):
os.makedirs(directory)
with open(os.path.join(directory, self.name + ".yaml"), "w") as f:
yaml.dump(cf_template, f, default_flow_style=False)
return self.outputs
def _upload_cf_template(self, cf_template):
"""Upload CF template to S3."""
bucket = config.boto_config.profile.get('bucket')
key = "{}{}-{}.json".format(self.s3_prefix, round(time.time()),
str(uuid.uuid4()))
cf_template = json.dumps(cf_template).encode()
S3().resource.Bucket(bucket).put_object(Key=key, Body=cf_template)
return "https://s3-{}.amazonaws.com/{}/{}".format(
config.boto_config.profile['aws_region'], bucket, key)
def create_with_changeset(self, cf_template, update=False):
"""Use a changeset to create a stack."""
changeset_type = "CREATE"
if update:
changeset_type = "UPDATE"
changeset_name = self.cf_name + str(uuid4())
template_url = self._upload_cf_template(cf_template)
self.cf.client.create_change_set(
StackName=self.cf_name,
TemplateURL=template_url,
Capabilities=["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"],
NotificationARNs=self.sns_topic_arn,
Tags=[{
"Key": k,
"Value": v
} for k, v in self.tags.items()],
ChangeSetName=changeset_name,
ChangeSetType=changeset_type)
self.wait_for_status_change()
self.wait_changeset_creation(changeset_name)
if update:
changeset = self.cf.client.describe_change_set(
ChangeSetName=changeset_name, StackName=self.cf_name)
if not changeset["Changes"]:
raise NoUpdatesError("Nothing to update")
self.cf.client.execute_change_set(ChangeSetName=changeset_name,
StackName=self.cf_name)
self.wait_for_status_change()
@staticmethod
def _is_bad_status(status):
"""True if a stack status is not healthy."""
return status is None \
or status not in {"CREATE_COMPLETE", "UPDATE_COMPLETE",
"REVIEW_IN_PROGRESS",
"UPDATE_ROLLBACK_COMPLETE"}
def _print_events(self, already_seen=None):
"""Prints the events reported by AWS."""
if already_seen is None:
already_seen = set()
events = self.cf.get_stack_events(self.cf_name)
new_events = [ev for ev in events if ev.id not in already_seen]
cm = config.EVENT_STATUS_COLOR_MAP
for event in new_events:
self.logger.info("{color}{status}\033[0m {restype} {logid} "
"{reason}".format(
color=cm.get(event.resource_status, ''),
status=event.resource_status,
restype=event.resource_type,
logid=event.logical_resource_id,
reason=event.resource_status_reason or "",
))
already_seen.add(event.id)
return already_seen
def wait_for_status_change(self):
"""Wait for the status deployment state to change."""
status, seen_events = self.watch_events()
if self._is_bad_status(status):
# One retry, also to flush all events
status, seen_events = self.watch_events(already_seen=seen_events)
if self._is_bad_status(status):
# One retry
msg = "Unable to deploy layer '{}': status is {}".format(
self.name, status)
raise CloudformationError(msg, logger=self.logger)
return status
def watch_events(
self,
progress_status={
'CREATE_IN_PROGRESS', 'UPDATE_IN_PROGRESS',
'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'
},
already_seen=None):
"""Watches CF events during stack creation."""
stack_status = self.cf.get_stack_status(self.cf_name)
if already_seen is None:
already_seen = set()
while (stack_status is None) or (stack_status in progress_status):
already_seen = self._print_events(already_seen)
time.sleep(5)
stack_status = self.cf.get_stack_status(self.cf_name)
return stack_status, already_seen
def wait_changeset_creation(
self,
changeset_name,
progress_status={"CREATE_PENDING", "CREATE_IN_PROGRESS"}):
"""Wait for a changeset to be in the right status to be executed."""
status = self.cf.client.describe_change_set(
ChangeSetName=changeset_name, StackName=self.cf_name)["Status"]
while (status is None) or (status in progress_status):
status = self.cf.client.describe_change_set(
ChangeSetName=changeset_name, StackName=self.cf_name)["Status"]
time.sleep(5)
if status != "CREATE_COMPLETE":
msg = "Unable to deploy layer '{}': changeset status is {}".format(
self.name, status)
raise CloudformationError(msg, logger=self.logger)
return status
def __repr__(self):
return str(self)
def __str__(self):
args = re.sub(r'\'(\w+)\'\s*:\s*', r'\1=', str(self.user_params))[1:-1]
if len(args) > 0:
basestr = "Layer({env}, '{name}', {args})"
else:
basestr = "Layer({env}, '{name}')"
return basestr.format(env=self.__environment_repr,
name=self.name,
args=args)
def __init__(self,
yml_path,
logger=None,
stage=None,
vault_layer=None,
parameters=None):
if logger is None:
self.logger = logging.getLogger(__name__)
# To prevent warnings
self.logger.addHandler(logging.NullHandler())
else:
self.logger = logger
self.__yml_path = yml_path
self.stage = stage and stage.upper()
basedir, envfile = os.path.split(yml_path)
self.basedir = os.path.abspath(basedir)
self._j2_env = j2.Environment(extensions=["jinja2.ext.with_"],
loader=j2.FileSystemLoader(self.basedir))
# Add custom functions and filters
utils.update_jinja2_env(self._j2_env)
if parameters is None:
parameters = {}
if "_default" in parameters:
def_params = parameters.get("_default", {})
del parameters["_default"]
parameters.update(def_params)
parameters.update(os.environ)
with open(yml_path, 'r') as f:
if os.path.splitext(yml_path)[1] == ".j2":
template = self._j2_env.get_template(envfile)
meta = yaml.load(template.render(stage=stage, **parameters))
else:
meta = yaml.load(f)
self.name = list(meta.keys())[0]
self.meta = meta.get(self.name)
if len(self.meta) == 0:
raise FileFormatError(yml_path,
"Error getting environment name ",
logger=self.logger)
self.cf = Cloudformation(config.boto_config)
self.sns_topic_arn = self.meta.get('sns-topic-arn', [])
self.tags = self.meta.get('tags', {})
self.tags['humilis:environment'] = self.name
self.layers = []
for layer in self.meta.get('layers', []):
layer_name = layer.get('layer', None)
if layer_name is None:
msg = "Wrongly formatted layer: {}".format(layer)
raise FileFormatError(yml_path, msg)
if layer.get('disable', False):
message = ("Layer {} is disabled by configuration. "
"Skipping.".format(layer.get('layer')))
self.logger.warning(message)
continue
# Get the layer params provided in the environment spec
layer_params = {k: v for k, v in layer.items() if k != 'layer'}
layer_obj = Layer(self, layer_name, **layer_params)
self.layers.append(layer_obj)
self.vault_layer = self.get_layer(vault_layer or 'secrets-vault')
self.__secrets_table_name = "{}-{}-secrets".format(
self.name, self.stage)
if self.stage:
self.__keychain_namespace = "{}:{}".format(self.name,
self.stage.lower())
else:
self.__keychain_namespace = self.name
self.__dynamodb = None
class Environment():
"""Manages the deployment of a collection of humilis layers."""
def __init__(self,
yml_path,
logger=None,
stage=None,
vault_layer=None,
parameters=None):
if logger is None:
self.logger = logging.getLogger(__name__)
# To prevent warnings
self.logger.addHandler(logging.NullHandler())
else:
self.logger = logger
self.__yml_path = yml_path
self.stage = stage and stage.upper()
basedir, envfile = os.path.split(yml_path)
self.basedir = os.path.abspath(basedir)
self._j2_env = j2.Environment(extensions=["jinja2.ext.with_"],
loader=j2.FileSystemLoader(self.basedir))
# Add custom functions and filters
utils.update_jinja2_env(self._j2_env)
if parameters is None:
parameters = {}
if "_default" in parameters:
def_params = parameters.get("_default", {})
del parameters["_default"]
parameters.update(def_params)
parameters.update(os.environ)
with open(yml_path, 'r') as f:
if os.path.splitext(yml_path)[1] == ".j2":
template = self._j2_env.get_template(envfile)
meta = yaml.load(template.render(stage=stage, **parameters))
else:
meta = yaml.load(f)
self.name = list(meta.keys())[0]
self.meta = meta.get(self.name)
if len(self.meta) == 0:
raise FileFormatError(yml_path,
"Error getting environment name ",
logger=self.logger)
self.cf = Cloudformation(config.boto_config)
self.sns_topic_arn = self.meta.get('sns-topic-arn', [])
self.tags = self.meta.get('tags', {})
self.tags['humilis:environment'] = self.name
self.layers = []
for layer in self.meta.get('layers', []):
layer_name = layer.get('layer', None)
if layer_name is None:
msg = "Wrongly formatted layer: {}".format(layer)
raise FileFormatError(yml_path, msg)
if layer.get('disable', False):
message = ("Layer {} is disabled by configuration. "
"Skipping.".format(layer.get('layer')))
self.logger.warning(message)
continue
# Get the layer params provided in the environment spec
layer_params = {k: v for k, v in layer.items() if k != 'layer'}
layer_obj = Layer(self, layer_name, **layer_params)
self.layers.append(layer_obj)
self.vault_layer = self.get_layer(vault_layer or 'secrets-vault')
self.__secrets_table_name = "{}-{}-secrets".format(
self.name, self.stage)
if self.stage:
self.__keychain_namespace = "{}:{}".format(self.name,
self.stage.lower())
else:
self.__keychain_namespace = self.name
self.__dynamodb = None
@property
def outputs(self):
"""Outputs produced by each environment layer."""
outputs = {}
self.cf.flush_cache()
for layer in self.layers:
try:
ly = layer.outputs
except CloudformationError:
self.logger.error("Could not retrieve outputs for layer"
" '{}'".format(layer.name))
ly = None
if ly is not None:
outputs[layer.name] = ly
return outputs
@property
def resources(self):
"""Resources produced by each environment layer."""
resources = {}
for layer in self.layers:
try:
ly = layer.resources
except CloudformationError:
self.logger.error("Could not retrieve resources for layer"
" '{}'".format(layer.name))
ly = None
if ly is not None:
resources[layer.name] = ly
return resources
@property
def kms_key_id(self):
"""The ID of the KMS Key associated to the environment vault."""
if not self.vault_layer:
raise RequiresVaultError("Requires a secrets-vault layer")
if self.vault_layer:
return self.outputs[self.vault_layer.name]['KmsKeyId']
@property
def dynamodb(self):
"""Connection to AWS DynamoDB."""
if self.__dynamodb is None:
self.__dynamodb = Dynamodb(config.boto_config)
return self.__dynamodb
def set_secret(self, key, plaintext):
"""Sets and environment secret."""
if not self.vault_layer:
msg = "No secrets-vault layer in this environment"
self.logger.error(msg)
raise RequiresVaultError(msg)
else:
client = Kms(config.boto_config).client
encrypted = client.encrypt(KeyId=self.kms_key_id,
Plaintext=plaintext)['CiphertextBlob']
resp = self.dynamodb.client.put_item(
TableName=self.__secrets_table_name,
Item={
'id': {
'S': key
},
'value': {
'B': encrypted
}
})
return resp
def get_secret(self, key):
"""Retrieves a secret."""
if not self.vault_layer:
msg = "No secrets-vault layer in this environment"
self.logger.error(msg)
raise RequiresVaultError(msg)
else:
client = Dynamodb(config.boto_config).client
encrypted = client.get_item(TableName=self.__secrets_table_name,
Key={'id': {
'S': key
}})['Item']['value']['B']
# Decrypt using KMS (assuming the secret value is a string)
client = boto3.client('kms')
plaintext = client.decrypt(CiphertextBlob=encrypted)['Plaintext']
return plaintext.decode()
def delete_secret(self, key):
"""Deletes a secret."""
if not self.vault_layer:
msg = "No secrets-vault layer in this environment"
self.logger.error(msg)
raise RequiresVaultError(msg)
else:
client = Dynamodb(config.boto_config).client
resp = client.delete_item(TableName=self.__secrets_table_name,
Key={'id': {
'S': key
}})['Item']['value']['B']
return resp
def create(self, output_file=None, update=False):
"""Creates or updates an environment."""
for layer in self.layers:
layer.create(update=update)
self.logger.info({"outputs": self.outputs})
if output_file is not None:
self.write_outputs(output_file)
def write_outputs(self, output_file=None):
"""Writes layer outputs to a YAML file."""
if output_file is None:
output_file = "{environment}-{stage}.outputs.yaml"
output_file = output_file.format(environment=self.name,
stage=self.stage)
with open(output_file, "w") as f:
f.write(yaml.dump(self.outputs, indent=4,
default_flow_style=False))
def get_layer(self, layer_name):
"""Gets a layer by name"""
sel_layer = [
layer for layer in self.layers
if layer.cf_name == layer_name or layer.name == layer_name
]
if len(sel_layer) > 0:
return sel_layer[0]
def delete(self):
"""Deletes the complete environment from CF."""
for layer in reversed(self.layers):
if layer.termination_protection:
self.logger.warning(
"Layer '%s' has termination protection set: "
"will not be deleted", layer.name)
else:
layer.delete()
@property
def in_cf(self):
"""Returns true if the environment has been deployed to CF."""
return self.name in {
utils.unroll_tags(stk['Tags']).get('humilis:environment')
for stk in self.cf.stacks
}
def __repr__(self):
return str(self)
def __str__(self):
return "Environment('{}')".format(self.__yml_path)
def cf():
"""Create a Cloudformation facade object"""
yield Cloudformation(config.boto_config)
def __init__(self,
yml_path,
logger=None,
stage=None,
vault_layer=None,
parameters=None):
if logger is None:
self.logger = logging.getLogger(__name__)
# To prevent warnings
self.logger.addHandler(logging.NullHandler())
else:
self.logger = logger
if stage is None:
raise ValueError("stage can't be None")
self.__yml_path = yml_path
self.stage = stage and stage.upper()
basedir, envfile = os.path.split(yml_path)
self.basedir = os.path.abspath(basedir)
self._j2_env = j2.Environment(loader=j2.FileSystemLoader(self.basedir))
# Add custom functions and filters
utils.update_jinja2_env(self._j2_env)
parameters = self._preprocess_parameters(parameters)
with open(yml_path, 'r') as f:
if os.path.splitext(yml_path)[1] == ".j2":
template = self._j2_env.get_template(envfile)
meta = yaml.load(
template.render(
stage=stage, # Backwards compatibility
__context={
'stage': stage,
'aws': {
'account_id':
boto3.client('sts').get_caller_identity().get(
'Account')
}
},
__env=os.environ,
**parameters),
Loader=yaml.FullLoader)
else:
meta = yaml.load(f, Loader=yaml.FullLoader)
self.name = list(meta.keys())[0]
self.meta = meta.get(self.name)
if len(self.meta) == 0:
raise FileFormatError(yml_path,
"Error getting environment name ",
logger=self.logger)
self.cf = Cloudformation(config.boto_config)
self.sns_topic_arn = self.meta.get('sns-topic-arn', [])
self.tags = self.meta.get('tags', {})
self.tags['humilis:environment'] = self.name
self.layers = []
for layer in self.meta.get('layers', []):
layer_name = layer.get('layer', None)
if layer_name is None:
msg = "Wrongly formatted layer: {}".format(layer)
raise FileFormatError(yml_path, msg)
if layer.get('disable', False):
message = ("Layer {} is disabled by configuration. "
"Skipping.".format(layer.get('layer')))
self.logger.warning(message)
continue
# Get the layer params provided in the environment spec
layer_params = {k: v for k, v in layer.items() if k != 'layer'}
layer_obj = Layer(self, layer_name, **layer_params)
self.layers.append(layer_obj)
self.vault_layer = self.get_layer(vault_layer or 'secrets-vault')
self.__secrets_table_name = "{}-{}-secrets".format(
self.name, self.stage)
self.__keychain_namespace = "{}:{}".format(self.name,
self.stage.lower())
self.__dynamodb = None