def _get_stack_resource(layer, config, stack_name, resource_name):
"""Gets the physical ID of a resource in a CF Stack.
:param stack_name: The name of the CF stack.
:param resource_name: The logical name of the CF resource.
:returns: The physical ID of the resource.
"""
cf = Cloudformation(config)
resource = cf.get_stack_resource(stack_name, resource_name)
if len(resource) < 1:
all_stack_resources = [x.logical_resource_id for x
in cf.get_stack_resources(stack_name)]
msg = "{} does not exist in stack {} (with resources {}).".format(
resource_name, stack_name, all_stack_resources)
raise ReferenceError(resource_name, msg, logger=layer.logger)
return resource[0].physical_resource_id
def _get_stack_resource(config, stack_name, resource_name):
"""Gets the physical ID of a resource in a CF Stack.
:param stack_name: The name of the CF stack.
:param resource_name: The logical name of the CF resource.
:returns: The physical ID of the resource.
"""
cf = Cloudformation(config)
resource = cf.get_stack_resource(stack_name, resource_name)
if len(resource) < 1:
all_stack_resources = [x.logical_resource_id for x
in cf.get_stack_resources(stack_name)]
msg = "{} does not exist in stack {} (with resources {}).".format(
resource_name, stack_name, all_stack_resources)
raise ReferenceError(resource_name, msg, logger=layer.logger)
return resource[0].physical_resource_id
class Layer:
"""A layer of infrastructure that translates into a single CF stack"""
def __init__(self,
__env,
__name,
layer_type=None,
logger=None,
loader=None,
humilis_profile=None,
**user_params):
self.__environment_repr = repr(__env)
self.environment = __env
if not humilis_profile:
self.cf = self.environment.cf
else:
config.boto_config.activate_profile(humilis_profile)
self.cf = Cloudformation(config.boto_config)
if logger is None:
self.logger = logging.getLogger(__name__)
# To prevent warnings
self.logger.addHandler(logging.NullHandler())
else:
self.logger = logger
self.name = __name
self.env_name = self.environment.name
self.env_stage = self.environment.stage
self.env_basedir = self.environment.basedir
self.depends_on = []
self.section = {}
self.type = layer_type
self.s3_prefix = "{base}{env}/{stage}/{layer}/".format(
base=config.boto_config.profile.get("s3prefix"),
env=self.environment.name,
stage=self.environment.stage,
layer=__name)
if layer_type is not None:
basedir = config.layers.get(layer_type)
if not basedir:
msg = ("The plugin providing the layer type '{}' is not "
"installed in this system. Please install it and "
"try again.").format(layer_type)
raise MissingPluginError(msg)
else:
basedir = None
if basedir is None:
basedir = os.path.join(self.env_basedir, 'layers', self.name)
self.basedir = basedir
if loader is None:
loader = DirTreeBackedObject(basedir, self.logger)
self.loader = loader
# These param set will be sent to the template compiler and will be
# populated once the layers this layer depend on have been created.
self.params = {}
# the parameters that will be used to compile meta.yaml
self.meta = {}
meta_params = {
p[0]: p[1]
for p in itertools.chain(self.loader_params.items(),
user_params.items())
}
self.meta = self.loader.load_section('meta', params=meta_params)
self.sns_topic_arn = self.environment.sns_topic_arn
self.tags = {
'humilis:environment': self.env_name,
'humilis:layer': self.name,
'humilis:stage': self.env_stage,
'humilis:created': str(datetime.datetime.now())
}
for tagname, tagvalue in self.environment.tags.items():
self.tags[tagname] = tagvalue
for tagname, tagvalue in self.meta.get('tags', {}).items():
self.tags[tagname] = tagvalue
self.yaml_params = self.meta.get('parameters', {})
for k, v in self.yaml_params.items():
# Set 1 as default priority for all parameters
v['priority'] = v.get('priority', 1)
# User params override what is in the layer definition file
self.user_params = user_params
for pname, pvalue in user_params.items():
if pname in self.yaml_params:
self.yaml_params[pname]['value'] = pvalue
self.__ec2 = None
self.__s3 = None
@property
def termination_protection(self):
"""Is termination protection set for this layer?."""
return self.meta.get('parameters', {}).get('termination_protection',
{}).get('value', False)
@property
def cf_name(self):
"""The name of the CF stack associated to this layer."""
return get_cf_name(self.env_name, self.name, stage=self.env_stage)
@property
def loader_params(self):
"""Produces a dictionary of parameters to pass to a section loader."""
# User parameters in the layer meta.yaml
# Not that some param values may not have been populated when this
# property is accessed since that may happen during the parsing of
# some references in the parameter list: thus the if 'value' in v
# For backwards compatibility, to be deprecated
params = {
k: v['value']
for k, v in self.params.items() if 'value' in v
}
params["__vars"] = dict(params)
# For backwards compatibility, to be deprecated
params['_env'] = {
'stage': self.env_stage,
'name': self.env_name,
'basedir': self.env_basedir
}
params['_os_env'] = os.environ
params['_layer'] = {'name': self.name}
params['env'] = os.environ
# The new format:
params['__env'] = os.environ
params['__context'] = {
'environment': {
'name': self.env_name,
'basedir': self.env_basedir,
'tags': self.environment.tags
},
'stage': self.env_stage,
'layer': {
'name': self.name,
'basedir': self.basedir
},
'aws': {
'account_id':
boto3.client('sts').get_caller_identity().get('Account')
}
}
# For backwards compatibility
params["context"] = params["__context"]
return params
@property
def in_cf(self):
"""Returns true if the layer has been already deployed to CF."""
return self.cf_name in {stk['StackName'] for stk in self.cf.stacks}
@property
def ec2(self):
"""Connection to AWS EC2 service."""
if self.__ec2 is None:
self.__ec2 = Ec2(config.boto_config)
return self.__ec2
@property
def s3(self):
"""Connection to AWS S3."""
if self.__s3 is None:
self.__s3 = S3(config.boto_config)
return self.__s3
@property
def ok(self):
"""Layer is fully deployed in CF and ready for use"""
return self.cf.stack_ok(self.cf_name)
@property
def outputs(self):
"""Layer CF outputs."""
ly = self.cf.stack_outputs.get(self.cf_name)
if ly:
ly = {o['OutputKey']: o['OutputValue'] for o in ly}
return ly
@property
def resources(self):
"""Layer CF resources."""
ly = self.cf.get_stack_resources(self.cf_name)
if ly:
ly = {o.logical_id: o.physical_resource_id for o in ly}
return ly
def compile(self):
"""Loads all files associated to a layer."""
# Some templates may refer to params, so populate them first
self.populate_params()
# Load all files with layer contents
for section in config.LAYER_SECTIONS:
self.section[section] = self.loader.load_section(
section, params=self.loader_params)
# Package the layer as a CF template
default_description = "{}-{} ({})".format(self.environment.name,
self.name,
self.environment.stage)
description = self.params.get('description', {}).get('value') or \
self.environment.meta['description'] or \
self.meta.get('description') or \
default_description
cf_template = {
'AWSTemplateFormatVersion': str(config.CF_TEMPLATE_VERSION),
'Description': description,
'Mappings': self.section.get('mappings', {}),
'Parameters': self.section.get('parameters', {}),
'Resources': self.section.get('resources', {}),
'Outputs': self.section.get('outputs', {})
}
if self.section.get('transform', {}).get('value', {}):
cf_template['Transform'] = self.section['transform']['value']
return cf_template
def populate_params(self):
"""Populates parameters in a layer by resolving references."""
if len(self.yaml_params) < 1:
return
for pname, param in sorted(self.yaml_params.items(),
key=lambda t: t[1].get('priority', '1')):
self.params[pname] = {}
self.params[pname]['description'] = param.get('description', None)
try:
self.params[pname]['value'] = self._parse_param_value(
param['value'])
except:
self.logger.error("Error parsing layer '{}'".format(self.name))
raise
def print_params(self):
"""Prints the params used during layer creation."""
if len(self.params) < 1:
print("No parameters. Did you forget to run populate_params()?")
return
print("Parameters for layer {}:".format(self.cf_name))
for pname, param in self.params.items():
pval = param.get('value', None)
if len(pval) > 30:
pval = pval[0:30]
print("{pname:<15}: {pval:>30}".format(pname=pname, pval=pval))
def _parse_param_value(self, pval):
"""Parses layer parameter values."""
if isinstance(pval, list):
return [self._parse_param_value(_) for _ in pval]
elif _is_reference(pval):
return self._resolve_ref(
list(pval.keys())[0][1:],
list(pval.values())[0])
elif _is_legacy_reference(pval):
return self._resolve_ref(pval['ref']['parser'],
pval['ref'].get('parameters', {}))
elif isinstance(pval, dict):
return {k: self._parse_param_value(v) for k, v in pval.items()}
else:
return pval
def _resolve_ref(self, parsername, parameters):
"""Resolves references."""
parser = config.reference_parsers.get(parsername)
if not parser:
msg = "Invalid reference parser '{}' in layer '{}'".format(
parsername, self.cf_name)
raise ReferenceError(ref, msg, logger=self.logger)
result = parser(self, config.boto_config, **parameters)
return result
def delete(self):
"""Deletes a stack in CF."""
msg = "Deleting stack {} from CF".format(self.cf_name)
self.logger.info(msg)
self.cf.delete_stack(self.cf_name)
def create(self, update=False, debug=False):
"""Deploys a layer as a CF stack."""
msg = "Starting checks for layer {}".format(self.name)
self.logger.info(msg)
cf_template = None
# CAPABILITY_IAM is needed only for layers that contain certain
# resources, but we add it always for simplicity.
if not self.in_cf:
self.logger.info("Creating layer '{}' (CF stack '{}')".format(
self.name, self.cf_name))
cf_template = self.compile()
try:
self.create_with_changeset(cf_template)
except Exception:
self.logger.error("Error deploying stack '{}'".format(
self.cf_name))
self.logger.error("Stack template: {}".format(
json.dumps(cf_template, indent=4)))
raise
elif update:
cf_template = self.compile()
try:
self.create_with_changeset(cf_template, update)
except NoUpdatesError:
msg = "Nothing to update on stack '{}'".format(self.cf_name)
self.logger.warning(msg)
except Exception:
self.logger.error("Error deploying stack '{}'".format(
self.cf_name))
self.logger.error("Stack template: {}".format(
json.dumps(cf_template, indent=4)))
raise
else:
msg = "Layer '{}' already in CF: not creating".format(self.name)
self.logger.info(msg)
if debug and cf_template:
directory = os.path.join(self.env_basedir, "debug_output")
if not os.path.exists(directory):
os.makedirs(directory)
with open(os.path.join(directory, self.name + ".yaml"), "w") as f:
yaml.dump(cf_template, f, default_flow_style=False)
return self.outputs
def _upload_cf_template(self, cf_template):
"""Upload CF template to S3."""
bucket = config.boto_config.profile.get('bucket')
key = "{}{}-{}.json".format(self.s3_prefix, round(time.time()),
str(uuid.uuid4()))
cf_template = json.dumps(cf_template).encode()
S3().resource.Bucket(bucket).put_object(Key=key, Body=cf_template)
return "https://s3-{}.amazonaws.com/{}/{}".format(
config.boto_config.profile['aws_region'], bucket, key)
def create_with_changeset(self, cf_template, update=False):
"""Use a changeset to create a stack."""
changeset_type = "CREATE"
if update:
changeset_type = "UPDATE"
changeset_name = self.cf_name + str(uuid4())
template_url = self._upload_cf_template(cf_template)
self.cf.client.create_change_set(
StackName=self.cf_name,
TemplateURL=template_url,
Capabilities=["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"],
NotificationARNs=self.sns_topic_arn,
Tags=[{
"Key": k,
"Value": v
} for k, v in self.tags.items()],
ChangeSetName=changeset_name,
ChangeSetType=changeset_type)
self.wait_for_status_change()
self.wait_changeset_creation(changeset_name)
if update:
changeset = self.cf.client.describe_change_set(
ChangeSetName=changeset_name, StackName=self.cf_name)
if not changeset["Changes"]:
raise NoUpdatesError("Nothing to update")
self.cf.client.execute_change_set(ChangeSetName=changeset_name,
StackName=self.cf_name)
self.wait_for_status_change()
@staticmethod
def _is_bad_status(status):
"""True if a stack status is not healthy."""
return status is None \
or status not in {"CREATE_COMPLETE", "UPDATE_COMPLETE",
"REVIEW_IN_PROGRESS",
"UPDATE_ROLLBACK_COMPLETE"}
def _print_events(self, already_seen=None):
"""Prints the events reported by AWS."""
if already_seen is None:
already_seen = set()
events = self.cf.get_stack_events(self.cf_name)
new_events = [ev for ev in events if ev.id not in already_seen]
cm = config.EVENT_STATUS_COLOR_MAP
for event in new_events:
self.logger.info("{color}{status}\033[0m {restype} {logid} "
"{reason}".format(
color=cm.get(event.resource_status, ''),
status=event.resource_status,
restype=event.resource_type,
logid=event.logical_resource_id,
reason=event.resource_status_reason or "",
))
already_seen.add(event.id)
return already_seen
def wait_for_status_change(self):
"""Wait for the status deployment state to change."""
status, seen_events = self.watch_events()
if self._is_bad_status(status):
# One retry, also to flush all events
status, seen_events = self.watch_events(already_seen=seen_events)
if self._is_bad_status(status):
# One retry
msg = "Unable to deploy layer '{}': status is {}".format(
self.name, status)
raise CloudformationError(msg, logger=self.logger)
return status
def watch_events(
self,
progress_status={
'CREATE_IN_PROGRESS', 'UPDATE_IN_PROGRESS',
'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'
},
already_seen=None):
"""Watches CF events during stack creation."""
stack_status = self.cf.get_stack_status(self.cf_name)
if already_seen is None:
already_seen = set()
while (stack_status is None) or (stack_status in progress_status):
already_seen = self._print_events(already_seen)
time.sleep(5)
stack_status = self.cf.get_stack_status(self.cf_name)
return stack_status, already_seen
def wait_changeset_creation(
self,
changeset_name,
progress_status={"CREATE_PENDING", "CREATE_IN_PROGRESS"}):
"""Wait for a changeset to be in the right status to be executed."""
status = self.cf.client.describe_change_set(
ChangeSetName=changeset_name, StackName=self.cf_name)["Status"]
while (status is None) or (status in progress_status):
status = self.cf.client.describe_change_set(
ChangeSetName=changeset_name, StackName=self.cf_name)["Status"]
time.sleep(5)
if status != "CREATE_COMPLETE":
msg = "Unable to deploy layer '{}': changeset status is {}".format(
self.name, status)
raise CloudformationError(msg, logger=self.logger)
return status
def __repr__(self):
return str(self)
def __str__(self):
args = re.sub(r'\'(\w+)\'\s*:\s*', r'\1=', str(self.user_params))[1:-1]
if len(args) > 0:
basestr = "Layer({env}, '{name}', {args})"
else:
basestr = "Layer({env}, '{name}')"
return basestr.format(env=self.__environment_repr,
name=self.name,
args=args)
