def construct_stacks(self):
"""
Traverses the files under the command path.
For each file encountered, a Stack is constructed
using the correct config. Dependencies are traversed
and a final set of Stacks is returned.
:returns: A set of Stacks.
:rtype: set
"""
stack_map = {}
command_stacks = set()
if self.context.ignore_dependencies:
root = self.context.full_command_path()
else:
root = self.context.full_config_path()
if path.isfile(root):
todo = {root}
else:
todo = set()
for directory_name, sub_directories, files in walk(root, followlinks=True):
for filename in fnmatch.filter(files, '*.yaml'):
if filename.startswith('config.'):
continue
todo.add(path.join(directory_name, filename))
stack_group_configs = {}
while todo:
abs_path = todo.pop()
rel_path = path.relpath(
abs_path, start=self.context.full_config_path())
directory, filename = path.split(rel_path)
if directory in stack_group_configs:
stack_group_config = stack_group_configs[directory]
else:
stack_group_config = stack_group_configs[directory] = \
self.read(path.join(directory, self.context.config_file))
stack = self._construct_stack(rel_path, stack_group_config)
stack_map[sceptreise_path(rel_path)] = stack
if abs_path.startswith(self.context.full_command_path()):
command_stacks.add(stack)
stacks = set()
for stack in stack_map.values():
if not self.context.ignore_dependencies:
stack.dependencies = [
stack_map[sceptreise_path(dep)]
for dep in stack.dependencies
]
else:
stack.dependencies = []
stacks.add(stack)
return stacks, command_stacks
def _construct_stack(self, rel_path, stack_group_config=None):
"""
Constructs an individual Stack object from a config path and a
base config.
:param rel_path: A relative config file path.
:type rel_path: str
:param stack_group_config: The Stack group config to use as defaults.
:type stack_group_config: dict
:returns: Stack object
:rtype: sceptre.stack.Stack
"""
directory, filename = path.split(rel_path)
if filename == self.context.config_file:
pass
self.templating_vars["stack_group_config"] = stack_group_config
parsed_stack_group_config = self._parsed_stack_group_config(
stack_group_config)
config = self.read(rel_path, stack_group_config)
stack_name = path.splitext(rel_path)[0]
# Check for missing mandatory attributes
for required_key in REQUIRED_KEYS:
if required_key not in config:
raise InvalidConfigFileError(
"Required attribute '{0}' not found in configuration of '{1}'."
.format(required_key, stack_name))
abs_template_path = path.join(self.context.project_path,
self.context.templates_path,
sceptreise_path(config["template_path"]))
s3_details = self._collect_s3_details(stack_name, config)
stack = Stack(name=stack_name,
project_code=config["project_code"],
template_path=abs_template_path,
region=config["region"],
template_bucket_name=config.get("template_bucket_name"),
template_key_prefix=config.get("template_key_prefix"),
required_version=config.get("required_version"),
iam_role=config.get("iam_role"),
profile=config.get("profile"),
parameters=config.get("parameters", {}),
sceptre_user_data=config.get("sceptre_user_data", {}),
hooks=config.get("hooks", {}),
s3_details=s3_details,
dependencies=config.get("dependencies", []),
role_arn=config.get("role_arn"),
protected=config.get("protect", False),
tags=config.get("stack_tags", {}),
external_name=config.get("stack_name"),
notifications=config.get("notifications"),
on_failure=config.get("on_failure"),
stack_timeout=config.get("stack_timeout", 0),
stack_group_config=parsed_stack_group_config)
del self.templating_vars["stack_group_config"]
return stack
def _collect_s3_details(stack_name, config):
"""
Collects and constructs details for where to store the Template in S3.
:param stack_name: Stack name.
:type stack_name: str
:param config: Config with details.
:type config: dict
:returns: S3 details.
:rtype: dict
"""
s3_details = None
if "template_bucket_name" in config:
template_key = "/".join([
sceptreise_path(stack_name),
"{time_stamp}.json".format(time_stamp=datetime.datetime.utcnow(
).strftime("%Y-%m-%d-%H-%M-%S-%fZ"))
])
bucket_region = config.get("region", None)
if "template_key_prefix" in config:
prefix = config["template_key_prefix"]
template_key = "/".join([prefix.strip("/"), template_key])
s3_details = {
"bucket_name": config["template_bucket_name"],
"bucket_key": template_key,
"bucket_region": bucket_region
}
return s3_details
def resolve_stacks(self, stack_map):
"""
Transforms map of Stacks into a set of Stacks, transforms dependencies
from a list of Strings (stack names) to a list of Stacks.
:param stack_map: Map of stacks, containing dependencies as list of Strings.
:type base_config: dict
:returns: Set of stacks, containing dependencies as list of Stacks.
:rtype: set
:raises: sceptre.exceptions.DependencyDoesNotExistError
"""
stacks = set()
for stack in stack_map.values():
if not self.context.ignore_dependencies:
for i, dep in enumerate(stack.dependencies):
try:
stack.dependencies[i] = stack_map[sceptreise_path(dep)]
except KeyError:
raise DependencyDoesNotExistError(
"{stackname}: Dependency {dep} not found. "
"Valid dependency names are: "
"{stackkeys}. "
"Please make sure that your dependencies stack_outputs "
"have their full path from `config` defined.".
format(stackname=stack.name,
dep=dep,
stackkeys=", ".join(stack_map.keys())))
else:
stack.dependencies = []
stacks.add(stack)
return stacks
def _collect_s3_details(stack_name, config):
"""
Collects and constructs details for where to store the Template in S3.
:param stack_name: Stack name.
:type stack_name: str
:param config: Config with details.
:type config: dict
:returns: S3 details.
:rtype: dict
"""
s3_details = None
# If the config explicitly sets the template_bucket_name to None, we don't want to enter
# this conditional block.
if config.get("template_bucket_name") is not None:
template_key = "/".join([
sceptreise_path(stack_name),
"{time_stamp}.json".format(time_stamp=datetime.datetime.utcnow(
).strftime("%Y-%m-%d-%H-%M-%S-%fZ"))
])
if "template_key_prefix" in config:
prefix = config["template_key_prefix"]
template_key = "/".join([prefix.strip("/"), template_key])
s3_details = {
"bucket_name": config["template_bucket_name"],
"bucket_key": template_key
}
return s3_details
def _valid_stack_paths(self):
return [
sceptreise_path(
path.relpath(path.join(dirpath, f), self.context.config_path))
for dirpath, dirnames, files in walk(self.context.config_path)
for f in files if not f.endswith(self.context.config_file)
]
def _generate_launch_order(self, reverse=False):
if self.context.ignore_dependencies:
return [self.command_stacks]
graph = self.graph.filtered(self.command_stacks, reverse)
if self.context.ignore_dependencies:
return [self.command_stacks]
launch_order = []
while graph.graph:
batch = set()
for stack in graph:
if graph.count_dependencies(stack) == 0:
batch.add(stack)
launch_order.append(batch)
for stack in batch:
graph.remove_stack(stack)
if not launch_order:
raise ConfigFileNotFoundError(
"No stacks detected from the given path '{}'. Valid stack paths are: {}"
.format(sceptreise_path(self.context.command_path),
self._valid_stack_paths()))
return launch_order
def __init__(
self, name, project_code, template_path, region, template_bucket_name=None,
template_key_prefix=None, required_version=None, parameters=None,
sceptre_user_data=None, hooks=None, s3_details=None,
dependencies=None, role_arn=None, protected=False, tags=None,
external_name=None, notifications=None, on_failure=None, profile=None,
stack_timeout=0, stack_group_config={}
):
self.logger = logging.getLogger(__name__)
self.name = sceptreise_path(name)
self.project_code = project_code
self.region = region
self.template_bucket_name = template_bucket_name
self.template_key_prefix = template_key_prefix
self.required_version = required_version
self.external_name = external_name or get_external_stack_name(self.project_code, self.name)
self.template_path = template_path
self.s3_details = s3_details
self._template = None
self.protected = protected
self.role_arn = role_arn
self.on_failure = on_failure
self.dependencies = dependencies or []
self.tags = tags or {}
self.stack_timeout = stack_timeout
self.profile = profile
self.hooks = hooks or {}
self.parameters = parameters or {}
self.sceptre_user_data = sceptre_user_data or {}
self.notifications = notifications or []
self.stack_group_config = stack_group_config or {}
def setup(self):
"""
Adds dependency to a Stack.
"""
dep_stack_name, self.output_key = self.argument.split("::")
self.dependency_stack_name = sceptreise_path(
normalise_path(dep_stack_name))
self.stack.dependencies.append(self.dependency_stack_name)
def get_stack_names(context, stack_group_name):
config_dir = Path(context.sceptre_dir) / 'config'
path = config_dir / stack_group_name
stack_names = []
for child in path.rglob('*'):
if child.is_dir() or child.stem == 'config':
continue
relative_path = child.relative_to(config_dir)
stack_name = sceptreise_path(
str(relative_path).replace(child.suffix, ''))
stack_names.append(stack_name)
return stack_names
def __init__(
self, name, project_code, region, template_path=None, template_handler_config=None,
template_bucket_name=None, template_key_prefix=None, required_version=None,
parameters=None, sceptre_user_data=None, hooks=None, s3_details=None,
iam_role=None, dependencies=None, role_arn=None, protected=False, tags=None,
external_name=None, notifications=None, on_failure=None, profile=None,
stack_timeout=0, stack_group_config={}
):
self.logger = logging.getLogger(__name__)
if template_path and template_handler_config:
raise InvalidConfigFileError("Both 'template_path' and 'template' are set, specify one or the other")
if not template_path and not template_handler_config:
raise InvalidConfigFileError("Neither 'template_path' nor 'template' is set")
self.name = sceptreise_path(name)
self.project_code = project_code
self.region = region
self.template_bucket_name = template_bucket_name
self.template_key_prefix = template_key_prefix
self.required_version = required_version
self.external_name = external_name or get_external_stack_name(self.project_code, self.name)
self.template_path = template_path
self.template_handler_config = template_handler_config
self.s3_details = s3_details
self._template = None
self._connection_manager = None
self.protected = protected
self.role_arn = role_arn
self.on_failure = on_failure
self.dependencies = dependencies or []
self.tags = tags or {}
self.stack_timeout = stack_timeout
self.iam_role = iam_role
self.profile = profile
self.hooks = hooks or {}
self.parameters = parameters or {}
self._sceptre_user_data = sceptre_user_data or {}
self._sceptre_user_data_is_resolved = False
self.notifications = notifications or []
self.stack_group_config = stack_group_config or {}
def construct_stacks(self) -> Set[Stack]:
"""
Traverses the files under the command path.
For each file encountered, a Stack is constructed
using the correct config. Dependencies are traversed
and a final set of Stacks is returned.
:returns: A set of Stacks.
"""
stack_map = {}
command_stacks = set()
root = self.context.full_command_path()
if self.context.full_scan:
root = self.context.full_config_path()
if path.isfile(root):
todo = {root}
else:
todo = set()
for directory_name, sub_directories, files in walk(
root, followlinks=True):
for filename in fnmatch.filter(files, '*.yaml'):
if filename.startswith('config.'):
continue
todo.add(path.join(directory_name, filename))
stack_group_configs = {}
full_todo = todo.copy()
deps_todo = set()
while todo:
abs_path = todo.pop()
rel_path = path.relpath(abs_path,
start=self.context.full_config_path())
directory, filename = path.split(rel_path)
if directory in stack_group_configs:
stack_group_config = stack_group_configs[directory]
else:
stack_group_config = stack_group_configs[directory] = \
self.read(path.join(directory, self.context.config_file))
stack = self._construct_stack(rel_path, stack_group_config)
for dep in stack.dependencies:
full_dep = str(Path(self.context.full_config_path(), dep))
if not path.exists(full_dep):
raise DependencyDoesNotExistError(
"{stackname}: Dependency {dep} not found. "
"Please make sure that your dependencies stack_outputs "
"have their full path from `config` defined.".format(
stackname=stack.name, dep=dep))
if full_dep not in full_todo and full_dep not in deps_todo:
todo.add(full_dep)
deps_todo.add(full_dep)
stack_map[sceptreise_path(rel_path)] = stack
full_command_path = self.context.full_command_path()
if abs_path == full_command_path\
or abs_path.startswith(full_command_path.rstrip(path.sep) + path.sep):
command_stacks.add(stack)
stacks = self.resolve_stacks(stack_map)
return stacks, command_stacks
def test_sceptreise_path_with_trailing_backslash(self):
with pytest.raises(PathConversionError):
sceptreise_path(
'this\path\is\invalid\\'
)
def test_sceptreise_path_with_trailing_slash(self):
with pytest.raises(PathConversionError):
sceptreise_path(
"this/path/is/invalid/"
)
def test_sceptreise_path_with_windows_path(self):
windows_path = 'dev\\app\\stack'
assert sceptreise_path(windows_path) == 'dev/app/stack'
def test_sceptreise_path_with_valid_path(self):
path = 'dev/app/stack'
assert sceptreise_path(path) == path
