def validate_attributes(cls, bundle, item_id, attributes):
if not attributes.get('delete', False):
if attributes.get('password') is None and attributes.get(
'password_hash') is None:
raise BundleError(
_("expected either 'password' or 'password_hash' on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
if attributes.get('password') is not None and attributes.get(
'password_hash') is not None:
raise BundleError(
_("can't define both 'password' and 'password_hash' on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
if not isinstance(attributes.get('delete', True), bool):
raise BundleError(
_("expected boolean for 'delete' on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
def validate_attributes(cls, bundle, item_id, attributes):
if attributes.get('delete', False):
for attr in attributes.keys():
if attr not in ['delete'] + list(BUILTIN_ITEM_ATTRIBUTES.keys()):
raise BundleError(_(
"{item} from bundle '{bundle}' cannot have other "
"attributes besides 'delete'"
).format(item=item_id, bundle=bundle.name))
if 'content' in attributes and 'source' in attributes:
raise BundleError(_(
"{item} from bundle '{bundle}' cannot have both 'content' and 'source'"
).format(item=item_id, bundle=bundle.name))
if (
attributes.get('content_type', None) == "any" and (
'content' in attributes or
'encoding' in attributes or
'source' in attributes
)
):
raise BundleError(_(
"{item} from bundle '{bundle}' with content_type 'any' "
"must not define 'content', 'encoding' and/or 'source'"
).format(item=item_id, bundle=bundle.name))
for key, value in attributes.items():
ATTRIBUTE_VALIDATORS[key](item_id, value)
def ask(self, status):
if not status.info["exists"] and not self.attributes["delete"]:
return _("Doesn't exist. Do you want to create it?")
if status.info["exists"] and self.attributes["delete"]:
return red(_("Will be deleted."))
if status.info["owner"] != self.attributes["owner"]:
return "{} {} → {}".format(bold(_("owner")), status.info["owner"], self.attributes["owner"])
def validate_name(cls, bundle, name):
for char in name:
if char not in _USERNAME_VALID_CHARACTERS:
raise BundleError(_(
"Invalid character in group name '{name}': {char} (bundle '{bundle}')"
).format(
char=char,
bundle=bundle.name,
name=name,
))
if name.endswith("_") or name.endswith("-"):
raise BundleError(_(
"Group name '{name}' must not end in dash or underscore (bundle '{bundle}')"
).format(
bundle=bundle.name,
name=name,
))
if len(name) > 30:
raise BundleError(_(
"Group name '{name}' is longer than 30 characters (bundle '{bundle}')"
).format(
bundle=bundle.name,
name=name,
))
def sdict(self):
version, flavor = pkg_installed(self.node, self.name)
return {
'installed': bool(version),
'flavor': flavor if flavor is not None else _("none"),
'version': version if version else _("none"),
}
def sdict(self):
version, flavor = pkg_installed(self.node, self.name)
return {
'installed': bool(version),
'flavor': flavor if flavor is not None else _("none"),
'version': version if version else _("none"),
}
def git_command(cmdline, repo_dir):
"""
Runs the given git command line in the given directory.
Returns stdout of the command.
"""
cmdline = ["git"] + cmdline
io.debug(_("running '{}' in {}").format(
" ".join(cmdline),
repo_dir,
))
git_process = Popen(
cmdline,
cwd=repo_dir,
preexec_fn=setpgrp,
stderr=PIPE,
stdout=PIPE,
)
stdout, stderr = git_process.communicate()
# FIXME integrate this into Item._command_results
if git_process.returncode != 0:
io.stderr(_("failed command: {}").format(" ".join(cmdline)))
io.stderr(_("stdout:\n{}").format(stdout))
io.stderr(_("stderr:\n{}").format(stderr))
raise RuntimeError(
_("`git {command}` failed in {dir}").format(
command=cmdline[1],
dir=repo_dir,
))
return stdout.decode('utf-8').strip()
def _skip_with_soft_locks(self, mine, others):
"""
Returns True/False depending on whether the item should be
skipped based on the given set of locks.
"""
for lock in mine:
for selector in lock['items']:
if self.covered_by_autoskip_selector(selector):
io.debug(
_("{item} on {node} whitelisted by lock {lock}").
format(
item=self.id,
lock=lock['id'],
node=self.node.name,
))
return False
for lock in others:
for selector in lock['items']:
if self.covered_by_autoskip_selector(selector):
io.debug(
_("{item} on {node} blacklisted by lock {lock}").
format(
item=self.id,
lock=lock['id'],
node=self.node.name,
))
return True
return False
def _validate_attribute_names(cls, bundle, item_id, attributes):
if not isinstance(attributes, dict):
raise BundleError(
_("invalid item '{item}' in bundle '{bundle}': not a dict").
format(
item=item_id,
bundle=bundle.name,
))
invalid_attributes = set(attributes.keys()).difference(
set(cls.ITEM_ATTRIBUTES.keys()).union(
set(BUILTIN_ITEM_ATTRIBUTES.keys())), )
if invalid_attributes:
raise BundleError(
_("invalid attribute(s) for '{item}' in bundle '{bundle}': {attrs}"
).format(
item=item_id,
bundle=bundle.name,
attrs=", ".join(invalid_attributes),
))
invalid_attributes = set(attributes.get(
'when_creating',
{}).keys()).difference(set(cls.WHEN_CREATING_ATTRIBUTES.keys()))
if invalid_attributes:
raise BundleError(
_("invalid when_creating attribute(s) for '{item}' in bundle '{bundle}': {attrs}"
).format(
item=item_id,
bundle=bundle.name,
attrs=", ".join(invalid_attributes),
))
def get_auto_deps(self, items):
deps = []
for item in items:
if item.ITEM_TYPE_NAME == "file" and is_subdirectory(item.name, self.name):
raise BundleError(
_("{item1} (from bundle '{bundle1}') blocking path to " "{item2} (from bundle '{bundle2}')").format(
item1=item.id, bundle1=item.bundle.name, item2=self.id, bundle2=self.bundle.name
)
)
elif item.ITEM_TYPE_NAME == "user" and item.name == self.attributes["owner"]:
if item.attributes["delete"]:
raise BundleError(
_(
"{item1} (from bundle '{bundle1}') depends on item "
"{item2} (from bundle '{bundle2}') which is set to be deleted"
).format(item1=self.id, bundle1=self.bundle.name, item2=item.id, bundle2=item.bundle.name)
)
else:
deps.append(item.id)
elif item.ITEM_TYPE_NAME == "group" and item.name == self.attributes["group"]:
if item.attributes["delete"]:
raise BundleError(
_(
"{item1} (from bundle '{bundle1}') depends on item "
"{item2} (from bundle '{bundle2}') which is set to be deleted"
).format(item1=self.id, bundle1=self.bundle.name, item2=item.id, bundle2=item.bundle.name)
)
else:
deps.append(item.id)
elif item.ITEM_TYPE_NAME in ("directory", "symlink"):
if is_subdirectory(item.name, self.name):
deps.append(item.id)
return deps
def fix(self, status):
for fix_type in ('type', 'content', 'mode', 'owner', 'group'):
if fix_type in status.info['needs_fixing']:
if fix_type == 'group' and \
'owner' in status.info['needs_fixing']:
# owner and group are fixed with a single chown
continue
if fix_type in ('mode', 'owner', 'group') and \
'content' in status.info['needs_fixing']:
# fixing content implies settings mode and owner/group
continue
if status.info['path_info'].exists:
if self.attributes['delete']:
LOG.info(_("{node}:{bundle}:{item}: deleting...").format(
bundle=self.bundle.name, node=self.node.name, item=self.id))
else:
LOG.info(_("{node}:{bundle}:{item}: fixing {type}...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
type=fix_type,
))
else:
LOG.info(_("{node}:{bundle}:{item}: creating...").format(
bundle=self.bundle.name, item=self.id, node=self.node.name))
getattr(self, "_fix_" + fix_type)(status)
def validate_name(cls, bundle, name):
for char in name:
if char not in _USERNAME_VALID_CHARACTERS:
raise BundleError(_(
"Invalid character in group name '{name}': {char} (bundle '{bundle}')"
).format(
char=char,
bundle=bundle.name,
name=name,
))
if name.endswith("_") or name.endswith("-"):
raise BundleError(_(
"Group name '{name}' must not end in dash or underscore (bundle '{bundle}')"
).format(
bundle=bundle.name,
name=name,
))
if len(name) > 30:
raise BundleError(_(
"Group name '{name}' is longer than 30 characters (bundle '{bundle}')"
).format(
bundle=bundle.name,
name=name,
))
def fix(self, status):
if not status.info['exists']:
LOG.info(_("{node}:{bundle}:{item}: creating...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
if self.attributes['gid'] is None:
command = "groupadd {}".format(self.name)
else:
command = "groupadd -g {gid} {groupname}".format(
gid=self.attributes['gid'],
groupname=self.name,
)
self.node.run(command, may_fail=True)
elif self.attributes['delete']:
LOG.info(_("{node}:{bundle}:{item}: deleting...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
self.node.run("groupdel {}".format(self.name), may_fail=True)
else:
LOG.info(_("{node}:{bundle}:{item}: updating...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
self.node.run(
"groupmod -g {gid} {groupname}".format(
gid=self.attributes['gid'],
groupname=self.name,
),
may_fail=True,
)
def validate_attributes(cls, bundle, item_id, attributes):
if attributes.get('delete', False):
for attr in attributes.keys():
if attr not in ['delete'] + list(
BUILTIN_ITEM_ATTRIBUTES.keys()):
raise BundleError(
_("{item} from bundle '{bundle}' cannot have other "
"attributes besides 'delete'").format(
item=item_id, bundle=bundle.name))
if 'hash_method' in attributes and \
attributes['hash_method'] not in HASH_METHODS:
raise BundleError(
_("Invalid hash method for {item} in bundle '{bundle}': '{method}'"
).format(
bundle=bundle.name,
item=item_id,
method=attributes['hash_method'],
))
if 'password_hash' in attributes and ('password' in attributes
or 'salt' in attributes):
raise BundleError(
_("{item} in bundle '{bundle}': 'password_hash' "
"cannot be used with 'password' or 'salt'").format(
bundle=bundle.name, item=item_id))
if 'salt' in attributes and 'password' not in attributes:
raise BundleError(
_("{}: salt given without a password").format(item_id))
def diff(content_old, content_new, filename, encoding_hint=None):
output = ""
LOG.debug("diffing {filename}: {len_before} B before, {len_after} B after".format(
filename=filename,
len_before=len(content_old),
len_after=len(content_new),
))
content_old = force_text(content_old)
content_new = force_text(content_new)
start = datetime.now()
for line in unified_diff(
content_old.splitlines(True),
content_new.splitlines(True),
fromfile=filename,
tofile=_("<bundlewrap content>"),
):
suffix = ""
line = force_text(line).rstrip("\n")
if len(line) > DIFF_MAX_LINE_LENGTH:
line = line[:DIFF_MAX_LINE_LENGTH]
suffix += _(" (line truncated after {} characters)").format(DIFF_MAX_LINE_LENGTH)
if line.startswith("+"):
line = green(line)
elif line.startswith("-"):
line = red(line)
output += line + suffix + "\n"
duration = datetime.now() - start
LOG.debug("diffing {file}: complete after {time}s".format(
file=filename,
time=duration.total_seconds(),
))
return output
def get_auto_deps(self, items):
deps = set()
for item in items:
if item == self:
continue
if ((item.ITEM_TYPE_NAME == "file"
and is_subdirectory(item.name, self.name))
or (item.ITEM_TYPE_NAME in ("file", "symlink")
and item.name == self.name)):
raise BundleError(
_("{item1} (from bundle '{bundle1}') blocking path to "
"{item2} (from bundle '{bundle2}')").format(
item1=item.id,
bundle1=item.bundle.name,
item2=self.id,
bundle2=self.bundle.name,
))
if (item.ITEM_TYPE_NAME == "directory" and item.name == self.name):
if item.attributes['purge']:
raise BundleError(
_("cannot git_deploy into purged directory {}").format(
item.name))
else:
deps.add(item.id)
return deps
def fix(self, status):
if 'type' in status.info['needs_fixing']:
# fixing the type fixes everything
if status.info['path_info'].exists:
LOG.info(_("{node}:{bundle}:{item}: fixing type...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
else:
LOG.info(_("{node}:{bundle}:{item}: creating...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
self._fix_type(status)
return
for fix_type in ('owner', 'group', 'target'):
if fix_type in status.info['needs_fixing']:
if fix_type == 'group' and \
'owner' in status.info['needs_fixing']:
# owner and group are fixed with a single chown
continue
LOG.info(_("{node}:{item}: fixing {type}...").format(
item=self.id,
node=self.node.name,
type=fix_type,
))
getattr(self, "_fix_" + fix_type)(status)
def _validate_attribute_names(cls, bundle, item_id, attributes):
if not isinstance(attributes, dict):
raise BundleError(_(
"invalid item '{item}' in bundle '{bundle}': not a dict"
).format(
item=item_id,
bundle=bundle.name,
))
invalid_attributes = set(attributes.keys()).difference(
set(cls.ITEM_ATTRIBUTES.keys()).union(
set(BUILTIN_ITEM_ATTRIBUTES.keys())
),
)
if invalid_attributes:
raise BundleError(_(
"invalid attribute(s) for '{item}' in bundle '{bundle}': {attrs}"
).format(
item=item_id,
bundle=bundle.name,
attrs=", ".join(invalid_attributes),
))
invalid_attributes = set(attributes.get('when_creating', {}).keys()).difference(
set(cls.WHEN_CREATING_ATTRIBUTES.keys())
)
if invalid_attributes:
raise BundleError(_(
"invalid when_creating attribute(s) for '{item}' in bundle '{bundle}': {attrs}"
).format(
item=item_id,
bundle=bundle.name,
attrs=", ".join(invalid_attributes),
))
def validate_attributes(cls, bundle, item_id, attributes):
if attributes.get('delete', False):
for attr in attributes.keys():
if attr not in ['delete'] + list(BUILTIN_ITEM_ATTRIBUTES.keys()):
raise BundleError(_(
"{item} from bundle '{bundle}' cannot have other "
"attributes besides 'delete'"
).format(item=item_id, bundle=bundle.name))
if 'hash_method' in attributes and \
attributes['hash_method'] not in HASH_METHODS:
raise BundleError(
_("Invalid hash method for {item} in bundle '{bundle}': '{method}'").format(
bundle=bundle.name,
item=item_id,
method=attributes['hash_method'],
)
)
if 'password_hash' in attributes and (
'password' in attributes or
'salt' in attributes
):
raise BundleError(_(
"{item} in bundle '{bundle}': 'password_hash' "
"cannot be used with 'password' or 'salt'"
).format(bundle=bundle.name, item=item_id))
if 'salt' in attributes and 'password' not in attributes:
raise BundleError(
_("{}: salt given without a password").format(item_id)
)
def git_command(cmdline, repo_dir):
"""
Runs the given git command line in the given directory.
Returns stdout of the command.
"""
cmdline = ["git"] + cmdline
io.debug(_("running '{}' in {}").format(
" ".join(cmdline),
repo_dir,
))
git_process = Popen(
cmdline,
cwd=repo_dir,
stderr=PIPE,
stdout=PIPE,
)
stdout, stderr = git_process.communicate()
if git_process.returncode != 0:
io.stderr(_("failed command: {}").format(" ".join(cmdline)))
io.stderr(_("stdout:\n{}").format(stdout))
io.stderr(_("stderr:\n{}").format(stderr))
raise RuntimeError(_("`git {command}` failed in {dir}").format(
command=cmdline[1],
dir=repo_dir,
))
return stdout.decode('utf-8').strip()
def apply(self, interactive=False, interactive_default=True):
self.node.repo.hooks.item_apply_start(
self.node.repo,
self.node,
self,
)
status_code = None
status_before = None
status_after = None
start_time = datetime.now()
if self.triggered and not self.has_been_triggered:
LOG.debug(_("skipping {} because it wasn't triggered").format(self.id))
status_code = self.STATUS_SKIPPED
if status_code is None and self.cached_unless_result:
LOG.debug(_("'unless' for {} succeeded, not fixing").format(self.id))
status_code = self.STATUS_SKIPPED
if status_code is None:
status_before = self.cached_status
if status_before.correct:
status_code = self.STATUS_OK
if status_code is None:
if not interactive:
self.fix(status_before)
status_after = self.get_status()
else:
question = wrap_question(
self.id,
self.ask(status_before),
_("Fix {}?").format(bold(self.id)),
)
if ask_interactively(question,
interactive_default):
self.fix(status_before)
status_after = self.get_status()
else:
status_code = self.STATUS_SKIPPED
if status_code is None:
if status_after.correct:
status_code = self.STATUS_FIXED
else:
status_code = self.STATUS_FAILED
self.node.repo.hooks.item_apply_end(
self.node.repo,
self.node,
self,
duration=datetime.now() - start_time,
status_code=status_code,
status_before=status_before,
status_after=status_after,
)
return status_code
def validate_name(cls, bundle, name):
if normpath(name) == "/":
raise BundleError(_("'/' cannot be a file"))
if normpath(name) != name:
raise BundleError(
_("'{path}' is an invalid file path, should be '{normpath}' (bundle '{bundle}')").format(
bundle=bundle.name, normpath=normpath(name), path=name
)
)
def ask(self, status):
before = _("running") if status.info['running'] \
else _("not running")
after = green(_("running")) if self.attributes['running'] \
else red(_("not running"))
return "{} {} → {}\n".format(
bold(_("status")),
before,
after,
)
def _get_result(self, autoskip_selector="", interactive=False, interactive_default=True):
if self.covered_by_autoskip_selector(autoskip_selector):
io.debug(_(
"autoskip matches {item} on {node}"
).format(item=self.id, node=self.node.name))
return (self.STATUS_SKIPPED, [_("cmdline")])
if interactive is False and self.attributes['interactive'] is True:
return (self.STATUS_SKIPPED, [_("interactive only")])
if self.triggered and not self.has_been_triggered:
io.debug(_("skipping {} because it wasn't triggered").format(self.id))
return (self.STATUS_SKIPPED, [_("no trigger")])
if self.unless:
with io.job(_(" {node} {bundle} {item} checking 'unless' condition...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
)):
unless_result = self.bundle.node.run(
self.unless,
may_fail=True,
)
if unless_result.return_code == 0:
io.debug(_("{node}:{bundle}:action:{name}: failed 'unless', not running").format(
bundle=self.bundle.name,
name=self.name,
node=self.bundle.node.name,
))
return (self.STATUS_SKIPPED, ["unless"])
if (
interactive and
self.attributes['interactive'] is not False and
not io.ask(
wrap_question(
self.id,
self.attributes['command'],
_("Run action {}?").format(
bold(self.name),
),
prefix="{x} {node} ".format(
node=bold(self.node.name),
x=blue("?"),
),
),
interactive_default,
epilogue="{x} {node}".format(
node=bold(self.node.name),
x=blue("?"),
),
)
):
return (self.STATUS_SKIPPED, [_("interactive")])
try:
self.run()
return (self.STATUS_ACTION_SUCCEEDED, None)
except ActionFailure:
return (self.STATUS_FAILED, None)
def ask(self, status):
if not status.info['exists']:
return _("'{}' not found in /etc/group").format(self.name)
elif self.attributes['delete']:
return _("'{}' found in /etc/group. Will be deleted.").format(self.name)
else:
return "{} {} → {}\n".format(
bold(_("GID")),
status.info['gid'],
self.attributes['gid'],
)
def validate_name(cls, bundle, name):
if normpath(name) == "/":
raise BundleError(_("'/' cannot be a file"))
if normpath(name) != name:
raise BundleError(
_("'{path}' is an invalid symlink path, should be '{normpath}' (bundle '{bundle}')"
).format(
path=name,
normpath=normpath(name),
bundle=bundle.name,
))
def ask(self, status):
before = status.info['version'] if status.info['version'] \
else _("not installed")
target = green(self.attributes['version']) if self.attributes['version'] else \
green(_("installed"))
after = target if self.attributes['installed'] \
else red(_("not installed"))
return "{} {} → {}\n".format(
bold(_("status")),
before,
after,
)
def get_auto_deps(self, items):
deps = []
for item in items:
if item == self:
continue
if ((
item.ITEM_TYPE_NAME == "file" and
is_subdirectory(item.name, self.name)
) or (
item.ITEM_TYPE_NAME in ("file", "symlink") and
item.name == self.name
)):
raise BundleError(_(
"{item1} (from bundle '{bundle1}') blocking path to "
"{item2} (from bundle '{bundle2}')"
).format(
item1=item.id,
bundle1=item.bundle.name,
item2=self.id,
bundle2=self.bundle.name,
))
elif item.ITEM_TYPE_NAME == "user" and item.name == self.attributes['owner']:
if item.attributes['delete']:
raise BundleError(_(
"{item1} (from bundle '{bundle1}') depends on item "
"{item2} (from bundle '{bundle2}') which is set to be deleted"
).format(
item1=self.id,
bundle1=self.bundle.name,
item2=item.id,
bundle2=item.bundle.name,
))
else:
deps.append(item.id)
elif item.ITEM_TYPE_NAME == "group" and item.name == self.attributes['group']:
if item.attributes['delete']:
raise BundleError(_(
"{item1} (from bundle '{bundle1}') depends on item "
"{item2} (from bundle '{bundle2}') which is set to be deleted"
).format(
item1=self.id,
bundle1=self.bundle.name,
item2=item.id,
bundle2=item.bundle.name,
))
else:
deps.append(item.id)
elif item.ITEM_TYPE_NAME in ("directory", "symlink"):
if is_subdirectory(item.name, self.name):
deps.append(item.id)
return deps
def validator_mode(item_id, value):
value = str(value)
if not value.isdigit():
raise BundleError(
_("mode for {item} should be written as digits, got: '{value}'"
"").format(item=item_id, value=value))
for digit in value:
if int(digit) > 7 or int(digit) < 0:
raise BundleError(
_("invalid mode for {item}: '{value}'").format(item=item_id,
value=value))
if not len(value) == 3 and not len(value) == 4:
raise BundleError(
_("mode for {item} should be three or four digits long, was: '{value}'"
).format(item=item_id, value=value))
def fix(self, status):
if self.attributes['installed'] is False:
LOG.info(_("{node}:{bundle}:{item}: removing...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
pkg_remove(self.node, self.name)
else:
LOG.info(_("{node}:{bundle}:{item}: installing...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
pkg_install(self.node, self.name)
def fix(self, status):
if self.attributes['running'] is False:
LOG.info(_("{node}:{bundle}:{item}: stopping...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
svc_stop(self.node, self.name)
else:
LOG.info(_("{node}:{bundle}:{item}: starting...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
svc_start(self.node, self.name)
def ask(self, status):
if not status.info['exists']:
return _("'{}' not found in /etc/passwd").format(self.name)
elif self.attributes['delete']:
return _("'{}' found in /etc/passwd. Will be deleted.").format(self.name)
output = ""
for key in status.info['needs_fixing']:
if key in ('groups', 'password', 'password_hash'):
continue
output += "{} {} → {}\n".format(
bold(_ATTRIBUTE_NAMES[key]),
status.info[key],
self.attributes[key],
)
if self.attributes['password_hash'] is not None:
if self.attributes['use_shadow']:
filename = "/etc/shadow"
found_hash = status.info['shadow_hash']
else:
filename = "/etc/passwd"
found_hash = status.info['passwd_hash']
if found_hash is None:
output += bold(_ATTRIBUTE_NAMES['password_hash']) + " " + \
_("not found in {}").format(filename) + "\n"
elif found_hash != self.attributes['password_hash']:
output += bold(_ATTRIBUTE_NAMES['password_hash']) + " " + \
found_hash + "\n"
output += " " * (len(_ATTRIBUTE_NAMES['password_hash']) - 1) + "→ " + \
self.attributes['password_hash'] + "\n"
if self.attributes['groups'] is not None:
groups_should = set(self.attributes['groups'])
groups_is = set(status.info['groups'])
missing_groups = sorted(groups_should.difference(groups_is))
extra_groups = sorted(groups_is.difference(groups_should))
if missing_groups:
output += bold(_("missing groups")) + " " + \
", ".join(missing_groups) + "\n"
if extra_groups:
output += bold(_("extra groups")) + " " + \
", ".join(extra_groups) + "\n"
return output
def content_processor_jinja2(item):
try:
from jinja2 import Environment, FileSystemLoader
except ImportError:
raise TemplateError(_(
"Unable to load Jinja2 (required to render {item}). "
"You probably have to install it using `pip install Jinja2`."
).format(item=item.id))
loader = FileSystemLoader(searchpath=[item.item_data_dir, item.item_dir])
env = Environment(loader=loader)
template = env.from_string(item._template_content)
io.debug("{node}:{bundle}:{item}: rendering with Jinja2...".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
))
start = datetime.now()
try:
content = template.render(
item=item,
bundle=item.bundle,
node=item.node,
repo=item.node.repo,
**item.attributes['context']
)
except FaultUnavailable:
raise
except Exception as e:
io.debug("".join(format_exception(*exc_info())))
raise TemplateError(_(
"Error while rendering template for {node}:{bundle}:{item}: {error}"
).format(
bundle=item.bundle.name,
error=e,
item=item.id,
node=item.node.name,
))
duration = datetime.now() - start
io.debug("{node}:{bundle}:{item}: rendered in {time}s".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
time=duration.total_seconds(),
))
return content.encode(item.attributes['encoding'])
def content_processor_mako(item):
from mako.lookup import TemplateLookup
from mako.template import Template
template = Template(
item._template_content.encode('utf-8'),
input_encoding='utf-8',
lookup=TemplateLookup(directories=[item.item_data_dir, item.item_dir]),
output_encoding=item.attributes['encoding'],
)
io.debug("{node}:{bundle}:{item}: rendering with Mako...".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
))
start = datetime.now()
try:
content = template.render(
item=item,
bundle=item.bundle,
node=item.node,
repo=item.node.repo,
**item.attributes['context']
)
except FaultUnavailable:
raise
except Exception as e:
io.debug("".join(format_exception(*exc_info())))
if isinstance(e, NameError) and str(e) == "Undefined":
# Mako isn't very verbose here. Try to give a more useful
# error message - even though we can't pinpoint the excat
# location of the error. :/
e = _("Undefined variable (look for '${...}')")
raise TemplateError(_(
"Error while rendering template for {node}:{bundle}:{item}: {error}"
).format(
bundle=item.bundle.name,
error=e,
item=item.id,
node=item.node.name,
))
duration = datetime.now() - start
io.debug("{node}:{bundle}:{item}: rendered in {time}s".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
time=duration.total_seconds(),
))
return content
def validate_attributes(cls, bundle, item_id, attributes):
if not attributes.get('sha256', None):
raise BundleError(_(
"at least one hash must be set on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
if not attributes.get('url', None):
raise BundleError(_(
"you need to specify the url on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
def validate_attributes(cls, bundle, item_id, attributes):
if not isinstance(attributes.get("running", True), bool):
raise BundleError(
_("expected boolean for 'running' on {item} in bundle '{bundle}'").format(
bundle=bundle.name, item=item_id
)
)
def KIND(self):
name = self.name.split("/", 2)[1]
if name.lower() in (
"clusterrole",
"clusterrolebinding",
"configmap",
"cronjob",
"customresourcedefinition",
"daemonset",
"deployment",
"ingress",
"namespace",
"persistentvolumeclaim",
"service",
"secret",
"statefulset",
):
raise BundleError(
_("Kind of {item_type}:{name} (bundle '{bundle}') "
"on {node} clashes with builtin k8s_* item").format(
item_type=self.ITEM_TYPE_NAME,
name=self.name,
bundle=self.bundle.name,
node=self.bundle.node.name,
regex=self.NAME_REGEX,
))
else:
return name
def get_auto_deps(self, items):
deps = []
for item in items:
if item == self:
continue
if (
(
item.ITEM_TYPE_NAME == "file" and
is_subdirectory(item.name, self.name)
)
or
(
item.ITEM_TYPE_NAME in ("file", "symlink") and
item.name == self.name
)
):
raise BundleError(_(
"{item1} (from bundle '{bundle1}') blocking path to "
"{item2} (from bundle '{bundle2}')"
).format(
item1=item.id,
bundle1=item.bundle.name,
item2=self.id,
bundle2=self.bundle.name,
))
elif item.ITEM_TYPE_NAME in ("directory", "symlink"):
if is_subdirectory(item.name, self.name):
deps.append(item.id)
return deps
def validate_attributes(cls, bundle, item_id, attributes):
if not attributes.get('sha256', None):
raise BundleError(_(
"at least one hash must be set on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
if not attributes.get('url', None):
raise BundleError(_(
"you need to specify the url on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
def _write_local_file(self):
"""
Makes the file contents available at the returned temporary path
and performs local verification if necessary or requested.
The calling method is responsible for cleaning up the file at
the returned path (only if not a binary).
"""
with tempfile() as tmp_file:
if self.attributes['content_type'] == 'binary':
local_path = self.template
else:
local_path = tmp_file
with open(local_path, 'wb') as f:
f.write(self.content)
if self.attributes['verify_with']:
cmd = self.attributes['verify_with'].format(quote(local_path))
io.debug("calling local verify command for {i}: {c}".format(
c=cmd, i=self.id))
if call(cmd, shell=True) == 0:
io.debug("{i} passed local validation".format(i=self.id))
else:
raise BundleError(
_("{i} failed local validation using: {c}").format(
c=cmd, i=self.id))
yield local_path
def parse_pkg_name(pkgname, line):
# Contains the assumption that version may not contain '-', which is covered
# according to the FreeBSD docs (Section 5.2.4, "PKGNAMEPREFIX and PKGNAMESUFFIX")
installed_package, _sep, installed_version = line.rpartition('-')
assert installed_package != "", _(
"Unexpected FreeBSD package name: {line}").format(line=line)
return installed_package == pkgname, installed_version
def sdict(self):
result = run_local([
"kubectl",
"--context={}".format(self.node.kubectl_context),
"--namespace={}".format(self.namespace),
"get",
"-o",
"json",
self.KUBECTL_RESOURCE_TYPE,
self.resource_name,
])
if result.return_code == 0:
full_json_response = json.loads(result.stdout)
if full_json_response.get("status", {}).get("phase") == "Terminating":
# this resource is currently being deleted, consider it gone
return None
return {'manifest': json.dumps(reduce_dict(
full_json_response,
json.loads(self.manifest),
), indent=4, sort_keys=True)}
elif result.return_code == 1 and "NotFound" in result.stderr.decode('utf-8'):
return None
else:
io.debug(result.stdout.decode('utf-8'))
io.debug(result.stderr.decode('utf-8'))
raise RuntimeError(_("error getting state of {}, check `bw --debug`".format(self.id)))
def _manifest_dict(self):
if self.attributes['manifest_processor'] == 'jinja2':
content_processor = content_processor_jinja2
elif self.attributes['manifest_processor'] == 'mako':
content_processor = content_processor_mako
else:
content_processor = lambda item: item._template_content.encode(
'utf-8')
if self.attributes['manifest'] is not None or self.attributes[
'manifest_file'] is None:
user_manifest = self.attributes['manifest'] or {}
elif (self.attributes['manifest_file'].endswith(".yaml")
or self.attributes['manifest_file'].endswith(".yml")):
user_manifest = yaml.load(content_processor(self),
Loader=yaml.SafeLoader)
elif self.attributes['manifest_file'].endswith(".json"):
user_manifest = json.loads(content_processor(self))
merged_manifest = merge_dict(
{
'kind': self.KIND,
'metadata': {
'name': self.name.split("/")[-1],
},
},
user_manifest,
)
if merged_manifest.get('apiVersion') is None:
raise BundleError(
_("{item} from bundle '{bundle}' needs an apiVersion in its manifest"
).format(item=self.id, bundle=self.bundle.name))
return merged_manifest
def sdict(self):
result = self.run_local(
self._kubectl +
["get", "-o", "json", self.KIND, self.resource_name])
if result.return_code == 0:
full_json_response = json.loads(result.stdout.decode('utf-8'))
if full_json_response.get("status",
{}).get("phase") == "Terminating":
# this resource is currently being deleted, consider it gone
return None
return {
'manifest':
json.dumps(reduce_dict(
full_json_response,
self.nuke_k8s_status(json.loads(self.manifest)),
),
indent=4,
sort_keys=True)
}
elif result.return_code == 1 and "NotFound" in result.stderr.decode(
'utf-8'):
return None
else:
io.debug(result.stdout.decode('utf-8'))
io.debug(result.stderr.decode('utf-8'))
raise RuntimeError(
_("error getting state of {}, check `bw --debug`".format(
self.id)))
def validate_attributes(cls, bundle, item_id, attributes):
if not isinstance(attributes.get('installed', True), bool):
raise BundleError(_(
"expected boolean for 'installed' on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
if 'version' in attributes and attributes.get('installed', True) is False:
raise BundleError(_(
"cannot set version for uninstalled package on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
def validator_mode(item_id, value):
value = str(value)
if not value.isdigit():
raise BundleError(
_("mode for {item} should be written as digits, got: '{value}'"
"").format(item=item_id, value=value)
)
for digit in value:
if int(digit) > 7 or int(digit) < 0:
raise BundleError(_(
"invalid mode for {item}: '{value}'"
).format(item=item_id, value=value))
if not len(value) == 3 and not len(value) == 4:
raise BundleError(_(
"mode for {item} should be three or four digits long, was: '{value}'"
).format(item=item_id, value=value))
def _write_local_file(self):
"""
Makes the file contents available at the returned temporary path
and performs local verification if necessary or requested.
The calling method is responsible for cleaning up the file at
the returned path (only if not a binary).
"""
if self.attributes['content_type'] == 'binary':
local_path = self.template
else:
handle, local_path = mkstemp()
with open(local_path, 'wb') as f:
f.write(self.content)
if self.attributes['verify_with']:
cmd = self.attributes['verify_with'].format(quote(local_path))
LOG.debug("calling local verify command for {i}: {c}".format(c=cmd, i=self.id))
if call(cmd, shell=True) == 0:
LOG.debug("{i} passed local validation".format(i=self.id))
else:
raise BundleError(_(
"{i} failed local validation using: {c}"
).format(c=cmd, i=self.id))
return local_path
def _repo_dir(self):
if "://" in self.attributes['repo']:
repo_dir = clone_to_dir(self.attributes['repo'], self.attributes['rev'])
io.debug(_("registering {} for deletion on exit").format(repo_dir))
at_exit(rmtree, repo_dir)
else:
repo_dir = get_local_repo_path(self.node.repo.path, self.attributes['repo'])
return repo_dir
def validate_attributes(cls, bundle, item_id, attributes):
if not isinstance(attributes.get('installed', True), bool):
raise BundleError(
_("expected boolean for 'installed' on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
def validate_attributes(cls, bundle, item_id, attributes):
if attributes.get('delete', False):
for attr in attributes.keys():
if attr not in ['delete'] + list(BUILTIN_ITEM_ATTRIBUTES.keys()):
raise BundleError(_(
"{item} from bundle '{bundle}' cannot have other "
"attributes besides 'delete'"
).format(item=item_id, bundle=bundle.name))
if attributes.get('manifest') and attributes.get('manifest_file'):
raise BundleError(_(
"{item} from bundle '{bundle}' cannot have both 'manifest' and 'manifest_file'"
).format(item=item_id, bundle=bundle.name))
if attributes.get('manifest_processor') not in (None, 'jinja2', 'mako'):
raise BundleError(_(
"{item} from bundle '{bundle}' has invalid manifest_processor "
"(must be 'jinja2' or 'mako')"
).format(item=item_id, bundle=bundle.name))
def validate_attributes(cls, bundle, item_id, attributes):
if item_id[len(cls.ITEM_TYPE_NAME) +
1:] not in ['filter', 'mangle', 'nat']:
raise BundleError(
_("table '{table}' is not allowed {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
table=item_id[len(bundle.name) + 1:],
))
if not attributes.get('chains', None):
raise BundleError(
_("chains must be set on {item} in bundle '{bundle}'").format(
bundle=bundle.name,
item=item_id,
))
def content_processor_jinja2(item):
try:
from jinja2 import Environment, FileSystemLoader
except ImportError:
raise TemplateError(
_("Unable to load Jinja2 (required to render {item}). "
"You probably have to install it using `pip install Jinja2`.").
format(item=item.id))
loader = FileSystemLoader(searchpath=[item.item_data_dir, item.item_dir])
env = Environment(loader=loader)
template = env.from_string(item._template_content)
io.debug("{node}:{bundle}:{item}: rendering with Jinja2...".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
))
start = datetime.now()
try:
content = template.render(item=item,
bundle=item.bundle,
node=item.node,
repo=item.node.repo,
**item.attributes['context'])
except FaultUnavailable:
raise
except Exception as e:
io.debug("".join(format_exception(*exc_info())))
raise TemplateError(
_("Error while rendering template for {node}:{bundle}:{item}: {error}"
).format(
bundle=item.bundle.name,
error=e,
item=item.id,
node=item.node.name,
))
duration = datetime.now() - start
io.debug("{node}:{bundle}:{item}: rendered in {time}s".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
time=duration.total_seconds(),
))
return content.encode(item.attributes['encoding'])
def content_processor_mako(item):
template = Template(
item._template_content.encode('utf-8'),
input_encoding='utf-8',
lookup=TemplateLookup(directories=[item.item_data_dir, item.item_dir]),
output_encoding=item.attributes['encoding'],
)
io.debug("{node}:{bundle}:{item}: rendering with Mako...".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
))
start = datetime.now()
try:
content = template.render(item=item,
bundle=item.bundle,
node=item.node,
repo=item.node.repo,
**item.attributes['context'])
except FaultUnavailable:
raise
except Exception as e:
io.debug("".join(format_exception(*exc_info())))
if isinstance(e, NameError) and str(e) == "Undefined":
# Mako isn't very verbose here. Try to give a more useful
# error message - even though we can't pinpoint the excat
# location of the error. :/
e = _("Undefined variable (look for '${...}')")
elif isinstance(e, KeyError):
e = _("KeyError: {}").format(str(e))
raise TemplateError(
_("Error while rendering template for {node}:{bundle}:{item}: {error}"
).format(
bundle=item.bundle.name,
error=e,
item=item.id,
node=item.node.name,
))
duration = datetime.now() - start
io.debug("{node}:{bundle}:{item}: rendered in {time}s".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
time=duration.total_seconds(),
))
return content
def validate_attributes(cls, bundle, item_id, attributes):
if not isinstance(attributes.get('installed', True), bool):
raise BundleError(
_("expected boolean for 'installed' on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
if 'version' in attributes and attributes.get('installed',
True) is False:
raise BundleError(
_("cannot set version for uninstalled package on {item} in bundle '{bundle}'"
).format(
bundle=bundle.name,
item=item_id,
))
def validate_attributes(cls, bundle, item_id, attributes):
if attributes.get('delete', False):
for attr in attributes.keys():
if attr not in ['delete'] + list(BUILTIN_ITEM_ATTRIBUTES.keys()):
raise BundleError(_(
"{item} from bundle '{bundle}' cannot have other "
"attributes besides 'delete'"
).format(item=item_id, bundle=bundle.name))
def get_local_repo_path(bw_repo_path, repo_name):
"""
From the given BundleWrap repo, get the filesystem path to the git
repo associated with the given internal repo name.
"""
repo_map_path = join(bw_repo_path, REPO_MAP_FILENAME)
if not isfile(repo_map_path):
io.stderr(_("missing repo map for git_deploy at {}").format(repo_map_path))
io.stderr(_("you must create this file with the following format:"))
io.stderr(_(" <value of repo attribute on git_deploy item>: "
"<absolute path to local git repo>"))
io.stderr(_("since the path is local, you should also add the "
"{} file to your gitignore").format(REPO_MAP_FILENAME))
raise RepositoryError(_("missing repo map for git_deploy"))
with open(join(bw_repo_path, REPO_MAP_FILENAME)) as f:
repo_map = f.readlines()
for line in repo_map:
if not line.strip() or line.startswith("#"):
continue
try:
repo, path = line.split(":", 1)
except:
raise RepositoryError(_("unable to parse line from {path}: '{line}'").format(
line=line,
path=repo_map_path,
))
if repo_name == repo:
return path.strip()
raise RepositoryError(_("no path found for repo '{repo}' in {path}").format(
path=repo_map_path,
repo=repo_name,
))
def _validate_name(cls, bundle, name):
if ":" in name:
raise BundleError(
_("invalid name for {type} in bundle '{bundle}': {name} (must not contain colon)"
).format(
bundle=bundle.name,
name=name,
type=cls.ITEM_TYPE_NAME,
))
def _test(self):
with io.job(_("{node} {bundle} {item}").format(
bundle=bold(self.bundle.name),
item=self.id,
node=bold(self.node.name),
)):
if self._faults_missing_for_attributes:
self._raise_for_faults()
return self.test()
def validate_name(cls, bundle, name):
if normpath(name) != name:
raise BundleError(
_("'{path}' is an invalid directory path, "
"should be '{normpath}' (bundle '{bundle}')").format(
bundle=bundle.name,
normpath=normpath(name),
path=name,
))
def _raise_for_faults(self):
raise FaultUnavailable(
_("{item} on {node} is missing faults "
"for these attributes: {attrs} "
"(most of the time this means you're missing "
"a required key in your .secrets.cfg)").format(
attrs=", ".join(sorted(self._faults_missing_for_attributes)),
item=self.id,
node=self.node.name,
))