def _write_local_file(self):
"""
Makes the file contents available at the returned temporary path
and performs local verification if necessary or requested.
The calling method is responsible for cleaning up the file at
the returned path (only if not a binary).
"""
if self.attributes['content_type'] == 'binary':
local_path = self.template
else:
handle, local_path = mkstemp()
with open(local_path, 'wb') as f:
f.write(self.content)
if self.attributes['verify_with']:
cmd = self.attributes['verify_with'].format(quote(local_path))
LOG.debug("calling local verify command for {i}: {c}".format(c=cmd, i=self.id))
if call(cmd, shell=True) == 0:
LOG.debug("{i} passed local validation".format(i=self.id))
else:
raise BundleError(_(
"{i} failed local validation using: {c}"
).format(c=cmd, i=self.id))
return local_path
def node_apply_end(repo, node, duration=None, interactive=None, result=None, **kwargs):
if environ.get('TERM_PROGRAM', None) != "iTerm.app" or not interactive:
LOG.debug("skipping iTerm stats (wrong terminal)")
return
if not IMPORTS:
LOG.error("failed to import dependencies of itermstats plugin")
return
css_file = NamedTemporaryFile(delete=False)
css_file.write(".text-overlay { display: none; }")
css_file.close()
config = Config(
height=150,
style=STYLE,
width=350,
)
config.css.append(css_file.name)
chart = Pie(config)
chart.add('correct', result.correct)
chart.add('fixed', result.fixed)
chart.add('skipped', result.skipped)
chart.add('failed', result.failed)
png_data = cairosvg.svg2png(bytestring=chart.render())
png_data_b64 = b64encode(png_data)
remove(css_file.name)
print("\033]1337;File=inline=1:{}\007".format(png_data_b64))
def diff(content_old, content_new, filename, encoding_hint=None):
output = ""
LOG.debug("diffing {filename}: {len_before} B before, {len_after} B after".format(
filename=filename,
len_before=len(content_old),
len_after=len(content_new),
))
content_old = force_text(content_old)
content_new = force_text(content_new)
start = datetime.now()
for line in unified_diff(
content_old.splitlines(True),
content_new.splitlines(True),
fromfile=filename,
tofile=_("<bundlewrap content>"),
):
suffix = ""
line = force_text(line).rstrip("\n")
if len(line) > DIFF_MAX_LINE_LENGTH:
line = line[:DIFF_MAX_LINE_LENGTH]
suffix += _(" (line truncated after {} characters)").format(DIFF_MAX_LINE_LENGTH)
if line.startswith("+"):
line = green(line)
elif line.startswith("-"):
line = red(line)
output += line + suffix + "\n"
duration = datetime.now() - start
LOG.debug("diffing {file}: complete after {time}s".format(
file=filename,
time=duration.total_seconds(),
))
return output
def apply(self, interactive=False, interactive_default=True):
self.node.repo.hooks.item_apply_start(
self.node.repo,
self.node,
self,
)
status_code = None
status_before = None
status_after = None
start_time = datetime.now()
if self.triggered and not self.has_been_triggered:
LOG.debug(_("skipping {} because it wasn't triggered").format(self.id))
status_code = self.STATUS_SKIPPED
if status_code is None and self.cached_unless_result:
LOG.debug(_("'unless' for {} succeeded, not fixing").format(self.id))
status_code = self.STATUS_SKIPPED
if status_code is None:
status_before = self.cached_status
if status_before.correct:
status_code = self.STATUS_OK
if status_code is None:
if not interactive:
self.fix(status_before)
status_after = self.get_status()
else:
question = wrap_question(
self.id,
self.ask(status_before),
_("Fix {}?").format(bold(self.id)),
)
if ask_interactively(question,
interactive_default):
self.fix(status_before)
status_after = self.get_status()
else:
status_code = self.STATUS_SKIPPED
if status_code is None:
if status_after.correct:
status_code = self.STATUS_FIXED
else:
status_code = self.STATUS_FAILED
self.node.repo.hooks.item_apply_end(
self.node.repo,
self.node,
self,
duration=datetime.now() - start_time,
status_code=status_code,
status_before=status_before,
status_after=status_after,
)
return status_code
def _create_config(path):
LOG.debug("writing initial config for Slack notifications to .slack.cfg")
config = SafeConfigParser()
config.add_section("configuration")
config.set("configuration", "enabled", "unconfigured")
config.set("configuration", "username", "your-slack-username")
config.add_section("connection")
config.set("connection", "url",
"<insert URL from https://my.slack.com/services/new/incoming-webhook>")
config.add_section("apply_notifications")
config.set("apply_notifications", "enabled", "yes")
config.set("apply_notifications", "allow_groups", "all")
config.set("apply_notifications", "deny_groups", "local")
with open(path, 'wb') as f:
config.write(f)
def fix(self, status):
if self.attributes['installed'] is False:
LOG.info(_("{node}:{bundle}:{item}: removing...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
pkg_remove(self.node, self.name)
else:
LOG.info(_("{node}:{bundle}:{item}: installing...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
pkg_install(self.node, self.name)
def fix(self, status):
if self.attributes['running'] is False:
LOG.info(_("{node}:{bundle}:{item}: stopping...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
svc_stop(self.node, self.name)
else:
LOG.info(_("{node}:{bundle}:{item}: starting...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
svc_start(self.node, self.name)
def _create_config(path):
LOG.debug("writing initial config for HipChat notifications to .hipchat_secrets.cfg")
config = SafeConfigParser()
config.add_section("configuration")
config.set("configuration", "enabled", "unconfigured")
config.add_section("connection")
config.set("connection", "server", "api.hipchat.com")
config.set("connection", "token", "<insert token from https://www.hipchat.com/account/api>")
config.add_section("apply_notifications")
config.set("apply_notifications", "enabled", "yes")
config.set("apply_notifications", "rooms", "name_or_id_of_room1,name_or_id_of_room2")
config.add_section("item_notifications")
config.set("item_notifications", "enabled", "no")
config.set("item_notifications", "rooms", "name_or_id_of_room1,name_or_id_of_room2")
with open(path, 'wb') as f:
config.write(f)
def item_apply_end(repo,
node,
item,
duration=None,
status_before=None,
status_after=None,
**kwargs):
config = _get_config(repo.path)
if config is None or \
not config.has_section("item_notifications") or \
not config.getboolean("item_notifications", "enabled"):
return
color = "gray"
if status_before.correct:
return
elif status_after is None:
color = "purple"
status_string = "(unknown)"
elif status_after.correct:
color = "green"
status_string = "(successful)"
else:
color = "red"
status_string = "(failed)"
for room in config.get("item_notifications", "rooms").split(","):
LOG.debug(
"posting item apply end notification to HipChat room {room}@{server}"
.format(
room=room,
server=config.get("connection", "server"),
))
_notify(
config.get("connection", "server"),
room.strip(),
config.get("connection", "token"),
"{status_string} {node}:{bundle}:{item}".format(
bundle=item.bundle.name,
item=item,
node=node.name,
status_string=status_string,
),
"text",
color=color,
)
def apply_end(repo, target, nodes, duration=None, **kwargs):
config = _get_config(repo.path)
if config is None or \
not config.has_section("apply_notifications") or \
not config.getboolean("apply_notifications", "enabled"):
return
for room in config.get("apply_notifications", "rooms").split(","):
LOG.debug("posting apply end notification to HipChat room {room}@{server}".format(
room=room,
server=config.get("connection", "server"),
))
_notify(
config.get("connection", "server"),
room.strip(),
config.get("connection", "token"),
"Finished bw apply on <b>{target}</b>.".format(target=target),
"html",
)
def diff(content_old, content_new, filename, encoding_hint=None):
output = ""
LOG.debug("diffing {filename}: {len_before} B before, {len_after} B after".format(
filename=filename,
len_before=len(content_old),
len_after=len(content_new),
))
start = datetime.now()
for line in unified_diff(
content_old.splitlines(True),
content_new.splitlines(True),
fromfile=filename,
tofile=_("<bundlewrap content>"),
):
suffix = ""
try:
line = line.decode('utf-8')
except UnicodeDecodeError:
if encoding_hint and encoding_hint.lower() != "utf-8":
try:
line = line.decode(encoding_hint)
suffix += _(" (line encoded in {})").format(encoding_hint)
except UnicodeDecodeError:
line = line[0]
suffix += _(" (line not encoded in UTF-8 or {})").format(encoding_hint)
else:
line = line[0]
suffix += _(" (line not encoded in UTF-8)")
line = line.rstrip("\n")
if len(line) > DIFF_MAX_LINE_LENGTH:
line = line[:DIFF_MAX_LINE_LENGTH]
suffix += _(" (line truncated after {} characters)").format(DIFF_MAX_LINE_LENGTH)
if line.startswith("+"):
line = green(line)
elif line.startswith("-"):
line = red(line)
output += line + suffix + "\n"
duration = datetime.now() - start
LOG.debug("diffing {file}: complete after {time}s".format(
file=filename,
time=duration.total_seconds(),
))
return output
def _notify(server, room, token, message, message_format, color="gray"):
try:
post(
"https://{server}/v2/room/{room}/notification?auth_token={token}".format(
token=token,
room=room,
server=server,
),
headers={
'content-type': 'application/json',
},
data=dumps({
'color': color,
'message': message,
'message_format': message_format,
'notify': True,
}),
)
except ConnectionError as e:
LOG.error("Failed to submit HipChat notification: {}".format(e))
def apply_end(repo, target, nodes, duration=None, **kwargs):
config = _get_config(repo.path)
if config is None or \
not config.has_section("apply_notifications") or \
not config.getboolean("apply_notifications", "enabled") or \
not _check_allowed_groups(config, nodes):
return
LOG.debug("posting apply end notification to Slack")
_notify(
config.get("connection", "url"),
color="good",
fallback="Finished bw apply to {target} as {user} after {duration}s.".format(
duration=duration.total_seconds(),
target=target,
user=config.get("configuration", "username"),
),
target=target,
title="Finished bw apply after {}s.".format(duration.total_seconds()),
user=config.get("configuration", "username"),
)
def apply_end(repo, target, nodes, duration=None, **kwargs):
config = _get_config(repo.path)
if config is None or \
not config.has_section("apply_notifications") or \
not config.getboolean("apply_notifications", "enabled"):
return
for room in config.get("apply_notifications", "rooms").split(","):
LOG.debug(
"posting apply end notification to HipChat room {room}@{server}".
format(
room=room,
server=config.get("connection", "server"),
))
_notify(
config.get("connection", "server"),
room.strip(),
config.get("connection", "token"),
"Finished bw apply on <b>{target}</b>.".format(target=target),
"html",
)
def apply_start(repo, target, nodes, interactive=False, **kwargs):
config = _get_config(repo.path)
if config is None or \
not config.has_section("apply_notifications") or \
not config.getboolean("apply_notifications", "enabled") or \
not _check_allowed_groups(config, nodes):
return
LOG.debug("posting apply start notification to Slack")
_notify(
config.get("connection", "url"),
fallback="Starting bw apply to {target} as {user}".format(
target=target,
user=config.get("configuration", "username"),
),
target=target,
title=(
"Starting {interactive}interactive bw apply..."
).format(interactive="non-" if not interactive else ""),
user=config.get("configuration", "username"),
)
def _get_result(self, interactive=False, interactive_default=True):
if interactive is False and self.attributes['interactive'] is True:
return self.STATUS_SKIPPED
if self.triggered and not self.has_been_triggered:
LOG.debug(_("skipping {} because it wasn't triggered").format(self.id))
return self.STATUS_SKIPPED
if self.unless:
unless_result = self.bundle.node.run(
self.unless,
may_fail=True,
)
if unless_result.return_code == 0:
LOG.debug(_("{node}:{bundle}:action:{name}: failed 'unless', not running").format(
bundle=self.bundle.name,
name=self.name,
node=self.bundle.node.name,
))
return self.STATUS_SKIPPED
if (
interactive and
self.attributes['interactive'] is not False
and not ask_interactively(
wrap_question(
self.id,
self.attributes['command'],
_("Run action {}?").format(
bold(self.name),
),
),
interactive_default,
)
):
return self.STATUS_SKIPPED
try:
self.run(interactive=interactive)
return self.STATUS_ACTION_SUCCEEDED
except ActionFailure:
return self.STATUS_FAILED
def _create_config(path):
LOG.debug(
"writing initial config for HipChat notifications to .hipchat_secrets.cfg"
)
config = SafeConfigParser()
config.add_section("configuration")
config.set("configuration", "enabled", "unconfigured")
config.add_section("connection")
config.set("connection", "server", "api.hipchat.com")
config.set("connection", "token",
"<insert token from https://www.hipchat.com/account/api>")
config.add_section("apply_notifications")
config.set("apply_notifications", "enabled", "yes")
config.set("apply_notifications", "rooms",
"name_or_id_of_room1,name_or_id_of_room2")
config.add_section("item_notifications")
config.set("item_notifications", "enabled", "no")
config.set("item_notifications", "rooms",
"name_or_id_of_room1,name_or_id_of_room2")
with open(path, 'wb') as f:
config.write(f)
def _notify(server, room, token, message, message_format, color="gray"):
try:
post(
"https://{server}/v2/room/{room}/notification?auth_token={token}".
format(
token=token,
room=room,
server=server,
),
headers={
'content-type': 'application/json',
},
data=dumps({
'color': color,
'message': message,
'message_format': message_format,
'notify': True,
}),
)
except ConnectionError as e:
LOG.error("Failed to submit HipChat notification: {}".format(e))
def fix(self, status):
if 'type' in status.info['needs_fixing']:
# fixing the type fixes everything
if status.info['path_info'].exists:
LOG.info(_("{node}:{bundle}:{item}: fixing type...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
else:
LOG.info(_("{node}:{bundle}:{item}: creating...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
self._fix_type(status)
return
for fix_type in ('owner', 'group', 'target'):
if fix_type in status.info['needs_fixing']:
if fix_type == 'group' and \
'owner' in status.info['needs_fixing']:
# owner and group are fixed with a single chown
continue
LOG.info(_("{node}:{item}: fixing {type}...").format(
item=self.id,
node=self.node.name,
type=fix_type,
))
getattr(self, "_fix_" + fix_type)(status)
def fix(self, status):
if not status.info['exists']:
LOG.info(_("{node}:{bundle}:{item}: creating...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
if self.attributes['gid'] is None:
command = "groupadd {}".format(self.name)
else:
command = "groupadd -g {gid} {groupname}".format(
gid=self.attributes['gid'],
groupname=self.name,
)
self.node.run(command, may_fail=True)
elif self.attributes['delete']:
LOG.info(_("{node}:{bundle}:{item}: deleting...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
self.node.run("groupdel {}".format(self.name), may_fail=True)
else:
LOG.info(_("{node}:{bundle}:{item}: updating...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
))
self.node.run(
"groupmod -g {gid} {groupname}".format(
gid=self.attributes['gid'],
groupname=self.name,
),
may_fail=True,
)
def _notify(url, message=None, title=None, fallback=None, user=None, target=None, color="#000000"):
payload = {
"icon_url": "http://bundlewrap.org/img/icon.png",
"username": "******",
}
if fallback:
payload["attachments"] = [{
"color": color,
"fallback": fallback,
}]
if message:
payload["attachments"][0]["text"] = message
if title:
payload["attachments"][0]["title"] = title
if target and user:
payload["attachments"][0]["fields"] = [
{
"short": True,
"title": "User",
"value": user,
},
{
"short": True,
"title": "Target",
"value": target,
},
]
else:
payload["text"] = message
try:
post(
url,
headers={
'content-type': 'application/json',
},
data=dumps(payload),
)
except ConnectionError as e:
LOG.error("Failed to submit Slack notification: {}".format(e))
def fix(self, status):
for fix_type in ('type', 'content', 'mode', 'owner', 'group'):
if fix_type in status.info['needs_fixing']:
if fix_type == 'group' and \
'owner' in status.info['needs_fixing']:
# owner and group are fixed with a single chown
continue
if fix_type in ('mode', 'owner', 'group') and \
'content' in status.info['needs_fixing']:
# fixing content implies settings mode and owner/group
continue
if status.info['path_info'].exists:
if self.attributes['delete']:
LOG.info(_("{node}:{bundle}:{item}: deleting...").format(
bundle=self.bundle.name, node=self.node.name, item=self.id))
else:
LOG.info(_("{node}:{bundle}:{item}: fixing {type}...").format(
bundle=self.bundle.name,
item=self.id,
node=self.node.name,
type=fix_type,
))
else:
LOG.info(_("{node}:{bundle}:{item}: creating...").format(
bundle=self.bundle.name, item=self.id, node=self.node.name))
getattr(self, "_fix_" + fix_type)(status)
def item_apply_end(
repo, node, item, duration=None, status_before=None, status_after=None, **kwargs
):
config = _get_config(repo.path)
if config is None or \
not config.has_section("item_notifications") or \
not config.getboolean("item_notifications", "enabled"):
return
color = "gray"
if status_before.correct:
return
elif status_after is None:
color = "purple"
status_string = "(unknown)"
elif status_after.correct:
color = "green"
status_string = "(successful)"
else:
color = "red"
status_string = "(failed)"
for room in config.get("item_notifications", "rooms").split(","):
LOG.debug("posting item apply end notification to HipChat room {room}@{server}".format(
room=room,
server=config.get("connection", "server"),
))
_notify(
config.get("connection", "server"),
room.strip(),
config.get("connection", "token"),
"{status_string} {node}:{bundle}:{item}".format(
bundle=item.bundle.name,
item=item,
node=node.name,
status_string=status_string,
),
"text",
color=color,
)
def fix(self, status):
if status.info['exists']:
if self.attributes['delete']:
msg = _("{node}:{bundle}:{item}: deleting...")
else:
msg = _("{node}:{bundle}:{item}: updating...")
else:
msg = _("{node}:{bundle}:{item}: creating...")
LOG.info(msg.format(bundle=self.bundle.name, item=self.id, node=self.node.name))
if self.attributes['delete']:
self.node.run("userdel {}".format(self.name), may_fail=True)
else:
command = "useradd " if not status.info['exists'] else "usermod "
for attr, option in sorted(_ATTRIBUTE_OPTIONS.items()):
if attr in status.info['needs_fixing'] and self.attributes[attr] is not None:
if attr == 'groups':
value = ",".join(self.attributes[attr])
else:
value = str(self.attributes[attr])
command += "{} {} ".format(option, quote(value))
command += self.name
self.node.run(command, may_fail=True)
def apply_start(repo, target, nodes, interactive=False, **kwargs):
config = _get_config(repo.path)
if config is None or \
not config.has_section("apply_notifications") or \
not config.getboolean("apply_notifications", "enabled"):
return
for room in config.get("apply_notifications", "rooms").split(","):
LOG.debug("posting apply start notification to HipChat room {room}@{server}".format(
room=room,
server=config.get("connection", "server"),
))
_notify(
config.get("connection", "server"),
room.strip(),
config.get("connection", "token"),
(
"Starting {interactive}interactive "
"bw apply on <b>{target}</b>..."
).format(
interactive="non-" if not interactive else "",
target=target,
),
"html",
)
def apply_start(repo, target, nodes, interactive=False, **kwargs):
config = _get_config(repo.path)
if config is None or \
not config.has_section("apply_notifications") or \
not config.getboolean("apply_notifications", "enabled"):
return
for room in config.get("apply_notifications", "rooms").split(","):
LOG.debug(
"posting apply start notification to HipChat room {room}@{server}".
format(
room=room,
server=config.get("connection", "server"),
))
_notify(
config.get("connection", "server"),
room.strip(),
config.get("connection", "token"),
("Starting {interactive}interactive "
"bw apply on <b>{target}</b>...").format(
interactive="non-" if not interactive else "",
target=target,
),
"html",
)
def content_processor_mako(item):
from mako.lookup import TemplateLookup
from mako.template import Template
template = Template(
item._template_content.encode('utf-8'),
input_encoding='utf-8',
lookup=TemplateLookup(directories=[item.item_data_dir, item.item_dir]),
output_encoding=item.attributes['encoding'],
)
LOG.debug("{node}:{bundle}:{item}: rendering with Mako...".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
))
start = datetime.now()
try:
content = template.render(
item=item,
bundle=item.bundle,
node=item.node,
repo=item.node.repo,
**item.attributes['context']
)
except Exception as e:
LOG.debug("".join(format_exception(*exc_info())))
if isinstance(e, NameError) and e.message == "Undefined":
# Mako isn't very verbose here. Try to give a more useful
# error message - even though we can't pinpoint the excat
# location of the error. :/
e = _("Undefined variable (look for '${...}')")
raise TemplateError(_(
"Error while rendering template for {node}:{bundle}:{item}: {error}"
).format(
bundle=item.bundle.name,
error=e,
item=item.id,
node=item.node.name,
))
duration = datetime.now() - start
LOG.debug("{node}:{bundle}:{item}: rendered in {time}s".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
time=duration.total_seconds(),
))
return content
def content_processor_jinja2(item):
try:
from jinja2 import Environment, FileSystemLoader
except ImportError:
raise TemplateError(_(
"Unable to load Jinja2 (required to render {item}). "
"You probably have to install it using `pip install Jinja2`."
).format(item=item.id))
loader = FileSystemLoader(searchpath=[item.item_data_dir, item.item_dir])
env = Environment(loader=loader)
template = env.from_string(item._template_content)
LOG.debug("{node}:{bundle}:{item}: rendering with Jinja2...".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
))
start = datetime.now()
try:
content = template.render(
item=item,
bundle=item.bundle,
node=item.node,
repo=item.node.repo,
**item.attributes['context']
)
except Exception as e:
LOG.debug("".join(format_exception(*exc_info())))
raise TemplateError(_(
"Error while rendering template for {node}:{bundle}:{item}: {error}"
).format(
bundle=item.bundle.name,
error=e,
item=item.id,
node=item.node.name,
))
duration = datetime.now() - start
LOG.debug("{node}:{bundle}:{item}: rendered in {time}s".format(
bundle=item.bundle.name,
item=item.id,
node=item.node.name,
time=duration.total_seconds(),
))
return content.encode(item.attributes['encoding'])
def _get_config(repo_path):
config_path = join(repo_path, ".slack.cfg")
if not exists(config_path):
_create_config(config_path)
config = SafeConfigParser()
config.read(config_path)
if config.get("configuration", "enabled") == "unconfigured":
LOG.error("Slack notifications not configured. Please edit .slack.cfg "
"(it has already been created) and set enabled to 'yes' "
"(or 'no' to silence this message and disable Slack notifications).")
return None
elif config.get("configuration", "enabled").lower() not in ("yes", "true", "1"):
LOG.debug("Slack notifications not enabled in .slack.cfg, skipping...")
return None
elif not REQUESTS:
LOG.error("Slack notifications need the requests library. "
"You can usually install it with `pip install requests`.")
return None
return config
def _get_config(repo_path):
config_path = join(repo_path, ".hipchat_secrets.cfg")
if not exists(config_path):
_create_config(config_path)
config = SafeConfigParser()
config.read(config_path)
if config.get("configuration", "enabled") == "unconfigured":
LOG.error(
"HipChat notifications not configured. Please edit .hipchat_secrets.cfg "
"(it has already been created) and set enabled to 'yes' "
"(or 'no' to silence this message and disable HipChat notifications)."
)
return None
elif config.get("configuration",
"enabled").lower() not in ("yes", "true", "1"):
LOG.debug(
"HipChat notifications not enabled in .hipchat_secrets.cfg, skipping..."
)
return None
elif not REQUESTS:
LOG.error("HipChat notifications need the requests library. "
"You can usually install it with `pip install requests`.")
return None
return config