def sdict(self): result = run_local( self._kubectl + ["get", "-o", "json", self.KIND, self.resource_name]) if result.return_code == 0: full_json_response = json.loads(result.stdout) if full_json_response.get("status", {}).get("phase") == "Terminating": # this resource is currently being deleted, consider it gone return None return { 'manifest': json.dumps(reduce_dict( full_json_response, json.loads(self.manifest), ), indent=4, sort_keys=True) } elif result.return_code == 1 and "NotFound" in result.stderr.decode( 'utf-8'): return None else: io.debug(result.stdout.decode('utf-8')) io.debug(result.stderr.decode('utf-8')) raise RuntimeError( _("error getting state of {}, check `bw --debug`".format( self.id)))
def _get_paths_to_purge(self): result = self.node.run("find {} -maxdepth 1 -print0".format(quote(self.name))) for line in result.stdout.split(b"\0"): line = line.decode('utf-8') found = False for item_type in ('directory', 'file', 'symlink'): if found: break for item in self.node.items: if ( item.id == "{}:{}".format(item_type, line) or item.id.startswith("{}:{}/".format(item_type, line)) ): found = True break if not found: # this file or directory is not managed io.debug(( "found unmanaged path below {dirpath} on {node}, " "marking for removal: {path}" ).format( dirpath=self.name, node=self.node.name, path=line, )) yield line
def git_command(cmdline, repo_dir): """ Runs the given git command line in the given directory. Returns stdout of the command. """ cmdline = ["git"] + cmdline io.debug(_("running '{}' in {}").format( " ".join(cmdline), repo_dir, )) git_process = Popen( cmdline, cwd=repo_dir, stderr=PIPE, stdout=PIPE, ) stdout, stderr = git_process.communicate() if git_process.returncode != 0: io.stderr(_("failed command: {}").format(" ".join(cmdline))) io.stderr(_("stdout:\n{}").format(stdout)) io.stderr(_("stderr:\n{}").format(stderr)) raise RuntimeError(_("`git {command}` failed in {dir}").format( command=cmdline[1], dir=repo_dir, )) return stdout.decode('utf-8').strip()
def git_command(cmdline, repo_dir): """ Runs the given git command line in the given directory. Returns stdout of the command. """ cmdline = ["git"] + cmdline io.debug(_("running '{}' in {}").format( " ".join(cmdline), repo_dir, )) git_process = Popen( cmdline, cwd=repo_dir, preexec_fn=setpgrp, stderr=PIPE, stdout=PIPE, ) stdout, stderr = git_process.communicate() # FIXME integrate this into Item._command_results if git_process.returncode != 0: io.stderr(_("failed command: {}").format(" ".join(cmdline))) io.stderr(_("stdout:\n{}").format(stdout)) io.stderr(_("stderr:\n{}").format(stderr)) raise RuntimeError( _("`git {command}` failed in {dir}").format( command=cmdline[1], dir=repo_dir, )) return stdout.decode('utf-8').strip()
def sdict(self): result = run_local([ "kubectl", "--context={}".format(self.node.kubectl_context), "--namespace={}".format(self.namespace), "get", "-o", "json", self.KUBECTL_RESOURCE_TYPE, self.resource_name, ]) if result.return_code == 0: full_json_response = json.loads(result.stdout) if full_json_response.get("status", {}).get("phase") == "Terminating": # this resource is currently being deleted, consider it gone return None return {'manifest': json.dumps(reduce_dict( full_json_response, json.loads(self.manifest), ), indent=4, sort_keys=True)} elif result.return_code == 1 and "NotFound" in result.stderr.decode('utf-8'): return None else: io.debug(result.stdout.decode('utf-8')) io.debug(result.stderr.decode('utf-8')) raise RuntimeError(_("error getting state of {}, check `bw --debug`".format(self.id)))
def _write_local_file(self): """ Makes the file contents available at the returned temporary path and performs local verification if necessary or requested. The calling method is responsible for cleaning up the file at the returned path (only if not a binary). """ with tempfile() as tmp_file: if self.attributes['content_type'] == 'binary': local_path = self.template else: local_path = tmp_file with open(local_path, 'wb') as f: f.write(self.content) if self.attributes['verify_with']: cmd = self.attributes['verify_with'].format(quote(local_path)) io.debug("calling local verify command for {i}: {c}".format( c=cmd, i=self.id)) if call(cmd, shell=True) == 0: io.debug("{i} passed local validation".format(i=self.id)) else: raise BundleError( _("{i} failed local validation using: {c}").format( c=cmd, i=self.id)) yield local_path
def _write_local_file(self): """ Makes the file contents available at the returned temporary path and performs local verification if necessary or requested. The calling method is responsible for cleaning up the file at the returned path (only if not a binary). """ with tempfile() as tmp_file: if self.attributes['content_type'] == 'binary': local_path = self.template else: local_path = tmp_file with open(local_path, 'wb') as f: f.write(self.content) if self.attributes['verify_with']: cmd = self.attributes['verify_with'].format(quote(local_path)) io.debug("calling local verify command for {i}: {c}".format(c=cmd, i=self.id)) if call(cmd, shell=True) == 0: io.debug("{i} passed local validation".format(i=self.id)) else: raise BundleError(_( "{i} failed local validation using: {c}" ).format(c=cmd, i=self.id)) yield local_path
def _repo_dir(self): if "://" in self.attributes['repo']: repo_dir = clone_to_dir(self.attributes['repo'], self.attributes['rev']) io.debug(_("registering {} for deletion on exit").format(repo_dir)) at_exit(rmtree, repo_dir) else: repo_dir = get_local_repo_path(self.node.repo.path, self.attributes['repo']) return repo_dir
def _repo_dir(self): if "://" in self.attributes['repo']: repo_dir, remove_dir = clone_to_dir(self.attributes['repo'], self.attributes['rev']) if remove_dir is not None: io.debug( _("registering {} for deletion on exit").format( remove_dir)) at_exit(rmtree, remove_dir, ignore_errors=True) else: repo_dir = get_local_repo_path(self.node.repo.path, self.attributes['repo']) return repo_dir
def _create_config(path): io.debug("writing initial config for Slack notifications to .slack.cfg") config = SafeConfigParser() config.add_section("configuration") config.set("configuration", "enabled", "unconfigured") config.set("configuration", "username", "your-slack-username") config.add_section("connection") config.set("connection", "url", "<insert URL from https://my.slack.com/services/new/incoming-webhook>") config.add_section("apply_notifications") config.set("apply_notifications", "enabled", "yes") config.set("apply_notifications", "allow_groups", "all") config.set("apply_notifications", "deny_groups", "local") with open(path, 'w') as f: config.write(f)
def _create_config(path): io.debug("writing initial config for Slack notifications to .slack.cfg") config = SafeConfigParser() config.add_section("configuration") config.set("configuration", "enabled", "unconfigured") config.set("configuration", "username", "your-slack-username") config.add_section("connection") config.set( "connection", "url", "<insert URL from https://my.slack.com/services/new/incoming-webhook>") config.add_section("apply_notifications") config.set("apply_notifications", "enabled", "yes") config.set("apply_notifications", "allow_groups", "all") config.set("apply_notifications", "deny_groups", "local") with open(path, 'w') as f: config.write(f)
def content_processor_mako(item): template = Template( item._template_content.encode('utf-8'), input_encoding='utf-8', lookup=TemplateLookup(directories=[item.item_data_dir, item.item_dir]), output_encoding=item.attributes['encoding'], ) io.debug("{node}:{bundle}:{item}: rendering with Mako...".format( bundle=item.bundle.name, item=item.id, node=item.node.name, )) start = datetime.now() try: content = template.render(item=item, bundle=item.bundle, node=item.node, repo=item.node.repo, **item.attributes['context']) except FaultUnavailable: raise except Exception as e: io.stderr("".join(format_exception(*exc_info()))) if isinstance(e, NameError) and str(e) == "Undefined": # Mako isn't very verbose here. Try to give a more useful # error message - even though we can't pinpoint the excat # location of the error. :/ e = _("Undefined variable (look for '${...}')") elif isinstance(e, KeyError): e = _("KeyError: {}").format(str(e)) raise TemplateError( _("Error while rendering template for {node}:{bundle}:{item}: {error}" ).format( bundle=item.bundle.name, error=e, item=item.id, node=item.node.name, )) duration = datetime.now() - start io.debug("{node}:{bundle}:{item}: rendered in {time}s".format( bundle=item.bundle.name, item=item.id, node=item.node.name, time=duration.total_seconds(), )) return content
def _get_config(repo_path): config_path = join(repo_path, ".slack.cfg") if not exists(config_path): _create_config(config_path) config = SafeConfigParser() config.read(config_path) if config.get("configuration", "enabled", fallback="unconfigured") == "unconfigured": io.stderr("Slack notifications not configured. Please edit .slack.cfg " "(it has already been created) and set enabled to 'yes' " "(or 'no' to silence this message and disable Slack notifications).") return None elif config.get("configuration", "enabled").lower() not in ("yes", "true", "1"): io.debug("Slack notifications not enabled in .slack.cfg, skipping...") return None elif not REQUESTS: io.stderr("Slack notifications need the requests library. " "You can usually install it with `pip install requests`.") return None return config
def apply_start(repo, target, nodes, interactive=False, **kwargs): config = _get_config(repo.path) if config is None or \ not config.has_section("apply_notifications") or \ not config.getboolean("apply_notifications", "enabled") or \ not _check_allowed_groups(config, nodes): return io.debug("posting apply start notification to Slack") _notify( config.get("connection", "url"), fallback="Starting bw apply to {target} as {user}".format( target=target, user=config.get("configuration", "username"), ), target=target, title=("Starting {interactive}interactive bw apply...").format( interactive="non-" if not interactive else ""), user=config.get("configuration", "username"), )
def apply_start(repo, target, nodes, interactive=False, **kwargs): config = _get_config(repo.path) if config is None or \ not config.has_section("apply_notifications") or \ not config.getboolean("apply_notifications", "enabled") or \ not _check_allowed_groups(config, nodes): return io.debug("posting apply start notification to Slack") _notify( config.get("connection", "url"), fallback="Starting bw apply to {target} as {user}".format( target=target, user=config.get("configuration", "username"), ), target=target, title=( "Starting {interactive}interactive bw apply..." ).format(interactive="non-" if not interactive else ""), user=config.get("configuration", "username"), )
def apply_end(repo, target, nodes, duration=None, **kwargs): config = _get_config(repo.path) if config is None or \ not config.has_section("apply_notifications") or \ not config.getboolean("apply_notifications", "enabled") or \ not _check_allowed_groups(config, nodes): return io.debug("posting apply end notification to Slack") _notify( config.get("connection", "url"), color="good", fallback="Finished bw apply to {target} as {user} after {duration}s.".format( duration=duration.total_seconds(), target=target, user=config.get("configuration", "username"), ), target=target, title="Finished bw apply after {}s.".format(duration.total_seconds()), user=config.get("configuration", "username"), )
def _get_result(self, interactive=False, interactive_default=True): if interactive is False and self.attributes['interactive'] is True: return (self.STATUS_SKIPPED, None) if self.triggered and not self.has_been_triggered: io.debug(_("skipping {} because it wasn't triggered").format(self.id)) return (self.STATUS_SKIPPED, None) if self.unless: unless_result = self.bundle.node.run( self.unless, may_fail=True, ) if unless_result.return_code == 0: io.debug(_("{node}:{bundle}:action:{name}: failed 'unless', not running").format( bundle=self.bundle.name, name=self.name, node=self.bundle.node.name, )) return (self.STATUS_SKIPPED, None) if ( interactive and self.attributes['interactive'] is not False and not io.ask( wrap_question( self.id, self.attributes['command'], _("Run action {}?").format( bold(self.name), ), ), interactive_default, ) ): return (self.STATUS_SKIPPED, None) try: self.run(interactive=interactive) return (self.STATUS_ACTION_SUCCEEDED, None) except ActionFailure: return (self.STATUS_FAILED, None)
def apply_end(repo, target, nodes, duration=None, **kwargs): config = _get_config(repo.path) if config is None or \ not config.has_section("apply_notifications") or \ not config.getboolean("apply_notifications", "enabled") or \ not _check_allowed_groups(config, nodes): return io.debug("posting apply end notification to Slack") _notify( config.get("connection", "url"), color="good", fallback="Finished bw apply to {target} as {user} after {duration}s.". format( duration=duration.total_seconds(), target=target, user=config.get("configuration", "username"), ), target=target, title="Finished bw apply after {}s.".format(duration.total_seconds()), user=config.get("configuration", "username"), )
def _get_result(self, autoskip_selector="", interactive=False, interactive_default=True): if self.covered_by_autoskip_selector(autoskip_selector): io.debug(_( "autoskip matches {item} on {node}" ).format(item=self.id, node=self.node.name)) return (self.STATUS_SKIPPED, [_("cmdline")]) if interactive is False and self.attributes['interactive'] is True: return (self.STATUS_SKIPPED, [_("interactive only")]) if self.triggered and not self.has_been_triggered: io.debug(_("skipping {} because it wasn't triggered").format(self.id)) return (self.STATUS_SKIPPED, [_("no trigger")]) if self.unless: with io.job(_(" {node} {bundle} {item} checking 'unless' condition...").format( bundle=self.bundle.name, item=self.id, node=self.node.name, )): unless_result = self.bundle.node.run( self.unless, may_fail=True, ) if unless_result.return_code == 0: io.debug(_("{node}:{bundle}:action:{name}: failed 'unless', not running").format( bundle=self.bundle.name, name=self.name, node=self.bundle.node.name, )) return (self.STATUS_SKIPPED, ["unless"]) if ( interactive and self.attributes['interactive'] is not False and not io.ask( wrap_question( self.id, self.attributes['command'], _("Run action {}?").format( bold(self.name), ), prefix="{x} {node} ".format( node=bold(self.node.name), x=blue("?"), ), ), interactive_default, epilogue="{x} {node}".format( node=bold(self.node.name), x=blue("?"), ), ) ): return (self.STATUS_SKIPPED, [_("interactive")]) try: self.run() return (self.STATUS_ACTION_SUCCEEDED, None) except ActionFailure: return (self.STATUS_FAILED, None)
def content_processor_jinja2(item): loader = FileSystemLoader(searchpath=[item.item_data_dir, item.item_dir]) env = Environment(loader=loader) template = env.from_string(item._template_content) io.debug("{node}:{bundle}:{item}: rendering with Jinja2...".format( bundle=item.bundle.name, item=item.id, node=item.node.name, )) start = datetime.now() try: content = template.render(item=item, bundle=item.bundle, node=item.node, repo=item.node.repo, **item.attributes['context']) except FaultUnavailable: raise except Exception as e: io.stderr("".join(format_exception(*exc_info()))) raise TemplateError( _("Error while rendering template for {node}:{bundle}:{item}: {error}" ).format( bundle=item.bundle.name, error=e, item=item.id, node=item.node.name, )) duration = datetime.now() - start io.debug("{node}:{bundle}:{item}: rendered in {time}s".format( bundle=item.bundle.name, item=item.id, node=item.node.name, time=duration.total_seconds(), )) return content.encode(item.attributes['encoding'])
def _get_paths_to_purge(self): result = self.run("find {} -maxdepth 1 -print0".format(quote( self.name))) for line in result.stdout.split(b"\0"): line = line.decode('utf-8') for item_type in ('directory', 'file', 'symlink'): for item in self.node.items: if (item.id == "{}:{}".format(item_type, line) or item.id.startswith("{}:{}/".format( item_type, line))): break else: continue break else: # this file or directory is not managed io.debug(("found unmanaged path below {dirpath} on {node}, " "marking for removal: {path}").format( dirpath=self.name, node=self.node.name, path=line, )) yield line
def _get_config(repo_path): config_path = join(repo_path, ".slack.cfg") if not exists(config_path): _create_config(config_path) config = SafeConfigParser() config.read(config_path) if config.get("configuration", "enabled", fallback="unconfigured") == "unconfigured": io.stderr( "Slack notifications not configured. Please edit .slack.cfg " "(it has already been created) and set enabled to 'yes' " "(or 'no' to silence this message and disable Slack notifications)." ) return None elif config.get("configuration", "enabled").lower() not in ("yes", "true", "1"): io.debug("Slack notifications not enabled in .slack.cfg, skipping...") return None elif not REQUESTS: io.stderr("Slack notifications need the requests library. " "You can usually install it with `pip install requests`.") return None return config
def _skip_with_soft_locks(self, mine, others): """ Returns True/False depending on whether the item should be skipped based on the given set of locks. """ for lock in mine: for selector in lock['items']: if self.covered_by_autoskip_selector(selector): io.debug(_("{item} on {node} whitelisted by lock {lock}").format( item=self.id, lock=lock['id'], node=self.node.name, )) return False for lock in others: for selector in lock['items']: if self.covered_by_autoskip_selector(selector): io.debug(_("{item} on {node} blacklisted by lock {lock}").format( item=self.id, lock=lock['id'], node=self.node.name, )) return True return False
def content_processor_mako(item): from mako.lookup import TemplateLookup from mako.template import Template template = Template( item._template_content.encode('utf-8'), input_encoding='utf-8', lookup=TemplateLookup(directories=[item.item_data_dir, item.item_dir]), output_encoding=item.attributes['encoding'], ) io.debug("{node}:{bundle}:{item}: rendering with Mako...".format( bundle=item.bundle.name, item=item.id, node=item.node.name, )) start = datetime.now() try: content = template.render( item=item, bundle=item.bundle, node=item.node, repo=item.node.repo, **item.attributes['context'] ) except FaultUnavailable: raise except Exception as e: io.debug("".join(format_exception(*exc_info()))) if isinstance(e, NameError) and str(e) == "Undefined": # Mako isn't very verbose here. Try to give a more useful # error message - even though we can't pinpoint the excat # location of the error. :/ e = _("Undefined variable (look for '${...}')") raise TemplateError(_( "Error while rendering template for {node}:{bundle}:{item}: {error}" ).format( bundle=item.bundle.name, error=e, item=item.id, node=item.node.name, )) duration = datetime.now() - start io.debug("{node}:{bundle}:{item}: rendered in {time}s".format( bundle=item.bundle.name, item=item.id, node=item.node.name, time=duration.total_seconds(), )) return content
def content_processor_jinja2(item): try: from jinja2 import Environment, FileSystemLoader except ImportError: raise TemplateError(_( "Unable to load Jinja2 (required to render {item}). " "You probably have to install it using `pip install Jinja2`." ).format(item=item.id)) loader = FileSystemLoader(searchpath=[item.item_data_dir, item.item_dir]) env = Environment(loader=loader) template = env.from_string(item._template_content) io.debug("{node}:{bundle}:{item}: rendering with Jinja2...".format( bundle=item.bundle.name, item=item.id, node=item.node.name, )) start = datetime.now() try: content = template.render( item=item, bundle=item.bundle, node=item.node, repo=item.node.repo, **item.attributes['context'] ) except FaultUnavailable: raise except Exception as e: io.debug("".join(format_exception(*exc_info()))) raise TemplateError(_( "Error while rendering template for {node}:{bundle}:{item}: {error}" ).format( bundle=item.bundle.name, error=e, item=item.id, node=item.node.name, )) duration = datetime.now() - start io.debug("{node}:{bundle}:{item}: rendered in {time}s".format( bundle=item.bundle.name, item=item.id, node=item.node.name, time=duration.total_seconds(), )) return content.encode(item.attributes['encoding'])
def _get_result( self, autoskip_selector="", my_soft_locks=(), other_peoples_soft_locks=(), interactive=False, interactive_default=True, ): if self.covered_by_autoskip_selector(autoskip_selector): io.debug( _("autoskip matches {item} on {node}").format( item=self.id, node=self.node.name)) return (self.STATUS_SKIPPED, [_("cmdline")]) if self._skip_with_soft_locks(my_soft_locks, other_peoples_soft_locks): return (self.STATUS_SKIPPED, [_("soft locked")]) if interactive is False and self.attributes['interactive'] is True: return (self.STATUS_SKIPPED, [_("interactive only")]) if self.triggered and not self.has_been_triggered: io.debug( _("skipping {} because it wasn't triggered").format(self.id)) return (self.STATUS_SKIPPED, [_("no trigger")]) if self.unless: with io.job( _(" {node} {bundle} {item} checking 'unless' condition..." ).format( bundle=self.bundle.name, item=self.id, node=self.node.name, )): unless_result = self.bundle.node.run( self.unless, may_fail=True, ) if unless_result.return_code == 0: io.debug( _("{node}:{bundle}:action:{name}: failed 'unless', not running" ).format( bundle=self.bundle.name, name=self.name, node=self.bundle.node.name, )) return (self.STATUS_SKIPPED, ["unless"]) question_body = "" if self.attributes['data_stdin'] is not None: question_body += "<" + _("data") + "> | " question_body += self.attributes['command'] if self.comment: question_body += format_comment(self.comment) if (interactive and self.attributes['interactive'] is not False and not io.ask( wrap_question( self.id, question_body, _("Run action {}?").format(bold(self.name), ), prefix="{x} {node} ".format( node=bold(self.node.name), x=blue("?"), ), ), interactive_default, epilogue="{x} {node}".format( node=bold(self.node.name), x=blue("?"), ), )): return (self.STATUS_SKIPPED, [_("interactive")]) try: self.run() return (self.STATUS_ACTION_SUCCEEDED, None) except ActionFailure as exc: return (self.STATUS_FAILED, [str(exc)])
def log_error(run_result): if run_result.return_code != 0: io.debug(run_result.stdout.decode('utf-8')) io.debug(run_result.stderr.decode('utf-8'))
def _get_result( self, autoskip_selector="", my_soft_locks=(), other_peoples_soft_locks=(), interactive=False, interactive_default=True, ): if self._faults_missing_for_attributes: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug(_( "skipping {item} on {node} because it is missing faults " "for these attributes: {attrs} " "(most of the time this means you're missing " "a required key in your .secrets.cfg)" ).format( attrs=", ".join(sorted(self._faults_missing_for_attributes)), item=self.id, node=self.node.name, )) return (self.STATUS_SKIPPED, self.SKIP_REASON_FAULT_UNAVAILABLE) if self.covered_by_autoskip_selector(autoskip_selector): io.debug(_( "autoskip matches {item} on {node}" ).format(item=self.id, node=self.node.name)) return (self.STATUS_SKIPPED, self.SKIP_REASON_CMDLINE) if self._skip_with_soft_locks(my_soft_locks, other_peoples_soft_locks): return (self.STATUS_SKIPPED, self.SKIP_REASON_SOFTLOCK) if interactive is False and self.attributes['interactive'] is True: return (self.STATUS_SKIPPED, self.SKIP_REASON_INTERACTIVE_ONLY) for item in self._precedes_items: if item._triggers_preceding_items(interactive=interactive): io.debug(_( "preceding item {item} on {node} has been triggered by {other_item}" ).format(item=self.id, node=self.node.name, other_item=item.id)) self.has_been_triggered = True break else: io.debug(_( "preceding item {item} on {node} has NOT been triggered by {other_item}" ).format(item=self.id, node=self.node.name, other_item=item.id)) if self.triggered and not self.has_been_triggered: io.debug(_("skipping {} because it wasn't triggered").format(self.id)) return (self.STATUS_SKIPPED, self.SKIP_REASON_NO_TRIGGER) if self.unless: with io.job(_("{node} {bundle} {item} checking 'unless' condition").format( bundle=bold(self.bundle.name), item=self.id, node=bold(self.node.name), )): unless_result = self.bundle.node.run( self.unless, may_fail=True, ) if unless_result.return_code == 0: io.debug(_("{node}:{bundle}:action:{name}: failed 'unless', not running").format( bundle=self.bundle.name, name=self.name, node=self.bundle.node.name, )) return (self.STATUS_SKIPPED, self.SKIP_REASON_UNLESS) question_body = "" if self.attributes['data_stdin'] is not None: question_body += "<" + _("data") + "> | " question_body += self.attributes['command'] if self.comment: question_body += format_comment(self.comment) if ( interactive and self.attributes['interactive'] is not False and not io.ask( wrap_question( self.id, question_body, _("Run action {}?").format( bold(self.name), ), prefix="{x} {node} ".format( node=bold(self.node.name), x=blue("?"), ), ), interactive_default, epilogue="{x} {node}".format( node=bold(self.node.name), x=blue("?"), ), ) ): return (self.STATUS_SKIPPED, self.SKIP_REASON_INTERACTIVE) try: self.run() return (self.STATUS_ACTION_SUCCEEDED, None) except ActionFailure as exc: return (self.STATUS_FAILED, [str(exc)])
def _get_result( self, autoskip_selector="", my_soft_locks=(), other_peoples_soft_locks=(), interactive=False, interactive_default=True, ): if self._faults_missing_for_attributes: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug( _("skipping {item} on {node} because it is missing faults " "for these attributes: {attrs} " "(most of the time this means you're missing " "a required key in your .secrets.cfg)").format( attrs=", ".join( sorted(self._faults_missing_for_attributes)), item=self.id, node=self.node.name, )) return (self.STATUS_SKIPPED, self.SKIP_REASON_FAULT_UNAVAILABLE) if self.covered_by_autoskip_selector(autoskip_selector): io.debug( _("autoskip matches {item} on {node}").format( item=self.id, node=self.node.name)) return (self.STATUS_SKIPPED, self.SKIP_REASON_CMDLINE) if self._skip_with_soft_locks(my_soft_locks, other_peoples_soft_locks): return (self.STATUS_SKIPPED, self.SKIP_REASON_SOFTLOCK) if interactive is False and self.attributes['interactive'] is True: return (self.STATUS_SKIPPED, self.SKIP_REASON_INTERACTIVE_ONLY) for item in self._precedes_items: if item._triggers_preceding_items(interactive=interactive): io.debug( _("preceding item {item} on {node} has been triggered by {other_item}" ).format(item=self.id, node=self.node.name, other_item=item.id)) self.has_been_triggered = True break else: io.debug( _("preceding item {item} on {node} has NOT been triggered by {other_item}" ).format(item=self.id, node=self.node.name, other_item=item.id)) if self.triggered and not self.has_been_triggered: io.debug( _("skipping {} because it wasn't triggered").format(self.id)) return (self.STATUS_SKIPPED, self.SKIP_REASON_NO_TRIGGER) if self.unless: with io.job( _("{node} {bundle} {item} checking 'unless' condition"). format( bundle=bold(self.bundle.name), item=self.id, node=bold(self.node.name), )): unless_result = self.bundle.node.run( self.unless, may_fail=True, ) if unless_result.return_code == 0: io.debug( _("{node}:{bundle}:action:{name}: failed 'unless', not running" ).format( bundle=self.bundle.name, name=self.name, node=self.bundle.node.name, )) return (self.STATUS_SKIPPED, self.SKIP_REASON_UNLESS) question_body = "" if self.attributes['data_stdin'] is not None: question_body += "<" + _("data") + "> | " question_body += self.attributes['command'] if self.comment: question_body += format_comment(self.comment) if (interactive and self.attributes['interactive'] is not False and not io.ask( wrap_question( self.id, question_body, _("Run action {}?").format(bold(self.name), ), prefix="{x} {node} ".format( node=bold(self.node.name), x=blue("?"), ), ), interactive_default, epilogue="{x} {node}".format( node=bold(self.node.name), x=blue("?"), ), )): return (self.STATUS_SKIPPED, self.SKIP_REASON_INTERACTIVE) try: self.run() return (self.STATUS_ACTION_SUCCEEDED, None) except ActionFailure as exc: return (self.STATUS_FAILED, [str(exc)])
def apply( self, autoskip_selector="", my_soft_locks=(), other_peoples_soft_locks=(), interactive=False, interactive_default=True, ): self.node.repo.hooks.item_apply_start( self.node.repo, self.node, self, ) keys_to_fix = None status_code = None status_before = None status_after = None start_time = datetime.now() if self.covered_by_autoskip_selector(autoskip_selector): io.debug( _("autoskip matches {item} on {node}").format( item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED keys_to_fix = [_("cmdline")] if self._skip_with_soft_locks(my_soft_locks, other_peoples_soft_locks): status_code = self.STATUS_SKIPPED keys_to_fix = [_("soft locked")] if self.triggered and not self.has_been_triggered and status_code is None: io.debug( _("skipping {item} on {node} because it wasn't triggered"). format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED keys_to_fix = [_("not triggered")] if status_code is None and self.cached_unless_result and status_code is None: io.debug( _("'unless' for {item} on {node} succeeded, not fixing"). format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED keys_to_fix = ["unless"] if self._faults_missing_for_attributes and status_code is None: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug( _("skipping {item} on {node} because it is missing faults " "for these attributes: {attrs} " "(most of the time this means you're missing " "a required key in your .secrets.cfg)").format( attrs=", ".join( sorted(self._faults_missing_for_attributes)), item=self.id, node=self.node.name, )) status_code = self.STATUS_SKIPPED keys_to_fix = [_("Fault unavailable")] if status_code is None: try: status_before = self.cached_status except FaultUnavailable: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug( _("skipping {item} on {node} because it is missing Faults " "(most of the time this means you're missing " "a required key in your .secrets.cfg)").format( item=self.id, node=self.node.name, )) status_code = self.STATUS_SKIPPED keys_to_fix = [_("Fault unavailable")] else: if status_before.correct: status_code = self.STATUS_OK if status_code is None: keys_to_fix = self.display_keys( copy(self.cached_cdict), copy(status_before.sdict), status_before.keys_to_fix[:], ) if not interactive: with io.job( _(" {node} {bundle} {item} fixing...").format( bundle=self.bundle.name, item=self.id, node=self.node.name, )): self.fix(status_before) else: if status_before.must_be_created: question_text = _("Doesn't exist. Will be created.") elif status_before.must_be_deleted: question_text = _("Found on node. Will be removed.") else: cdict, sdict = self.display_dicts( copy(self.cached_cdict), copy(status_before.sdict), keys_to_fix, ) question_text = self.ask(cdict, sdict, keys_to_fix) if self.comment: question_text += format_comment(self.comment) question = wrap_question( self.id, question_text, _("Fix {}?").format(bold(self.id)), prefix="{x} {node} ".format( node=bold(self.node.name), x=blue("?"), ), ) answer = io.ask( question, interactive_default, epilogue="{x} {node}".format( node=bold(self.node.name), x=blue("?"), ), ) if answer: with io.job( _(" {node} {bundle} {item} fixing...").format( bundle=self.bundle.name, item=self.id, node=self.node.name, )): self.fix(status_before) else: status_code = self.STATUS_SKIPPED keys_to_fix = [_("interactive")] if status_code is None: status_after = self.get_status(cached=False) status_code = self.STATUS_FIXED if status_after.correct else self.STATUS_FAILED if status_code == self.STATUS_SKIPPED: # can't use else for this because status_before is None changes = keys_to_fix elif status_before.must_be_created: changes = True elif status_before.must_be_deleted: changes = False elif status_code == self.STATUS_FAILED: changes = self.display_keys( self.cached_cdict.copy(), status_after.sdict.copy(), status_after.keys_to_fix[:], ) else: changes = keys_to_fix self.node.repo.hooks.item_apply_end( self.node.repo, self.node, self, duration=datetime.now() - start_time, status_code=status_code, status_before=status_before, status_after=status_after, ) return (status_code, changes)
def apply( self, autoskip_selector="", my_soft_locks=(), other_peoples_soft_locks=(), interactive=False, interactive_default=True, ): self.node.repo.hooks.item_apply_start( self.node.repo, self.node, self, ) status_code = None status_before = None status_after = None start_time = datetime.now() if self.covered_by_autoskip_selector(autoskip_selector): io.debug(_( "autoskip matches {item} on {node}" ).format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_CMDLINE if self._skip_with_soft_locks(my_soft_locks, other_peoples_soft_locks): status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_SOFTLOCK for item in self._precedes_items: if item._triggers_preceding_items(interactive=interactive): io.debug(_( "preceding item {item} on {node} has been triggered by {other_item}" ).format(item=self.id, node=self.node.name, other_item=item.id)) self.has_been_triggered = True break else: io.debug(_( "preceding item {item} on {node} has NOT been triggered by {other_item}" ).format(item=self.id, node=self.node.name, other_item=item.id)) if self.triggered and not self.has_been_triggered and status_code is None: io.debug(_( "skipping {item} on {node} because it wasn't triggered" ).format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_NO_TRIGGER if status_code is None and self.cached_unless_result and status_code is None: io.debug(_( "'unless' for {item} on {node} succeeded, not fixing" ).format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_UNLESS if self._faults_missing_for_attributes and status_code is None: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug(_( "skipping {item} on {node} because it is missing faults " "for these attributes: {attrs} " "(most of the time this means you're missing " "a required key in your .secrets.cfg)" ).format( attrs=", ".join(sorted(self._faults_missing_for_attributes)), item=self.id, node=self.node.name, )) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_FAULT_UNAVAILABLE if status_code is None: try: status_before = self.cached_status except FaultUnavailable: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug(_( "skipping {item} on {node} because it is missing Faults " "(most of the time this means you're missing " "a required key in your .secrets.cfg)" ).format( item=self.id, node=self.node.name, )) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_FAULT_UNAVAILABLE else: if status_before.correct: status_code = self.STATUS_OK if status_code is None: if not interactive: with io.job(_("{node} {bundle} {item}").format( bundle=bold(self.bundle.name), item=self.id, node=bold(self.node.name), )): self.fix(status_before) else: if status_before.must_be_created: question_text = _("Doesn't exist. Will be created.") elif status_before.must_be_deleted: question_text = _("Found on node. Will be removed.") else: question_text = self.ask( status_before.display_cdict, status_before.display_sdict, status_before.display_keys_to_fix, ) if self.comment: question_text += format_comment(self.comment) question = wrap_question( self.id, question_text, _("Fix {}?").format(bold(self.id)), prefix="{x} {node} ".format( node=bold(self.node.name), x=blue("?"), ), ) answer = io.ask( question, interactive_default, epilogue="{x} {node}".format( node=bold(self.node.name), x=blue("?"), ), ) if answer: with io.job(_("{node} {bundle} {item}").format( bundle=bold(self.bundle.name), item=self.id, node=bold(self.node.name), )): self.fix(status_before) else: status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_INTERACTIVE if status_code is None: status_after = self.get_status(cached=False) status_code = self.STATUS_FIXED if status_after.correct else self.STATUS_FAILED if status_code == self.STATUS_OK: details = None elif status_code == self.STATUS_SKIPPED: details = skip_reason elif status_before.must_be_created: details = True elif status_before.must_be_deleted: details = False elif status_code == self.STATUS_FAILED: details = status_after.display_keys_to_fix else: details = status_before.display_keys_to_fix self.node.repo.hooks.item_apply_end( self.node.repo, self.node, self, duration=datetime.now() - start_time, status_code=status_code, status_before=status_before, status_after=status_after, ) return (status_code, details)
def clone_to_dir(remote_url, rev): """ Clones the given URL to a temporary directory, using a shallow clone if the given revision is definitely not a commit hash. Clones to the base directory $BW_GIT_DEPLOY_CACHE if set. Returns the path to the repo directory and to another directory to be deleted when the process exits (may be None). """ repo_dir_hashed = md5(remote_url.encode('UTF-8')).hexdigest() cache_dir_env = getenv("BW_GIT_DEPLOY_CACHE") if cache_dir_env: # Do not remove this, because it was not created by us. remove_dir = None repo_dir = join(cache_dir_env, repo_dir_hashed) lock_dir = join(cache_dir_env, repo_dir_hashed + ".bw_lock") else: remove_dir = join(gettempdir(), "bw-git-cache-{}".format(getpid())) repo_dir = join(remove_dir, repo_dir_hashed) lock_dir = join(remove_dir, repo_dir_hashed + ".bw_lock") makedirs(repo_dir, exist_ok=True) io.debug( _("{pid}: lock_dir {lock_dir}").format(lock_dir=lock_dir, pid=getpid())) io.debug( _("{pid}: remove_dir {remove_dir}").format(remove_dir=remove_dir, pid=getpid())) io.debug( _("{pid}: repo_dir {repo_dir}").format(repo_dir=repo_dir, pid=getpid())) if is_ref(rev) and not remote_url.startswith('http'): git_cmdline = [ "clone", "--bare", "--depth", "1", "--no-single-branch", remote_url, "." ] else: git_cmdline = ["clone", "--bare", remote_url, "."] # Use a lock directory to cooperate with other running instances # of bw (in cases where $BW_GIT_DEPLOY_CACHE is used). while True: try: mkdir(lock_dir) io.debug( _("{pid}: Have lock on {lock_dir}").format( lock_dir=lock_dir, pid=getpid(), )) break except FileExistsError: io.debug( _("{pid}: Waiting for lock on {lock_dir} ...").format( lock_dir=lock_dir, pid=getpid(), )) sleep(1) try: # We now have a lock, but another process may have cloned # the repo in the meantime. (It is vital to use a git command # here, which does not traverse to parent directories.) try: git_command( ["rev-parse", "--resolve-git-dir", "."], repo_dir, error_messages=False, ) io.debug( _("{pid}: Repo already existed in {repo_dir}").format( repo_dir=repo_dir, pid=getpid(), )) except RuntimeError: git_command(git_cmdline, repo_dir) io.debug( _("{pid}: Cloned repo to {repo_dir}").format( repo_dir=repo_dir, pid=getpid(), )) finally: rmdir(lock_dir) io.debug( _("{pid}: Released lock on {lock_dir}").format( lock_dir=lock_dir, pid=getpid(), )) return repo_dir, remove_dir
def apply( self, autoskip_selector="", autoonly_selector="", my_soft_locks=(), other_peoples_soft_locks=(), interactive=False, interactive_default=True, ): self.node.repo.hooks.item_apply_start( self.node.repo, self.node, self, ) status_code = None status_before = None status_after = None start_time = datetime.now() if not self.covered_by_autoonly_selector(autoonly_selector): io.debug( _("autoonly does not match {item} on {node}").format( item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_CMDLINE if self.covered_by_autoskip_selector(autoskip_selector): io.debug( _("autoskip matches {item} on {node}").format( item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_CMDLINE if self._skip_with_soft_locks(my_soft_locks, other_peoples_soft_locks): status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_SOFTLOCK for item in self._precedes_items: if item._triggers_preceding_items(interactive=interactive): io.debug( _("preceding item {item} on {node} has been triggered by {other_item}" ).format(item=self.id, node=self.node.name, other_item=item.id)) self.has_been_triggered = True break else: io.debug( _("preceding item {item} on {node} has NOT been triggered by {other_item}" ).format(item=self.id, node=self.node.name, other_item=item.id)) if self.triggered and not self.has_been_triggered and status_code is None: io.debug( _("skipping {item} on {node} because it wasn't triggered"). format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_NO_TRIGGER if status_code is None and self.cached_unless_result and status_code is None: io.debug( _("'unless' for {item} on {node} succeeded, not fixing"). format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_UNLESS if self._faults_missing_for_attributes and status_code is None: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug( _("skipping {item} on {node} because it is missing faults " "for these attributes: {attrs} " "(most of the time this means you're missing " "a required key in your .secrets.cfg)").format( attrs=", ".join( sorted(self._faults_missing_for_attributes)), item=self.id, node=self.node.name, )) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_FAULT_UNAVAILABLE if status_code is None: try: status_before = self.cached_status except FaultUnavailable: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug( _("skipping {item} on {node} because it is missing Faults " "(most of the time this means you're missing " "a required key in your .secrets.cfg)").format( item=self.id, node=self.node.name, )) status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_FAULT_UNAVAILABLE else: if status_before.correct: status_code = self.STATUS_OK if status_code is None: if not interactive: with io.job( _("{node} {bundle} {item}").format( bundle=bold(self.bundle.name), item=self.id, node=bold(self.node.name), )): self.fix(status_before) else: if status_before.must_be_created: question_text = _("Doesn't exist. Will be created.") elif status_before.must_be_deleted: question_text = _("Found on node. Will be removed.") else: question_text = self.ask( status_before.display_cdict, status_before.display_sdict, status_before.display_keys_to_fix, ) if self.comment: question_text += format_comment(self.comment) question = wrap_question( self.id, question_text, _("Fix {}?").format(bold(self.id)), prefix="{x} {node} ".format( node=bold(self.node.name), x=blue("?"), ), ) answer = io.ask( question, interactive_default, epilogue="{x} {node}".format( node=bold(self.node.name), x=blue("?"), ), ) if answer: with io.job( _("{node} {bundle} {item}").format( bundle=bold(self.bundle.name), item=self.id, node=bold(self.node.name), )): self.fix(status_before) else: status_code = self.STATUS_SKIPPED skip_reason = self.SKIP_REASON_INTERACTIVE if status_code is None: status_after = self.get_status(cached=False) status_code = self.STATUS_FIXED if status_after.correct else self.STATUS_FAILED if status_code == self.STATUS_OK: details = None elif status_code == self.STATUS_SKIPPED: details = skip_reason elif status_before.must_be_created: details = True elif status_before.must_be_deleted: details = False elif status_code == self.STATUS_FAILED: details = status_after.display_keys_to_fix else: details = status_before.display_keys_to_fix self.node.repo.hooks.item_apply_end( self.node.repo, self.node, self, duration=datetime.now() - start_time, status_code=status_code, status_before=status_before, status_after=status_after, ) return (status_code, details)
def apply( self, autoskip_selector="", my_soft_locks=(), other_peoples_soft_locks=(), interactive=False, interactive_default=True, ): self.node.repo.hooks.item_apply_start(self.node.repo, self.node, self) keys_to_fix = None status_code = None status_before = None status_after = None start_time = datetime.now() if self.covered_by_autoskip_selector(autoskip_selector): io.debug(_("autoskip matches {item} on {node}").format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED keys_to_fix = [_("cmdline")] if self._skip_with_soft_locks(my_soft_locks, other_peoples_soft_locks): status_code = self.STATUS_SKIPPED keys_to_fix = [_("soft locked")] if self.triggered and not self.has_been_triggered and status_code is None: io.debug( _("skipping {item} on {node} because it wasn't triggered").format(item=self.id, node=self.node.name) ) status_code = self.STATUS_SKIPPED keys_to_fix = [_("not triggered")] if status_code is None and self.cached_unless_result and status_code is None: io.debug(_("'unless' for {item} on {node} succeeded, not fixing").format(item=self.id, node=self.node.name)) status_code = self.STATUS_SKIPPED keys_to_fix = ["unless"] if self._faults_missing_for_attributes and status_code is None: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug( _( "skipping {item} on {node} because it is missing faults " "for these attributes: {attrs} " "(most of the time this means you're missing " "a required key in your .secrets.cfg)" ).format( attrs=", ".join(sorted(self._faults_missing_for_attributes)), item=self.id, node=self.node.name ) ) status_code = self.STATUS_SKIPPED keys_to_fix = [_("Fault unavailable")] if status_code is None: try: status_before = self.cached_status except FaultUnavailable: if self.error_on_missing_fault: self._raise_for_faults() else: io.debug( _( "skipping {item} on {node} because it is missing Faults " "(most of the time this means you're missing " "a required key in your .secrets.cfg)" ).format(item=self.id, node=self.node.name) ) status_code = self.STATUS_SKIPPED keys_to_fix = [_("Fault unavailable")] else: if status_before.correct: status_code = self.STATUS_OK if status_code is None: keys_to_fix = self.display_keys( copy(self.cached_cdict), copy(status_before.sdict), status_before.keys_to_fix[:] ) if not interactive: with io.job( _(" {node} {bundle} {item} fixing...").format( bundle=self.bundle.name, item=self.id, node=self.node.name ) ): self.fix(status_before) else: if status_before.must_be_created: question_text = _("Doesn't exist. Will be created.") elif status_before.must_be_deleted: question_text = _("Found on node. Will be removed.") else: cdict, sdict = self.display_dicts(copy(self.cached_cdict), copy(status_before.sdict), keys_to_fix) question_text = self.ask(cdict, sdict, keys_to_fix) question = wrap_question( self.id, question_text, _("Fix {}?").format(bold(self.id)), prefix="{x} {node} ".format(node=bold(self.node.name), x=blue("?")), ) answer = io.ask( question, interactive_default, epilogue="{x} {node}".format(node=bold(self.node.name), x=blue("?")) ) if answer: with io.job( _(" {node} {bundle} {item} fixing...").format( bundle=self.bundle.name, item=self.id, node=self.node.name ) ): self.fix(status_before) else: status_code = self.STATUS_SKIPPED keys_to_fix = [_("interactive")] if status_code is None: status_after = self.get_status(cached=False) status_code = self.STATUS_FIXED if status_after.correct else self.STATUS_FAILED if status_code == self.STATUS_SKIPPED: # can't use else for this because status_before is None changes = keys_to_fix elif status_before.must_be_created: changes = True elif status_before.must_be_deleted: changes = False elif status_code == self.STATUS_FAILED: changes = self.display_keys( self.cached_cdict.copy(), status_after.sdict.copy(), status_after.keys_to_fix[:] ) else: changes = keys_to_fix self.node.repo.hooks.item_apply_end( self.node.repo, self.node, self, duration=datetime.now() - start_time, status_code=status_code, status_before=status_before, status_after=status_after, ) return (status_code, changes)