def cibadmin(command_args, timeout=120): result = _cibadmin(command_args, timeout, True) if result.rc != 0: raise AgentShell.CommandExecutionError(result, command_args) return result
def cibadmin(command_args, timeout=120): assert timeout > 0, 'timeout must be greater than zero' # I think these are "errno" values, but I'm not positive # but going forward, any additions to this should try to be informative # about the type of exit code and why it's OK to retry RETRY_CODES = { 10: "something unknown", 41: "something unknown", 62: "Timer expired", 107: "Transport endpoint is not connected" } command_args.insert(0, 'cibadmin') # NB: This isn't a "true" timeout, in that it won't forcibly stop the # subprocess after a timeout. We'd need more invasive changes to # shell._run() for that. for _ in util.wait(timeout): result = AgentShell.run(command_args) if result.rc == 0: return result elif result.rc not in RETRY_CODES: break # Add some harmless diagnostics which will be visible in the logs. AgentShell.run(['service', 'corosync', 'status']) AgentShell.run(['service', 'pacemaker', 'status']) if result.rc in RETRY_CODES: raise PacemakerError( "%s timed out after %d seconds: rc: %s, stderr: %s" % (" ".join(command_args), timeout, result.rc, result.stderr)) else: raise AgentShell.CommandExecutionError(result, command_args)
def mock_list_params(self, path): fl = glob(path) if fl: daemon_log.info("mock_list_params: " + path) return fl else: raise AgentShell.CommandExecutionError( AgentShell.RunResult( 2, "", "error: get_param: param_path '" + path + "': No such file or directory", 0, ), ["lctl", "get_param", "-N", path], )
def mock_get_param_raw(self, path): param = path.replace("/", ".") daemon_log.info("mock_get_params_lines: " + param) data = "" for fn in glob(param): with open(fn, "r") as content_file: data += content_file.read() if data: return data else: raise AgentShell.CommandExecutionError( AgentShell.RunResult( 2, "", "error: get_param: param_path '" + param + "': No such file or directory", 0, ), ["lctl", "get_param", "-n", path], )
def mock_get_param_lines(self, path, filter_f=None): param = path.replace("/", ".") daemon_log.info("mock_get_params_lines: " + param) flist = glob(param) if not flist: raise AgentShell.CommandExecutionError( AgentShell.RunResult( 2, "", "error: get_param: param_path '" + param + "': No such file or directory", 0, ), ["lctl", "get_param", "-n", path], ) for fn in flist: with open(fn, "r") as content_file: for line in content_file: if filter_f: if filter_f(line): yield line.strip() else: yield line.strip()
def yum_util(action, packages=[], fromrepo=None, enablerepo=None, narrow_updates=False): ''' A wrapper to perform yum actions in encapsulated way. :param action: clean, install, remove, update, requires etc :param packages: Packages to install or remove :param fromrepo: The repo the action should be carried out from, others are disabled. :param enablerepo: The repo to enable for the action, others are not disabled or enabled :param narrow_updates: ? :return: No return but throws CommandExecutionError on error. ''' if fromrepo and enablerepo: raise ValueError("Cannot provide fromrepo and enablerepo simultaneously") repo_arg = [] valid_rc_values = [0] # Some errors values other than 0 are valid. tries = 2 if fromrepo: repo_arg = ['--disablerepo=*'] + ['--enablerepo=%s' % r for r in fromrepo] elif enablerepo: repo_arg = ['--enablerepo=%s' % r for r in enablerepo] if narrow_updates and action == 'query': repo_arg.extend(['--upgrades']) if action == 'clean': cmd = ['dnf', 'clean', 'all'] + (repo_arg if repo_arg else ["--enablerepo=*"]) elif action == 'install': cmd = ['dnf', 'install', '--allowerasing', '-y', '--exclude', 'kernel-debug'] + \ repo_arg + list(packages) elif action == 'remove': cmd = ['dnf', 'remove', '-y'] + repo_arg + list(packages) elif action == 'update': cmd = ['dnf', 'update', '--allowerasing', '-y', '--exclude', 'kernel-debug'] + \ repo_arg + list(packages) elif action == 'requires': cmd = ['dnf', 'repoquery', '--requires'] + repo_arg + list(packages) elif action == 'query': cmd = ['dnf', 'repoquery', '--available'] + repo_arg + list(packages) elif action == 'repoquery': cmd = ['dnf', 'repoquery', '--available'] + repo_arg + ['--queryformat=%{EPOCH} %{NAME} %{VERSION} %{RELEASE} %{ARCH}'] elif action == 'check-update': cmd = ['dnf', 'repoquery', '--queryformat=%{name} %{version}-%{release}.' '%{arch} %{repoid}', '--upgrades'] + repo_arg + \ list(packages) else: raise RuntimeError('Unknown yum util action %s' % action) # This is a poor solution for HYD-3855 but not one that carries any known cost. # We sometimes see intermittent failures in test, and possibly out of test, that occur # 1 in 50 (estimate) times. yum commands are idempotent and so trying the command three # times has no downside and changes the estimated chance of fail to 1 in 12500. for hyd_3885 in range(tries, -1, -1): result = AgentShell.run(cmd) if result.rc in valid_rc_values: return result.stdout else: # if we were trying to install, clean the metadata before # trying again if action == 'install': AgentShell.run(['dnf', 'clean', 'metadata']) daemon_log.info("HYD-3885 Retrying yum command '%s'" % " ".join(cmd)) if hyd_3885 == 0: daemon_log.info("HYD-3885 Retry yum command failed '%s'" % " ".join(cmd)) raise AgentShell.CommandExecutionError(result, cmd) # Out of retries so raise for the caller..
def yum_util(action, packages=[], fromrepo=None, enablerepo=None, narrow_updates=False): ''' A wrapper to perform yum actions in encapsulated way. :param action: clean, install, remove, update, requires etc :param packages: Packages to install or remove :param fromrepo: The repo the action should be carried out from, others are disabled. :param enablerepo: The repo to enable for the action, others are not disabled or enabled :param narrow_updates: ? :return: No return but throws CommandExecutionError on error. ''' if fromrepo and enablerepo: raise ValueError( "Cannot provide fromrepo and enablerepo simultaneously") repo_arg = [] valid_rc_values = [0] # Some errors values other than 0 are valid. if fromrepo: repo_arg = ['--disablerepo=*', '--enablerepo=%s' % ','.join(fromrepo)] elif enablerepo: repo_arg = ['--enablerepo=%s' % ','.join(enablerepo)] if narrow_updates and action == 'query': repo_arg.extend(['--pkgnarrow=updates', '-a']) if action == 'clean': cmd = ['yum', 'clean', 'all' ] + (repo_arg if repo_arg else ["--enablerepo=*"]) elif action == 'install': cmd = ['yum', 'install', '-y'] + repo_arg + list(packages) elif action == 'remove': cmd = ['yum', 'remove', '-y'] + repo_arg + list(packages) elif action == 'update': cmd = ['yum', 'update', '-y'] + repo_arg + list(packages) elif action == 'requires': cmd = ['repoquery', '--requires'] + repo_arg + list(packages) elif action == 'query': cmd = ['repoquery'] + repo_arg + list(packages) elif action == 'repoquery': cmd = ['repoquery'] + repo_arg + [ '-a', '--qf=%{EPOCH} %{NAME} %{VERSION} %{RELEASE} %{ARCH}' ] elif action == 'check-update': cmd = ['yum', 'check-update', '-q'] + repo_arg + list(packages) valid_rc_values = [ 0, 100 ] # check-update returns 100 if updates are available. else: raise RuntimeError('Unknown yum util action %s' % action) # This is a poor solution for HYD-3855 but not one that carries any known cost. # We sometimes see intermittent failures in test, and possibly out of test, that occur # 1 in 50 (estimate) times. yum commands are idempotent and so trying the command three # times has no downside and changes the estimated chance of fail to 1 in 12500. for hyd_3885 in range(2, -1, -1): rc, stdout, stderr = AgentShell.run_old(cmd) if rc in valid_rc_values: return stdout else: daemon_log.info("HYD-3885 Retrying yum command '%s'" % " ".join(cmd)) if hyd_3885 == 0: daemon_log.info("HYD-3885 Retry yum command failed '%s'" % " ".join(cmd)) raise AgentShell.CommandExecutionError( AgentShell.RunResult(rc, stdout, stderr, False), cmd) # Out of retries so raise for the caller..
def yum_util(action, packages=[], fromrepo=None, enablerepo=None, narrow_updates=False): """ A wrapper to perform yum actions in encapsulated way. :param action: clean, install, remove, update, requires etc :param packages: Packages to install or remove :param fromrepo: The repo the action should be carried out from, others are disabled. :param enablerepo: The repo to enable for the action, others are not disabled or enabled :param narrow_updates: ? :return: No return but throws CommandExecutionError on error. """ if fromrepo and enablerepo: raise ValueError( "Cannot provide fromrepo and enablerepo simultaneously") repo_arg = [] valid_rc_values = [0] # Some errors values other than 0 are valid. tries = 2 if fromrepo: repo_arg = ["--disablerepo=*" ] + ["--enablerepo=%s" % r for r in fromrepo] elif enablerepo: repo_arg = ["--enablerepo=%s" % r for r in enablerepo] if narrow_updates and action == "query": repo_arg.extend(["--upgrades"]) if action == "clean": cmd = ["yum", "clean", "all" ] + (repo_arg if repo_arg else ["--enablerepo=*"]) elif action == "install": cmd = (["yum", "install", "-y", "--exclude", "kernel-debug"] + repo_arg + list(packages)) elif action == "remove": cmd = ["yum", "remove", "-y"] + repo_arg + list(packages) elif action == "update": cmd = (["yum", "update", "-y", "--exclude", "kernel-debug"] + repo_arg + list(packages)) elif action == "requires": cmd = ["repoquery", "--requires"] + repo_arg + list(packages) elif action == "query": cmd = ["repoquery"] + repo_arg + list(packages) elif action == "repoquery": cmd = (["repoquery", "--show-duplicates"] + repo_arg + [ "--queryformat=%{EPOCH} %{NAME} " "%{VERSION} %{RELEASE} %{ARCH}" ]) else: raise RuntimeError("Unknown yum util action %s" % action) # This is a poor solution for HYD-3855 but not one that carries any known cost. # We sometimes see intermittent failures in test, and possibly out of test, that occur # 1 in 50 (estimate) times. yum commands are idempotent and so trying the command three # times has no downside and changes the estimated chance of fail to 1 in 12500. for hyd_3885 in range(tries, -1, -1): result = AgentShell.run(cmd) if result.rc in valid_rc_values: return result.stdout else: # if we were trying to install, clean the metadata before # trying again if action == "install": AgentShell.run(["yum", "clean", "metadata"]) daemon_log.info("HYD-3885 Retrying yum command '%s'" % " ".join(cmd)) if hyd_3885 == 0: daemon_log.info("HYD-3885 Retry yum command failed '%s'" % " ".join(cmd)) raise AgentShell.CommandExecutionError( result, cmd) # Out of retries so raise for the caller..