def cibadmin(command_args, timeout=120):
result = _cibadmin(command_args, timeout, True)
if result.rc != 0:
raise AgentShell.CommandExecutionError(result, command_args)
return result
def cibadmin(command_args, timeout=120):
assert timeout > 0, 'timeout must be greater than zero'
# I think these are "errno" values, but I'm not positive
# but going forward, any additions to this should try to be informative
# about the type of exit code and why it's OK to retry
RETRY_CODES = {
10: "something unknown",
41: "something unknown",
62: "Timer expired",
107: "Transport endpoint is not connected"
}
command_args.insert(0, 'cibadmin')
# NB: This isn't a "true" timeout, in that it won't forcibly stop the
# subprocess after a timeout. We'd need more invasive changes to
# shell._run() for that.
for _ in util.wait(timeout):
result = AgentShell.run(command_args)
if result.rc == 0:
return result
elif result.rc not in RETRY_CODES:
break
# Add some harmless diagnostics which will be visible in the logs.
AgentShell.run(['service', 'corosync', 'status'])
AgentShell.run(['service', 'pacemaker', 'status'])
if result.rc in RETRY_CODES:
raise PacemakerError(
"%s timed out after %d seconds: rc: %s, stderr: %s" %
(" ".join(command_args), timeout, result.rc, result.stderr))
else:
raise AgentShell.CommandExecutionError(result, command_args)
def mock_list_params(self, path):
fl = glob(path)
if fl:
daemon_log.info("mock_list_params: " + path)
return fl
else:
raise AgentShell.CommandExecutionError(
AgentShell.RunResult(
2,
"",
"error: get_param: param_path '" + path +
"': No such file or directory",
0,
),
["lctl", "get_param", "-N", path],
)
def mock_get_param_raw(self, path):
param = path.replace("/", ".")
daemon_log.info("mock_get_params_lines: " + param)
data = ""
for fn in glob(param):
with open(fn, "r") as content_file:
data += content_file.read()
if data:
return data
else:
raise AgentShell.CommandExecutionError(
AgentShell.RunResult(
2,
"",
"error: get_param: param_path '" + param +
"': No such file or directory",
0,
),
["lctl", "get_param", "-n", path],
)
def mock_get_param_lines(self, path, filter_f=None):
param = path.replace("/", ".")
daemon_log.info("mock_get_params_lines: " + param)
flist = glob(param)
if not flist:
raise AgentShell.CommandExecutionError(
AgentShell.RunResult(
2,
"",
"error: get_param: param_path '" + param +
"': No such file or directory",
0,
),
["lctl", "get_param", "-n", path],
)
for fn in flist:
with open(fn, "r") as content_file:
for line in content_file:
if filter_f:
if filter_f(line):
yield line.strip()
else:
yield line.strip()
def yum_util(action, packages=[], fromrepo=None, enablerepo=None, narrow_updates=False):
'''
A wrapper to perform yum actions in encapsulated way.
:param action: clean, install, remove, update, requires etc
:param packages: Packages to install or remove
:param fromrepo: The repo the action should be carried out from, others are disabled.
:param enablerepo: The repo to enable for the action, others are not disabled or enabled
:param narrow_updates: ?
:return: No return but throws CommandExecutionError on error.
'''
if fromrepo and enablerepo:
raise ValueError("Cannot provide fromrepo and enablerepo simultaneously")
repo_arg = []
valid_rc_values = [0] # Some errors values other than 0 are valid.
tries = 2
if fromrepo:
repo_arg = ['--disablerepo=*'] + ['--enablerepo=%s' % r for r in fromrepo]
elif enablerepo:
repo_arg = ['--enablerepo=%s' % r for r in enablerepo]
if narrow_updates and action == 'query':
repo_arg.extend(['--upgrades'])
if action == 'clean':
cmd = ['dnf', 'clean', 'all'] + (repo_arg if repo_arg else ["--enablerepo=*"])
elif action == 'install':
cmd = ['dnf', 'install', '--allowerasing', '-y', '--exclude', 'kernel-debug'] + \
repo_arg + list(packages)
elif action == 'remove':
cmd = ['dnf', 'remove', '-y'] + repo_arg + list(packages)
elif action == 'update':
cmd = ['dnf', 'update', '--allowerasing', '-y', '--exclude', 'kernel-debug'] + \
repo_arg + list(packages)
elif action == 'requires':
cmd = ['dnf', 'repoquery', '--requires'] + repo_arg + list(packages)
elif action == 'query':
cmd = ['dnf', 'repoquery', '--available'] + repo_arg + list(packages)
elif action == 'repoquery':
cmd = ['dnf', 'repoquery', '--available'] + repo_arg + ['--queryformat=%{EPOCH} %{NAME} %{VERSION} %{RELEASE} %{ARCH}']
elif action == 'check-update':
cmd = ['dnf', 'repoquery', '--queryformat=%{name} %{version}-%{release}.'
'%{arch} %{repoid}', '--upgrades'] + repo_arg + \
list(packages)
else:
raise RuntimeError('Unknown yum util action %s' % action)
# This is a poor solution for HYD-3855 but not one that carries any known cost.
# We sometimes see intermittent failures in test, and possibly out of test, that occur
# 1 in 50 (estimate) times. yum commands are idempotent and so trying the command three
# times has no downside and changes the estimated chance of fail to 1 in 12500.
for hyd_3885 in range(tries, -1, -1):
result = AgentShell.run(cmd)
if result.rc in valid_rc_values:
return result.stdout
else:
# if we were trying to install, clean the metadata before
# trying again
if action == 'install':
AgentShell.run(['dnf', 'clean', 'metadata'])
daemon_log.info("HYD-3885 Retrying yum command '%s'" % " ".join(cmd))
if hyd_3885 == 0:
daemon_log.info("HYD-3885 Retry yum command failed '%s'" % " ".join(cmd))
raise AgentShell.CommandExecutionError(result, cmd) # Out of retries so raise for the caller..
def yum_util(action,
packages=[],
fromrepo=None,
enablerepo=None,
narrow_updates=False):
'''
A wrapper to perform yum actions in encapsulated way.
:param action: clean, install, remove, update, requires etc
:param packages: Packages to install or remove
:param fromrepo: The repo the action should be carried out from, others are disabled.
:param enablerepo: The repo to enable for the action, others are not disabled or enabled
:param narrow_updates: ?
:return: No return but throws CommandExecutionError on error.
'''
if fromrepo and enablerepo:
raise ValueError(
"Cannot provide fromrepo and enablerepo simultaneously")
repo_arg = []
valid_rc_values = [0] # Some errors values other than 0 are valid.
if fromrepo:
repo_arg = ['--disablerepo=*', '--enablerepo=%s' % ','.join(fromrepo)]
elif enablerepo:
repo_arg = ['--enablerepo=%s' % ','.join(enablerepo)]
if narrow_updates and action == 'query':
repo_arg.extend(['--pkgnarrow=updates', '-a'])
if action == 'clean':
cmd = ['yum', 'clean', 'all'
] + (repo_arg if repo_arg else ["--enablerepo=*"])
elif action == 'install':
cmd = ['yum', 'install', '-y'] + repo_arg + list(packages)
elif action == 'remove':
cmd = ['yum', 'remove', '-y'] + repo_arg + list(packages)
elif action == 'update':
cmd = ['yum', 'update', '-y'] + repo_arg + list(packages)
elif action == 'requires':
cmd = ['repoquery', '--requires'] + repo_arg + list(packages)
elif action == 'query':
cmd = ['repoquery'] + repo_arg + list(packages)
elif action == 'repoquery':
cmd = ['repoquery'] + repo_arg + [
'-a', '--qf=%{EPOCH} %{NAME} %{VERSION} %{RELEASE} %{ARCH}'
]
elif action == 'check-update':
cmd = ['yum', 'check-update', '-q'] + repo_arg + list(packages)
valid_rc_values = [
0, 100
] # check-update returns 100 if updates are available.
else:
raise RuntimeError('Unknown yum util action %s' % action)
# This is a poor solution for HYD-3855 but not one that carries any known cost.
# We sometimes see intermittent failures in test, and possibly out of test, that occur
# 1 in 50 (estimate) times. yum commands are idempotent and so trying the command three
# times has no downside and changes the estimated chance of fail to 1 in 12500.
for hyd_3885 in range(2, -1, -1):
rc, stdout, stderr = AgentShell.run_old(cmd)
if rc in valid_rc_values:
return stdout
else:
daemon_log.info("HYD-3885 Retrying yum command '%s'" %
" ".join(cmd))
if hyd_3885 == 0:
daemon_log.info("HYD-3885 Retry yum command failed '%s'" %
" ".join(cmd))
raise AgentShell.CommandExecutionError(
AgentShell.RunResult(rc, stdout, stderr, False),
cmd) # Out of retries so raise for the caller..
def yum_util(action,
packages=[],
fromrepo=None,
enablerepo=None,
narrow_updates=False):
"""
A wrapper to perform yum actions in encapsulated way.
:param action: clean, install, remove, update, requires etc
:param packages: Packages to install or remove
:param fromrepo: The repo the action should be carried out from, others are disabled.
:param enablerepo: The repo to enable for the action, others are not disabled or enabled
:param narrow_updates: ?
:return: No return but throws CommandExecutionError on error.
"""
if fromrepo and enablerepo:
raise ValueError(
"Cannot provide fromrepo and enablerepo simultaneously")
repo_arg = []
valid_rc_values = [0] # Some errors values other than 0 are valid.
tries = 2
if fromrepo:
repo_arg = ["--disablerepo=*"
] + ["--enablerepo=%s" % r for r in fromrepo]
elif enablerepo:
repo_arg = ["--enablerepo=%s" % r for r in enablerepo]
if narrow_updates and action == "query":
repo_arg.extend(["--upgrades"])
if action == "clean":
cmd = ["yum", "clean", "all"
] + (repo_arg if repo_arg else ["--enablerepo=*"])
elif action == "install":
cmd = (["yum", "install", "-y", "--exclude", "kernel-debug"] +
repo_arg + list(packages))
elif action == "remove":
cmd = ["yum", "remove", "-y"] + repo_arg + list(packages)
elif action == "update":
cmd = (["yum", "update", "-y", "--exclude", "kernel-debug"] +
repo_arg + list(packages))
elif action == "requires":
cmd = ["repoquery", "--requires"] + repo_arg + list(packages)
elif action == "query":
cmd = ["repoquery"] + repo_arg + list(packages)
elif action == "repoquery":
cmd = (["repoquery", "--show-duplicates"] + repo_arg + [
"--queryformat=%{EPOCH} %{NAME} "
"%{VERSION} %{RELEASE} %{ARCH}"
])
else:
raise RuntimeError("Unknown yum util action %s" % action)
# This is a poor solution for HYD-3855 but not one that carries any known cost.
# We sometimes see intermittent failures in test, and possibly out of test, that occur
# 1 in 50 (estimate) times. yum commands are idempotent and so trying the command three
# times has no downside and changes the estimated chance of fail to 1 in 12500.
for hyd_3885 in range(tries, -1, -1):
result = AgentShell.run(cmd)
if result.rc in valid_rc_values:
return result.stdout
else:
# if we were trying to install, clean the metadata before
# trying again
if action == "install":
AgentShell.run(["yum", "clean", "metadata"])
daemon_log.info("HYD-3885 Retrying yum command '%s'" %
" ".join(cmd))
if hyd_3885 == 0:
daemon_log.info("HYD-3885 Retry yum command failed '%s'" %
" ".join(cmd))
raise AgentShell.CommandExecutionError(
result, cmd) # Out of retries so raise for the caller..
