def GetNodeInstances(node, secondaries=False):
"""Gets a list of instances on a node.
"""
master = qa_config.GetMasterNode()
node_name = ResolveNodeName(node)
# Get list of all instances
cmd = ["gnt-instance", "list", "--separator=:", "--no-headers",
"--output=name,pnode,snodes"]
output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
instances = []
for line in output.splitlines():
(name, pnode, snodes) = line.split(":", 2)
if ((not secondaries and pnode == node_name) or
(secondaries and node_name in snodes.split(","))):
instances.append(name)
return instances
def AssertCommand(cmd, fail=False, node=None, log_cmd=True, max_seconds=None):
"""Checks that a remote command succeeds.
@param cmd: either a string (the command to execute) or a list (to
be converted using L{utils.ShellQuoteArgs} into a string)
@type fail: boolean
@param fail: if the command is expected to fail instead of succeeding
@param node: if passed, it should be the node on which the command
should be executed, instead of the master node (can be either a
dict or a string)
@param log_cmd: if False, the command won't be logged (simply passed to
StartSSH)
@type max_seconds: double
@param max_seconds: fail if the command takes more than C{max_seconds}
seconds
@return: the return code of the command
@raise qa_error.Error: if the command fails when it shouldn't or vice versa
"""
if node is None:
node = qa_config.GetMasterNode()
nodename = _GetName(node, operator.attrgetter("primary"))
if isinstance(cmd, basestring):
cmdstr = cmd
else:
cmdstr = utils.ShellQuoteArgs(cmd)
start = datetime.datetime.now()
rcode = StartSSH(nodename, cmdstr, log_cmd=log_cmd).wait()
duration_seconds = TimedeltaToTotalSeconds(datetime.datetime.now() - start)
_AssertRetCode(rcode, fail, cmdstr, nodename)
if max_seconds is not None:
if duration_seconds > max_seconds:
raise qa_error.Error(
"Cmd '%s' took %f seconds, maximum of %f was exceeded" %
(cmdstr, duration_seconds, max_seconds))
return rcode
def ReadRemoteSshPubKeys(keyfiles, node, cluster_name, port, ask_key,
strict_host_check):
"""Fetches the public SSH keys from a node via SSH.
@type keyfiles: dict from string to (string, string) tuples
@param keyfiles: a dictionary mapping the type of key (e.g. rsa, dsa) to a
tuple consisting of the file name of the private and public key
"""
ssh_runner = SshRunner(cluster_name)
failed_results = {}
fetched_keys = {}
for (kind, (_, public_key_file)) in keyfiles.items():
cmd = ["cat", public_key_file]
ssh_cmd = ssh_runner.BuildCmd(node,
constants.SSH_LOGIN_USER,
utils.ShellQuoteArgs(cmd),
batch=False,
ask_key=ask_key,
quiet=False,
strict_host_check=strict_host_check,
use_cluster_key=False,
port=port)
result = utils.RunCmd(ssh_cmd)
if result.failed:
failed_results[kind] = (result.cmd, result.fail_reason)
else:
fetched_keys[kind] = result.stdout
if len(fetched_keys.keys()) < 1:
error_msg = "Could not fetch any public SSH key."
for (kind, (cmd, fail_reason)) in failed_results.items():
error_msg += "Could not fetch the public '%s' SSH key from node '%s':" \
" ran command '%s', failure reason: '%s'. " % \
(kind, node, cmd, fail_reason)
raise errors.OpPrereqError(error_msg)
return fetched_keys
def RunInstanceCheck(instance, running):
"""Check if instance is running or not.
"""
instance_name = _GetName(instance, operator.attrgetter("name"))
script = qa_config.GetInstanceCheckScript()
if not script:
return
master_node = qa_config.GetMasterNode()
# Build command to connect to master node
master_ssh = GetSSHCommand(master_node.primary, "--")
if running:
running_shellval = "1"
running_text = ""
else:
running_shellval = ""
running_text = "not "
print(
FormatInfo("Checking if instance '%s' is %srunning" %
(instance_name, running_text)))
args = [script, instance_name]
env = {
"PATH": constants.HOOKS_PATH,
"RUN_UUID": _RUN_UUID,
"MASTER_SSH": utils.ShellQuoteArgs(master_ssh),
"INSTANCE_NAME": instance_name,
"INSTANCE_RUNNING": running_shellval,
}
result = os.spawnve(os.P_WAIT, script, args, env)
if result != 0:
raise qa_error.Error("Instance check failed with result %s" % result)
def CheckFileUnmodified(node, filename):
"""Checks that the content of a given file remains the same after running a
wrapped code.
@type node: string
@param node: node the command should run on
@type filename: string
@param filename: absolute filename to check
"""
cmd = utils.ShellQuoteArgs(["sha1sum", MakeNodePath(node, filename)])
def Read():
return GetCommandOutput(node, cmd).strip()
# read the configuration
before = Read()
yield
# check that the configuration hasn't changed
after = Read()
if before != after:
raise qa_error.Error("File '%s' has changed unexpectedly on node %s"
" during the last operation" % (filename, node))
def LookupRapiSecret(rapi_user):
"""Find the RAPI secret for the given user.
@param rapi_user: Login user
@return: Login secret for the user
"""
CTEXT = "{CLEARTEXT}"
master = qa_config.GetMasterNode()
cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)]
file_content = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
users = ParsePasswordFile(file_content)
entry = users.get(rapi_user)
if not entry:
raise qa_error.Error("User %s not found in RAPI users file" % rapi_user)
secret = entry.password
if secret.upper().startswith(CTEXT):
secret = secret[len(CTEXT):]
elif secret.startswith("{"):
raise qa_error.Error("Unsupported password schema for RAPI user %s:"
" not a clear text password" % rapi_user)
return secret
def AssertRedirectedCommand(cmd, fail=False, node=None, log_cmd=True):
"""Executes a command with redirected output.
The log will go to the qa-output log file in the ganeti log
directory on the node where the command is executed. The fail and
node parameters are passed unchanged to AssertCommand.
@param cmd: the command to be executed, as a list; a string is not
supported
"""
if not isinstance(cmd, list):
raise qa_error.Error("Non-list passed to AssertRedirectedCommand")
ofile = utils.ShellQuote(_QA_OUTPUT)
cmdstr = utils.ShellQuoteArgs(cmd)
AssertCommand("echo ---- $(date) %s ---- >> %s" % (cmdstr, ofile),
fail=False,
node=node,
log_cmd=False)
return AssertCommand(cmdstr + " >> %s" % ofile,
fail=fail,
node=node,
log_cmd=log_cmd)
def TestGanetiCommands():
"""Test availibility of Ganeti commands.
"""
cmds = (["gnt-backup", "--version"],
["gnt-cluster", "--version"],
["gnt-debug", "--version"],
["gnt-instance", "--version"],
["gnt-job", "--version"],
["gnt-node", "--version"],
["gnt-os", "--version"],
["ganeti-masterd", "--version"],
["ganeti-noded", "--version"],
["ganeti-rapi", "--version"],
["ganeti-watcher", "--version"],
["ganeti-confd", "--version"],
["ganeti-luxid", "--version"],
)
cmd = " && ".join([utils.ShellQuoteArgs(i) for i in cmds])
for node in qa_config.get("nodes"):
AssertCommand(cmd, node=node)
def _GetTransportCommand(self):
"""Returns the command for the transport part of the daemon.
"""
socat_cmd = ("%s 2>&%d" % (utils.ShellQuoteArgs(
self._GetSocatCommand()), self._socat_stderr_fd))
dd_cmd = self._GetDdCommand()
compr = self._opts.compress
assert compr in constants.IEC_ALL
parts = []
if self._mode == constants.IEM_IMPORT:
parts.append(socat_cmd)
if compr == constants.IEC_GZIP:
parts.append("gunzip -c")
parts.append(dd_cmd)
elif self._mode == constants.IEM_EXPORT:
parts.append(dd_cmd)
if compr == constants.IEC_GZIP:
parts.append("gzip -c")
parts.append(socat_cmd)
else:
raise errors.GenericError("Invalid mode '%s'" % self._mode)
# TODO: Run transport as separate user
# The transport uses its own shell to simplify running it as a separate user
# in the future.
return self.GetBashCommand(" | ".join(parts))
def ReadRemoteSshPubKeys(pub_key_file, node, cluster_name, port, ask_key,
strict_host_check):
"""Fetches the public DSA SSH key from a node via SSH.
@type pub_key_file: string
@param pub_key_file: a tuple consisting of the file name of the public DSA key
"""
ssh_runner = SshRunner(cluster_name)
cmd = ["cat", pub_key_file]
ssh_cmd = ssh_runner.BuildCmd(node, constants.SSH_LOGIN_USER,
utils.ShellQuoteArgs(cmd),
batch=False, ask_key=ask_key, quiet=False,
strict_host_check=strict_host_check,
use_cluster_key=False,
port=port)
result = utils.RunCmd(ssh_cmd)
if result.failed:
raise errors.OpPrereqError("Could not fetch a public DSA SSH key from node"
" '%s': ran command '%s', failure reason: '%s'."
% (node, cmd, result.fail_reason))
return result.stdout
def RunSshCmdWithStdin(cluster_name,
node,
basecmd,
port,
data,
debug=False,
verbose=False,
use_cluster_key=False,
ask_key=False,
strict_host_check=False,
ensure_version=False):
"""Runs a command on a remote machine via SSH and provides input in stdin.
@type cluster_name: string
@param cluster_name: Cluster name
@type node: string
@param node: Node name
@type basecmd: string
@param basecmd: Base command (path on the remote machine)
@type port: int
@param port: The SSH port of the remote machine or None for the default
@param data: JSON-serializable input data for script (passed to stdin)
@type debug: bool
@param debug: Enable debug output
@type verbose: bool
@param verbose: Enable verbose output
@type use_cluster_key: bool
@param use_cluster_key: See L{ssh.SshRunner.BuildCmd}
@type ask_key: bool
@param ask_key: See L{ssh.SshRunner.BuildCmd}
@type strict_host_check: bool
@param strict_host_check: See L{ssh.SshRunner.BuildCmd}
"""
cmd = [basecmd]
# Pass --debug/--verbose to the external script if set on our invocation
if debug:
cmd.append("--debug")
if verbose:
cmd.append("--verbose")
if ensure_version:
all_cmds = _EnsureCorrectGanetiVersion(cmd)
else:
all_cmds = [cmd]
if port is None:
port = netutils.GetDaemonPort(constants.SSH)
srun = SshRunner(cluster_name)
scmd = srun.BuildCmd(node,
constants.SSH_LOGIN_USER,
utils.ShellQuoteArgs(
utils.ShellCombineCommands(all_cmds)),
batch=False,
ask_key=ask_key,
quiet=False,
strict_host_check=strict_host_check,
use_cluster_key=use_cluster_key,
port=port)
tempfh = tempfile.TemporaryFile()
try:
tempfh.write(serializer.DumpJson(data))
tempfh.seek(0)
result = utils.RunCmd(scmd, interactive=True, input_fd=tempfh)
finally:
tempfh.close()
if result.failed:
raise errors.OpExecError("Command '%s' failed: %s" %
(result.cmd, result.fail_reason))
def TestNodeStorage():
"""gnt-node storage"""
master = qa_config.GetMasterNode()
# FIXME: test all storage_types in constants.STORAGE_TYPES
# as soon as they are implemented.
enabled_storage_types = qa_config.GetEnabledStorageTypes()
testable_storage_types = list(set(enabled_storage_types).intersection(
set([constants.ST_FILE, constants.ST_LVM_VG, constants.ST_LVM_PV])))
for storage_type in testable_storage_types:
cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
# Test simple list
AssertCommand(cmd)
# Test all storage fields
cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
"--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS))]
AssertCommand(cmd)
# Get list of valid storage devices
cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
"--output=node,name,allocatable", "--separator=|",
"--no-headers"]
output = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
# Test with up to two devices
testdevcount = 2
for line in output.splitlines()[:testdevcount]:
(node_name, st_name, st_allocatable) = line.split("|")
# Dummy modification without any changes
cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
AssertCommand(cmd)
# Make sure we end up with the same value as before
if st_allocatable.lower() == "y":
test_allocatable = ["no", "yes"]
else:
test_allocatable = ["yes", "no"]
fail = (constants.SF_ALLOCATABLE not in
constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
for i in test_allocatable:
AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
node_name, storage_type, st_name], fail=fail)
# Verify list output
cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
"--output=name,allocatable", "--separator=|",
"--no-headers", node_name]
listout = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
for line in listout.splitlines():
(vfy_name, vfy_allocatable) = line.split("|")
if vfy_name == st_name and not fail:
AssertEqual(vfy_allocatable, i[0].upper())
else:
AssertEqual(vfy_allocatable, st_allocatable)
# Test repair functionality
fail = (constants.SO_FIX_CONSISTENCY not in
constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
AssertCommand(["gnt-node", "repair-storage", node_name,
storage_type, st_name], fail=fail)
def AssertCommand(cmd,
fail=False,
node=None,
log_cmd=True,
forward_agent=True,
max_seconds=None):
"""Checks that a remote command succeeds.
@param cmd: either a string (the command to execute) or a list (to
be converted using L{utils.ShellQuoteArgs} into a string)
@type fail: boolean or None
@param fail: if the command is expected to fail instead of succeeding,
or None if we don't care
@param node: if passed, it should be the node on which the command
should be executed, instead of the master node (can be either a
dict or a string)
@param log_cmd: if False, the command won't be logged (simply passed to
StartSSH)
@type forward_agent: boolean
@param forward_agent: whether to forward the agent when starting the SSH
session or not, sometimes useful for crypto-related
operations which can use a key they should not
@type max_seconds: double
@param max_seconds: fail if the command takes more than C{max_seconds}
seconds
@return: the return code, stdout and stderr of the command
@raise qa_error.Error: if the command fails when it shouldn't or vice versa
"""
if node is None:
node = qa_config.GetMasterNode()
nodename = _GetName(node, operator.attrgetter("primary"))
if isinstance(cmd, basestring):
cmdstr = cmd
else:
cmdstr = utils.ShellQuoteArgs(cmd)
start = datetime.datetime.now()
popen = StartSSH(nodename,
cmdstr,
log_cmd=log_cmd,
forward_agent=forward_agent)
# Run the command
stdout, stderr = popen.communicate()
rcode = popen.returncode
duration_seconds = TimedeltaToTotalSeconds(datetime.datetime.now() - start)
try:
if fail is not None:
_AssertRetCode(rcode, fail, cmdstr, nodename)
finally:
if log_cmd:
_PrintCommandOutput(stdout, stderr)
if max_seconds is not None:
if duration_seconds > max_seconds:
raise qa_error.Error(
"Cmd '%s' took %f seconds, maximum of %f was exceeded" %
(cmdstr, duration_seconds, max_seconds))
return rcode, stdout, stderr
def RunNodeSetupCmd(cluster_name, node, basecmd, debug, verbose,
use_cluster_key, ask_key, strict_host_check,
port, data):
"""Runs a command to configure something on a remote machine.
@type cluster_name: string
@param cluster_name: Cluster name
@type node: string
@param node: Node name
@type basecmd: string
@param basecmd: Base command (path on the remote machine)
@type debug: bool
@param debug: Enable debug output
@type verbose: bool
@param verbose: Enable verbose output
@type use_cluster_key: bool
@param use_cluster_key: See L{ssh.SshRunner.BuildCmd}
@type ask_key: bool
@param ask_key: See L{ssh.SshRunner.BuildCmd}
@type strict_host_check: bool
@param strict_host_check: See L{ssh.SshRunner.BuildCmd}
@type port: int
@param port: The SSH port of the remote machine or None for the default
@param data: JSON-serializable input data for script (passed to stdin)
"""
cmd = [basecmd]
# Pass --debug/--verbose to the external script if set on our invocation
if debug:
cmd.append("--debug")
if verbose:
cmd.append("--verbose")
logging.debug("Node setup command: %s", cmd)
version = constants.DIR_VERSION
all_cmds = [["test", "-d", os.path.join(pathutils.PKGLIBDIR, version)]]
if constants.HAS_GNU_LN:
all_cmds.extend([["ln", "-s", "-f", "-T",
os.path.join(pathutils.PKGLIBDIR, version),
os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")],
["ln", "-s", "-f", "-T",
os.path.join(pathutils.SHAREDIR, version),
os.path.join(pathutils.SYSCONFDIR, "ganeti/share")]])
else:
all_cmds.extend([["rm", "-f",
os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")],
["ln", "-s", "-f",
os.path.join(pathutils.PKGLIBDIR, version),
os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")],
["rm", "-f",
os.path.join(pathutils.SYSCONFDIR, "ganeti/share")],
["ln", "-s", "-f",
os.path.join(pathutils.SHAREDIR, version),
os.path.join(pathutils.SYSCONFDIR, "ganeti/share")]])
all_cmds.append(cmd)
if port is None:
port = netutils.GetDaemonPort(constants.SSH)
family = ssconf.SimpleStore().GetPrimaryIPFamily()
srun = ssh.SshRunner(cluster_name,
ipv6=(family == netutils.IP6Address.family))
scmd = srun.BuildCmd(node, constants.SSH_LOGIN_USER,
utils.ShellQuoteArgs(
utils.ShellCombineCommands(all_cmds)),
batch=False, ask_key=ask_key, quiet=False,
strict_host_check=strict_host_check,
use_cluster_key=use_cluster_key,
port=port)
tempfh = tempfile.TemporaryFile()
try:
tempfh.write(serializer.DumpJson(data))
tempfh.seek(0)
result = utils.RunCmd(scmd, interactive=True, input_fd=tempfh)
finally:
tempfh.close()
if result.failed:
raise errors.OpExecError("Command '%s' failed: %s" %
(result.cmd, result.fail_reason))
_WaitForSshDaemon(node, port, family)
def testShellQuoteArgs(self):
self.assertEqual(utils.ShellQuoteArgs(["a", "b", "c"]), "a b c")
self.assertEqual(utils.ShellQuoteArgs(['a', 'b"', 'c']), "a 'b\"' c")
self.assertEqual(utils.ShellQuoteArgs(['a', 'b\'', 'c']),
"a 'b'\\\''' c")
def _TestOs(mode, rapi_cb):
"""Generic function for OS definition testing
"""
master = qa_config.GetMasterNode()
name = _TEMP_OS_NAME
variant = "default"
fullname = "%s+%s" % (name, variant)
dirname = _TEMP_OS_PATH
# Ensure OS is usable
cmd = ["gnt-os", "modify", "--hidden=no", "--blacklisted=no", name]
AssertCommand(cmd)
nodes = []
try:
for i, node in enumerate(qa_config.get("nodes")):
nodes.append(node)
if mode == _ALL_INVALID:
valid = False
elif mode == _ALL_VALID:
valid = True
elif mode == _PARTIALLY_VALID:
valid = bool(i % 2)
else:
raise AssertionError("Unknown mode %s" % mode)
_SetupTempOs(node, dirname, variant, valid)
# TODO: Use Python 2.6's itertools.permutations
for (hidden, blacklisted) in [(False, False), (True, False),
(False, True), (True, True)]:
# Change OS' visibility
cmd = ["gnt-os", "modify", "--hidden", ["no", "yes"][int(hidden)],
"--blacklisted", ["no", "yes"][int(blacklisted)], name]
AssertCommand(cmd)
# Diagnose, checking exit status
AssertCommand(["gnt-os", "diagnose"], fail=(mode != _ALL_VALID))
# Diagnose again, ignoring exit status
output = qa_utils.GetCommandOutput(master.primary,
"gnt-os diagnose || :")
for line in output.splitlines():
if line.startswith("OS: %s [global status:" % name):
break
else:
raise qa_error.Error("Didn't find OS '%s' in 'gnt-os diagnose'" % name)
# Check info for all
cmd = ["gnt-os", "info"]
output = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
AssertIn("%s:" % name, output.splitlines())
# Check info for OS
cmd = ["gnt-os", "info", name]
output = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd)).splitlines()
AssertIn("%s:" % name, output)
for (field, value) in [("valid", mode == _ALL_VALID),
("hidden", hidden),
("blacklisted", blacklisted)]:
AssertIn(" - %s: %s" % (field, value), output)
# Only valid OSes should be listed
cmd = ["gnt-os", "list", "--no-headers"]
output = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
if mode == _ALL_VALID and not (hidden or blacklisted):
assert_fn = AssertIn
else:
assert_fn = AssertNotIn
assert_fn(fullname, output.splitlines())
# Check via RAPI
if rapi_cb:
assert_fn(fullname, rapi_cb())
finally:
for node in nodes:
_RemoveTempOs(node, dirname)
def AssertInGroup(group, nodes):
real_output = GetCommandOutput(master_node,
"gnt-node list --no-headers -o group " +
utils.ShellQuoteArgs(nodes))
AssertEqual(real_output.splitlines(), [group] * len(nodes))