def _CreateRapiUser(rapi_user):
"""RAPI credentials creation, with the secret auto-generated.
"""
rapi_secret = utils.GenerateSecret()
master = qa_config.GetMasterNode()
rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
rapi_dir = os.path.dirname(rapi_users_path)
fh = tempfile.NamedTemporaryFile(mode="w")
try:
fh.write("%s %s write\n" % (rapi_user, rapi_secret))
fh.flush()
tmpru = qa_utils.UploadFile(master.primary, fh.name)
try:
AssertCommand(["mkdir", "-p", rapi_dir])
AssertCommand(["mv", tmpru, rapi_users_path])
finally:
AssertCommand(["rm", "-f", tmpru])
finally:
fh.close()
# The certificates have to be reloaded now
AssertCommand(["service", "ganeti", "restart"])
return rapi_secret
def _ResetWatcherDaemon():
"""Removes the watcher daemon's state file.
"""
path = \
qa_utils.MakeNodePath(qa_config.GetMasterNode(),
pathutils.WATCHER_GROUP_STATE_FILE % "*-*-*-*")
AssertCommand(["bash", "-c", "rm -vf %s" % path])
def RunCustomSshPortTests():
"""Test accessing nodes with custom SSH ports.
This requires removing nodes, adding them to a new group, and then undoing
the change.
"""
if not qa_config.TestEnabled("group-custom-ssh-port"):
return
std_port = netutils.GetDaemonPort(constants.SSH)
port = 211
master = qa_config.GetMasterNode()
with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes:
# Checks if the node(s) could be contacted through IPv6.
# If yes, better skip the whole test.
for node in nodes:
if qa_utils.UsesIPv6Connection(node.primary, std_port):
print("Node %s is likely to be reached using IPv6,"
"skipping the test" % (node.primary, ))
return
for node in nodes:
qa_node.NodeRemove(node)
with qa_iptables.RulesContext() as r:
with qa_group.NewGroupCtx() as group:
qa_group.ModifyGroupSshPort(r, group, nodes, port)
for node in nodes:
qa_node.NodeAdd(node, group=group)
# Make sure that the cluster doesn't have any pre-existing problem
qa_cluster.AssertClusterVerify()
# Create and allocate instances
instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
try:
instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
try:
# cluster-verify checks that disks are allocated correctly
qa_cluster.AssertClusterVerify()
# Remove instances
qa_instance.TestInstanceRemove(instance2)
qa_instance.TestInstanceRemove(instance1)
finally:
instance2.Release()
finally:
instance1.Release()
for node in nodes:
qa_node.NodeRemove(node)
for node in nodes:
qa_node.NodeAdd(node)
qa_cluster.AssertClusterVerify()
def TestNodeModify(node):
"""gnt-node modify"""
default_pool_size = 10
nodes = qa_config.GetAllNodes()
test_pool_size = len(nodes) - 1
# Reduce the number of master candidates, because otherwise all
# subsequent 'gnt-cluster verify' commands fail due to not enough
# master candidates.
AssertCommand(
["gnt-cluster", "modify",
"--candidate-pool-size=%s" % test_pool_size])
# make sure enough master candidates will be available by disabling the
# master candidate role first with --auto-promote
AssertCommand([
"gnt-node", "modify", "--master-candidate=no", "--auto-promote",
node.primary
])
# now it's save to force-remove the master candidate role
for flag in ["master-candidate", "drained", "offline"]:
for value in ["yes", "no"]:
AssertCommand([
"gnt-node", "modify", "--force",
"--%s=%s" % (flag, value), node.primary
])
AssertCommand(["gnt-cluster", "verify"])
AssertCommand(
["gnt-node", "modify", "--master-candidate=yes", node.primary])
# Test setting secondary IP address
AssertCommand([
"gnt-node", "modify",
"--secondary-ip=%s" % node.secondary, node.primary
])
AssertRedirectedCommand(["gnt-cluster", "verify"])
AssertCommand([
"gnt-cluster", "modify",
"--candidate-pool-size=%s" % default_pool_size
])
# For test clusters with more nodes than the default pool size,
# we now have too many master candidates. To readjust to the original
# size, manually demote all nodes and rely on auto-promotion to adjust.
if len(nodes) > default_pool_size:
master = qa_config.GetMasterNode()
for n in nodes:
if n.primary != master.primary:
AssertCommand([
"gnt-node", "modify", "--master-candidate=no",
"--auto-promote", n.primary
])
def MarkNodeAddedAll():
"""Mark all nodes as added.
This is useful if we don't create the cluster ourselves (in qa).
"""
master = qa_config.GetMasterNode()
for node in qa_config.get("nodes"):
if node != master:
node.MarkAdded()
def _AssertOobCall(verify_path, expected_args):
"""Assert the OOB call was performed with expetected args."""
master = qa_config.GetMasterNode()
verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
output = qa_utils.GetCommandOutput(master.primary,
verify_output_cmd,
tty=False)
AssertEqual(expected_args, output.strip())
def TestResumeWatcher():
"""Tests and unpauses the watcher.
"""
master = qa_config.GetMasterNode()
AssertCommand(["gnt-cluster", "watcher", "continue"])
cmd = ["gnt-cluster", "watcher", "info"]
output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
AssertMatch(output, r"^.*\bis not paused\b.*")
def _TestGroupModifyISpecs(groupname):
# This test is built on the assumption that the default ipolicy holds for
# the node group under test
old_values = _GetGroupIPolicy(groupname)
samevals = dict((p, 4) for p in constants.ISPECS_PARAMETERS)
base_specs = {
constants.ISPECS_MINMAX: [{
constants.ISPECS_MIN: samevals,
constants.ISPECS_MAX: samevals,
}],
}
mod_values = _TestGroupSetISpecs(groupname, new_specs=base_specs,
old_values=old_values)
for par in constants.ISPECS_PARAMETERS:
# First make sure that the test works with good values
good_specs = {
constants.ISPECS_MINMAX: [{
constants.ISPECS_MIN: {par: 8},
constants.ISPECS_MAX: {par: 8},
}],
}
mod_values = _TestGroupSetISpecs(groupname, diff_specs=good_specs,
old_values=mod_values)
bad_specs = {
constants.ISPECS_MINMAX: [{
constants.ISPECS_MIN: {par: 8},
constants.ISPECS_MAX: {par: 4},
}],
}
_TestGroupSetISpecs(groupname, diff_specs=bad_specs, fail=True,
old_values=mod_values)
AssertCommand(["gnt-group", "modify", "--ipolicy-bounds-specs", "default",
groupname])
AssertEqual(_GetGroupIPolicy(groupname), old_values)
# Get the ipolicy command (from the cluster config)
mnode = qa_config.GetMasterNode()
addcmd = GetCommandOutput(mnode.primary, utils.ShellQuoteArgs([
"gnt-group", "show-ispecs-cmd", "--include-defaults", groupname,
]))
modcmd = ["gnt-group", "modify"]
opts = addcmd.split()
assert opts[0:2] == ["gnt-group", "add"]
for k in range(2, len(opts) - 1):
if opts[k].startswith("--ipolicy-"):
assert k + 2 <= len(opts)
modcmd.extend(opts[k:k + 2])
modcmd.append(groupname)
# Apply the ipolicy to the group and verify the result
AssertCommand(modcmd)
new_addcmd = GetCommandOutput(mnode.primary, utils.ShellQuoteArgs([
"gnt-group", "show-ispecs-cmd", groupname,
]))
AssertEqual(addcmd, new_addcmd)
def TestJobCancellation():
"""gnt-job cancel"""
# The delay used for the first command should be large enough for the next
# command and the cancellation command to complete before the first job is
# done. The second delay should be small enough that not too much time is
# spend waiting in the case of a failed cancel and a running command.
FIRST_COMMAND_DELAY = 10.0
AssertCommand(["gnt-debug", "delay", "--submit", str(FIRST_COMMAND_DELAY)])
SECOND_COMMAND_DELAY = 3.0
master = qa_config.GetMasterNode()
# Forcing tty usage does not work on buildbot, so force all output of this
# command to be redirected to stdout
job_id_output = GetCommandOutput(
master.primary,
"gnt-debug delay --submit %s 2>&1" % SECOND_COMMAND_DELAY)
possible_job_ids = re.findall("JobID: ([0-9]+)", job_id_output)
if len(possible_job_ids) != 1:
raise qa_error.Error(
"Cannot parse gnt-debug delay output to find job id")
job_id = possible_job_ids[0]
AssertCommand(["gnt-job", "cancel", job_id])
# Now wait until the second job finishes, and expect the watch to fail due to
# job cancellation
AssertCommand(["gnt-job", "watch", job_id], fail=True)
# Then check for job cancellation
job_status = qa_job_utils.GetJobStatus(job_id)
if job_status != constants.JOB_STATUS_CANCELED:
# Try and see if the job is being cancelled, and wait until the status
# changes or we hit a timeout
if job_status == constants.JOB_STATUS_CANCELING:
retry_fn = functools.partial(qa_job_utils.RetryingWhileJobStatus,
constants.JOB_STATUS_CANCELING,
job_id)
try:
# The multiplier to use is arbitrary, setting it higher could prevent
# flakiness
WAIT_MULTIPLIER = 4.0
job_status = retry.Retry(retry_fn, 2.0,
WAIT_MULTIPLIER * FIRST_COMMAND_DELAY)
except retry.RetryTimeout:
# The job status remains the same
pass
if job_status != constants.JOB_STATUS_CANCELED:
raise qa_error.Error("Job was not successfully cancelled, status "
"found: %s" % job_status)
def ReloadCertificates(ensure_presence=True):
"""Reloads the client RAPI certificate with the one present on the node.
If the QA is set up to use a specific certificate using the
"rapi-files-location" parameter, it will be put in place prior to retrieving
it.
"""
if ensure_presence:
_EnsureRapiFilesPresence()
if _rapi_username is None or _rapi_password is None:
raise qa_error.Error("RAPI username and password have to be set before"
" attempting to reload a certificate.")
# pylint: disable=W0603
# due to global usage
global _rapi_ca
global _rapi_client
master = qa_config.GetMasterNode()
# Load RAPI certificate from master node
cmd = [
"openssl", "x509", "-in",
qa_utils.MakeNodePath(master, pathutils.RAPI_CERT_FILE)
]
# Write to temporary file
_rapi_ca = tempfile.NamedTemporaryFile(mode="w")
_rapi_ca.write(
qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)))
_rapi_ca.flush()
port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT)
cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name, proxy="")
if qa_config.UseVirtualCluster():
# TODO: Implement full support for RAPI on virtual clusters
print(
qa_logging.FormatWarning("RAPI tests are not yet supported on"
" virtual clusters and will be disabled"))
assert _rapi_client is None
else:
_rapi_client = rapi.client.GanetiRapiClient(master.primary,
port=port,
username=_rapi_username,
password=_rapi_password,
curl_config_fn=cfg_curl)
print("RAPI protocol version: %s" % _rapi_client.GetVersion())
def RunInstanceTestsFull(create_fun, inodes, supported_conversions, templ):
instance = RunTest(create_fun, inodes)
try:
RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown,
instance)
RunTestIf("instance-communication",
qa_instance.TestInstanceCommunication, instance,
qa_config.GetMasterNode())
RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
RunDaemonTests(instance)
for node in inodes:
RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node,
templ == constants.DT_DRBD8)
if len(inodes) > 1:
RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
constants.INITIAL_NODE_GROUP_NAME, inodes[0].primary,
inodes[1].primary)
# This test will run once but it will cover all the supported
# user-provided disk template conversions
if qa_config.TestEnabled("instance-convert-disk"):
if (len(supported_conversions) > 1
and instance.disk_template in supported_conversions):
RunTest(qa_instance.TestInstanceShutdown, instance)
RunTest(qa_instance.TestInstanceConvertDiskTemplate, instance,
supported_conversions)
RunTest(qa_instance.TestInstanceStartup, instance)
# At this point we clear the set because the requested conversions
# has been tested
supported_conversions.clear()
else:
test_desc = "Converting instance of template %s" % templ
ReportTestSkip(test_desc, "conversion feature")
RunTestIf("instance-modify-disks", qa_instance.TestInstanceModifyDisks,
instance)
RunCommonInstanceTests(instance, inodes)
if qa_config.TestEnabled("instance-modify-primary"):
othernode = qa_config.AcquireNode()
RunTest(qa_instance.TestInstanceModifyPrimaryAndBack, instance,
inodes[0], othernode)
othernode.Release()
RunGroupListTests()
RunExportImportTests(instance, inodes)
RunHardwareFailureTests(instance, inodes)
RunRepairDiskSizes()
RunTestIf(["rapi", "instance-data-censorship"],
qa_rapi.TestInstanceDataCensorship, instance, inodes)
RunTest(qa_instance.TestInstanceRemove, instance)
finally:
instance.Release()
del instance
qa_cluster.AssertClusterVerify()
def _ReadSsconfInstanceList():
"""Reads ssconf_instance_list from the master node.
"""
master = qa_config.GetMasterNode()
ssconf_path = utils.PathJoin(pathutils.DATA_DIR,
"ssconf_%s" % constants.SS_INSTANCE_LIST)
cmd = ["cat", qa_utils.MakeNodePath(master, ssconf_path)]
return qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd)).splitlines()
def _InstanceRunning(name):
"""Checks whether an instance is running.
@param name: full name of the instance
"""
master = qa_config.GetMasterNode()
cmd = (
utils.ShellQuoteArgs(["gnt-instance", "list", "-o", "status", name]) +
' | grep running')
ret = StartSSH(master.primary, cmd).wait()
return ret == 0
def GetObjectInfo(infocmd):
"""Get and parse information about a Ganeti object.
@type infocmd: list of strings
@param infocmd: command to be executed, e.g. ["gnt-cluster", "info"]
@return: the information parsed, appropriately stored in dictionaries,
lists...
"""
master = qa_config.GetMasterNode()
cmdline = utils.ShellQuoteArgs(infocmd)
info_out = GetCommandOutput(master.primary, cmdline)
return yaml.load(info_out)
def GetOutputFromMaster(cmd, use_multiplexer=True, log_cmd=True):
""" Gets the output of a command executed on master.
"""
if isinstance(cmd, str):
cmdstr = cmd
else:
cmdstr = utils.ShellQuoteArgs(cmd)
# Necessary due to the stderr stream not being captured properly on the
# buildbot
cmdstr += " 2>&1"
return GetCommandOutput(qa_config.GetMasterNode().primary, cmdstr,
use_multiplexer=use_multiplexer, log_cmd=log_cmd)
def _List(listcmd, fields, names):
"""Runs a list command.
"""
master = qa_config.GetMasterNode()
cmd = [
listcmd, "list", "--separator=|", "--no-headers", "--output",
",".join(fields)
]
if names:
cmd.extend(names)
return GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd)).splitlines()
def _CreateOobScriptStructure():
"""Create a simple OOB handling script and its structure."""
master = qa_config.GetMasterNode()
data_path = qa_utils.UploadData(master.primary, "")
verify_path = qa_utils.UploadData(master.primary, "")
exit_code_path = qa_utils.UploadData(master.primary, "")
oob_script = (("#!/bin/bash\n"
"echo \"$@\" > %s\n"
"cat %s\n"
"exit $(< %s)\n") %
(utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
utils.ShellQuote(exit_code_path)))
oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0o700)
return [oob_path, verify_path, data_path, exit_code_path]
def TestInterClusterInstanceMove(src_instance,
dest_instance,
inodes,
tnode,
perform_checks=True):
"""Test tools/move-instance"""
master = qa_config.GetMasterNode()
rapi_pw_file = tempfile.NamedTemporaryFile(mode="w")
rapi_pw_file.write(_rapi_password)
rapi_pw_file.flush()
# Needed only if checks are to be performed
if perform_checks:
dest_instance.SetDiskTemplate(src_instance.disk_template)
# TODO: Run some instance tests before moving back
if len(inodes) > 1:
# No disk template currently requires more than 1 secondary node. If this
# changes, either this test must be skipped or the script must be updated.
assert len(inodes) == 2
snode = inodes[1]
else:
# Instance is not redundant, but we still need to pass a node
# (which will be ignored)
snode = tnode
pnode = inodes[0]
# pnode:snode are the *current* nodes, and the first move is an
# iallocator-guided move outside of pnode. The node lock for the pnode
# assures that this happens, and while we cannot be sure where the instance
# will land, it is a real move.
locks = {locking.LEVEL_NODE: [pnode.primary]}
RunWithLocks(_InvokeMoveInstance, locks, 600.0, False, dest_instance.name,
src_instance.name, rapi_pw_file.name, master.primary,
perform_checks)
# And then back to pnode:snode
_InvokeMoveInstance(src_instance.name,
dest_instance.name,
rapi_pw_file.name,
master.primary,
perform_checks,
target_nodes=(pnode.primary, snode.primary))
def GenericQueryFieldsTest(cmd, fields):
master = qa_config.GetMasterNode()
# Listing fields
AssertRedirectedCommand([cmd, "list-fields"])
AssertRedirectedCommand([cmd, "list-fields"] + fields)
# Check listed fields (all, must be sorted)
realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"]
output = GetCommandOutput(master.primary,
utils.ShellQuoteArgs(realcmd)).splitlines()
AssertEqual([line.split("|", 1)[0] for line in output],
utils.NiceSort(fields))
# Check exit code for listing unknown field
rcode, _, _ = AssertCommand([cmd, "list-fields", "field/does/not/exist"],
fail=True)
AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD)
def RemoveFromEtcHosts(hostnames):
"""Remove hostnames from /etc/hosts.
@param hostnames: List of hostnames first used A records, all other CNAMEs
"""
master = qa_config.GetMasterNode()
tmp_hosts = UploadData(master.primary, "", mode=0o644)
quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)
sed_data = " ".join(hostnames)
try:
AssertCommand(
(r"sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s"
r" && mv %s %s") %
(sed_data, utils.ShellQuote(pathutils.ETC_HOSTS), quoted_tmp_hosts,
quoted_tmp_hosts, utils.ShellQuote(pathutils.ETC_HOSTS)))
except Exception:
AssertCommand(["rm", "-f", tmp_hosts])
raise
def AddToEtcHosts(hostnames):
"""Adds hostnames to /etc/hosts.
@param hostnames: List of hostnames first used A records, all other CNAMEs
"""
master = qa_config.GetMasterNode()
tmp_hosts = UploadData(master.primary, "", mode=0o644)
data = []
for localhost in ("::1", "127.0.0.1"):
data.append("%s %s" % (localhost, " ".join(hostnames)))
try:
AssertCommand(
"{ cat %s && echo -e '%s'; } > %s && mv %s %s" %
(utils.ShellQuote(pathutils.ETC_HOSTS), "\\n".join(data),
utils.ShellQuote(tmp_hosts), utils.ShellQuote(tmp_hosts),
utils.ShellQuote(pathutils.ETC_HOSTS)))
except Exception:
AssertCommand(["rm", "-f", tmp_hosts])
raise
def GetNodeInstances(node, secondaries=False):
"""Gets a list of instances on a node.
"""
master = qa_config.GetMasterNode()
node_name = ResolveNodeName(node)
# Get list of all instances
cmd = [
"gnt-instance", "list", "--separator=:", "--no-headers",
"--output=name,pnode,snodes"
]
output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
instances = []
for line in output.splitlines():
(name, pnode, snodes) = line.split(":", 2)
if ((not secondaries and pnode == node_name)
or (secondaries and node_name in snodes.split(","))):
instances.append(name)
return instances
def TestAssignNodesIncludingSplit(orig_group, node1, node2):
"""gnt-group assign-nodes --force
Expects node1 and node2 to be primary and secondary for a common instance.
"""
assert node1 != node2
(other_group, ) = qa_utils.GetNonexistentGroups(1)
master_node = qa_config.GetMasterNode().primary
def AssertInGroup(group, nodes):
real_output = GetCommandOutput(master_node,
"gnt-node list --no-headers -o group " +
utils.ShellQuoteArgs(nodes))
AssertEqual(real_output.splitlines(), [group] * len(nodes))
AssertInGroup(orig_group, [node1, node2])
AssertCommand(["gnt-group", "add", other_group])
try:
AssertCommand(["gnt-group", "assign-nodes", other_group, node1, node2])
AssertInGroup(other_group, [node1, node2])
# This should fail because moving node1 to orig_group would leave their
# common instance split between orig_group and other_group.
AssertCommand(["gnt-group", "assign-nodes", orig_group, node1], fail=True)
AssertInGroup(other_group, [node1, node2])
AssertCommand(["gnt-group", "assign-nodes", "--force", orig_group, node1])
AssertInGroup(orig_group, [node1])
AssertInGroup(other_group, [node2])
AssertCommand(["gnt-group", "assign-nodes", orig_group, node2])
AssertInGroup(orig_group, [node1, node2])
finally:
AssertCommand(["gnt-group", "remove", other_group])
def RunInstanceCheck(instance, running):
"""Check if instance is running or not.
"""
instance_name = _GetName(instance, operator.attrgetter("name"))
script = qa_config.GetInstanceCheckScript()
if not script:
return
master_node = qa_config.GetMasterNode()
# Build command to connect to master node
master_ssh = GetSSHCommand(master_node.primary, "--")
if running:
running_shellval = "1"
running_text = ""
else:
running_shellval = ""
running_text = "not "
print(
FormatInfo("Checking if instance '%s' is %srunning" %
(instance_name, running_text)))
args = [script, instance_name]
env = {
"PATH": constants.HOOKS_PATH,
"RUN_UUID": _RUN_UUID,
"MASTER_SSH": utils.ShellQuoteArgs(master_ssh),
"INSTANCE_NAME": instance_name,
"INSTANCE_RUNNING": running_shellval,
}
result = os.spawnve(os.P_WAIT, script, args, env)
if result != 0:
raise qa_error.Error("Instance check failed with result %s" % result)
def main():
"""Main program.
"""
colors.check_for_colors()
parser = optparse.OptionParser(usage="%prog [options] <config-file>")
parser.add_option("--yes-do-it",
dest="yes_do_it",
action="store_true",
help="Really execute the tests")
(opts, args) = parser.parse_args()
if len(args) == 1:
(config_file, ) = args
else:
parser.error("Wrong number of arguments.")
if not opts.yes_do_it:
print("Executing this script irreversibly destroys any Ganeti\n"
"configuration on all nodes involved. If you really want\n"
"to start testing, supply the --yes-do-it option.")
sys.exit(1)
qa_config.Load(config_file)
primary = qa_config.GetMasterNode().primary
qa_utils.StartMultiplexer(primary)
print("SSH command for primary node: %s" %
utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
print("SSH command for other nodes: %s" %
utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
try:
RunQa()
finally:
qa_utils.CloseMultiplexers()
def _LookupRapiSecret(rapi_user):
"""Find the RAPI secret for the given user on the QA machines.
@param rapi_user: Login user
@return: Login secret for the user
"""
CTEXT = "{CLEARTEXT}"
master = qa_config.GetMasterNode()
cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)]
file_content = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
users = ParsePasswordFile(file_content)
entry = users.get(rapi_user)
if not entry:
raise qa_error.Error("User %s not found in RAPI users file" %
rapi_user)
secret = entry.password
if secret.upper().startswith(CTEXT):
secret = secret[len(CTEXT):]
elif secret.startswith("{"):
raise qa_error.Error("Unsupported password schema for RAPI user %s:"
" not a clear text password" % rapi_user)
return secret
def TestNodeListDrbd(node, is_drbd):
"""gnt-node list-drbd"""
master = qa_config.GetMasterNode()
result_output = GetCommandOutput(
master.primary, "gnt-node list-drbd --no-header %s" % node.primary)
# Meaningful to note: there is but one instance, and the node is either the
# primary or one of the secondaries
if is_drbd:
# Invoked for both primary and secondary
per_disk_info = result_output.splitlines()
for line in per_disk_info:
try:
drbd_node, _, _, _, _, drbd_peer = line.split()
except ValueError:
raise qa_error.Error(
"Could not examine list-drbd output: expected a"
" single row of 6 entries, found the following:"
" %s" % line)
AssertIn(node.primary, [drbd_node, drbd_peer],
msg="The output %s does not contain the node" % line)
else:
# Output should be empty, barring newlines
AssertEqual(result_output.strip(), "")
def _TestOs(mode, rapi_cb):
"""Generic function for OS definition testing
"""
master = qa_config.GetMasterNode()
name = _TEMP_OS_NAME
variant = "default"
fullname = "%s+%s" % (name, variant)
dirname = _TEMP_OS_PATH
# Ensure OS is usable
cmd = ["gnt-os", "modify", "--hidden=no", "--blacklisted=no", name]
AssertCommand(cmd)
nodes = []
try:
for i, node in enumerate(qa_config.get("nodes")):
nodes.append(node)
if mode == _ALL_INVALID:
valid = False
elif mode == _ALL_VALID:
valid = True
elif mode == _PARTIALLY_VALID:
valid = bool(i % 2)
else:
raise AssertionError("Unknown mode %s" % mode)
_SetupTempOs(node, dirname, variant, valid)
# TODO: Use Python 2.6's itertools.permutations
for (hidden, blacklisted) in [(False, False), (True, False),
(False, True), (True, True)]:
# Change OS' visibility
cmd = ["gnt-os", "modify", "--hidden", ["no", "yes"][int(hidden)],
"--blacklisted", ["no", "yes"][int(blacklisted)], name]
AssertCommand(cmd)
# Diagnose, checking exit status
AssertCommand(["gnt-os", "diagnose"], fail=(mode != _ALL_VALID))
# Diagnose again, ignoring exit status
output = qa_utils.GetCommandOutput(master.primary,
"gnt-os diagnose || :")
for line in output.splitlines():
if line.startswith("OS: %s [global status:" % name):
break
else:
raise qa_error.Error("Didn't find OS '%s' in 'gnt-os diagnose'" % name)
# Check info for all
cmd = ["gnt-os", "info"]
output = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
AssertIn("%s:" % name, output.splitlines())
# Check info for OS
cmd = ["gnt-os", "info", name]
output = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd)).splitlines()
AssertIn("%s:" % name, output)
for (field, value) in [("valid", mode == _ALL_VALID),
("hidden", hidden),
("blacklisted", blacklisted)]:
AssertIn(" - %s: %s" % (field, value), output)
# Only valid OSes should be listed
cmd = ["gnt-os", "list", "--no-headers"]
output = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
if mode == _ALL_VALID and not (hidden or blacklisted):
assert_fn = AssertIn
else:
assert_fn = AssertNotIn
assert_fn(fullname, output.splitlines())
# Check via RAPI
if rapi_cb:
assert_fn(fullname, rapi_cb())
finally:
for node in nodes:
_RemoveTempOs(node, dirname)
def RunQa():
"""Main QA body.
"""
RunTestBlock(RunEnvTests)
SetupCluster()
RunTestBlock(RunClusterTests)
RunTestBlock(RunOsTests)
RunTestIf("tags", qa_tags.TestClusterTags)
RunTestBlock(RunCommonNodeTests)
RunTestBlock(RunGroupListTests)
RunTestBlock(RunGroupRwTests)
RunTestBlock(RunNetworkTests)
RunTestBlock(RunFilterTests)
# The master shouldn't be readded or put offline; "delay" needs a non-master
# node to test
pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
try:
RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
RunTestIf("delay", qa_cluster.TestDelay, pnode)
finally:
pnode.Release()
# Make sure the cluster is clean before running instance tests
qa_cluster.AssertClusterVerify()
pnode = qa_config.AcquireNode()
try:
RunTestIf("tags", qa_tags.TestNodeTags, pnode)
if qa_rapi.Enabled():
RunTest(qa_rapi.TestNode, pnode)
if (qa_config.TestEnabled("instance-add-plain-disk")
and qa_config.IsTemplateSupported(constants.DT_PLAIN)):
# Normal instance allocation via RAPI
for use_client in [True, False]:
rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
use_client)
try:
if qa_config.TestEnabled(
"instance-plain-rapi-common-tests"):
RunCommonInstanceTests(rapi_instance, [pnode])
RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance,
use_client)
finally:
rapi_instance.Release()
del rapi_instance
# Multi-instance allocation
rapi_instance_one, rapi_instance_two = \
RunTest(qa_rapi.TestRapiInstanceMultiAlloc, pnode)
try:
RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_one,
True)
RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_two,
True)
finally:
rapi_instance_one.Release()
rapi_instance_two.Release()
finally:
pnode.Release()
config_list = [
("default-instance-tests", lambda: None, lambda _: None),
(IsExclusiveStorageInstanceTestEnabled,
lambda: qa_cluster.TestSetExclStorCluster(True),
qa_cluster.TestSetExclStorCluster),
]
for (conf_name, setup_conf_f, restore_conf_f) in config_list:
if qa_config.TestEnabled(conf_name):
oldconf = setup_conf_f()
RunTestBlock(RunInstanceTests)
restore_conf_f(oldconf)
pnode = qa_config.AcquireNode()
try:
if qa_config.TestEnabled(
["instance-add-plain-disk", "instance-export"]):
for shutdown in [False, True]:
instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk,
[pnode])
try:
expnode = qa_config.AcquireNode(exclude=pnode)
try:
if shutdown:
# Stop instance before exporting and removing it
RunTest(qa_instance.TestInstanceShutdown, instance)
RunTest(qa_instance.TestInstanceExportWithRemove,
instance, expnode)
RunTest(qa_instance.TestBackupList, expnode)
finally:
expnode.Release()
finally:
instance.Release()
del expnode
del instance
qa_cluster.AssertClusterVerify()
finally:
pnode.Release()
if qa_rapi.Enabled():
RunTestIf("filters", qa_rapi.TestFilters)
RunTestIf("cluster-upgrade", qa_cluster.TestUpgrade)
RunTestBlock(RunExclusiveStorageTests)
RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
TestIPolicyPlainInstance)
RunTestBlock(RunCustomSshPortTests)
RunTestIf("instance-add-restricted-by-disktemplates",
qa_instance.TestInstanceCreationRestrictedByDiskTemplates)
RunTestIf("instance-add-osparams", qa_instance.TestInstanceAddOsParams)
RunTestIf("instance-add-osparams", qa_instance.TestSecretOsParams)
# Test removing instance with offline drbd secondary
if qa_config.TestEnabled(
["instance-remove-drbd-offline", "instance-add-drbd-disk"]):
# Make sure the master is not put offline
snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
try:
pnode = qa_config.AcquireNode(exclude=snode)
try:
instance = qa_instance.TestInstanceAddWithDrbdDisk(
[pnode, snode])
set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance,
snode, set_offline, set_online)
finally:
pnode.Release()
finally:
snode.Release()
qa_cluster.AssertClusterVerify()
RunTestBlock(RunMonitoringTests)
RunPerformanceTests()
RunTestIf("cluster-destroy", qa_node.TestNodeRemoveAll)
RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
def TestRapiQuery():
"""Testing resource queries via remote API.
"""
# FIXME: the tests are failing if no LVM is enabled, investigate
# if it is a bug in the QA or in the code
if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG):
return
master_name = qa_utils.ResolveNodeName(qa_config.GetMasterNode())
rnd = random.Random(7818)
for what in constants.QR_VIA_RAPI:
namefield = {
constants.QR_JOB: "id",
constants.QR_EXPORT: "export",
constants.QR_FILTER: "uuid",
}.get(what, "name")
all_fields = list(query.ALL_FIELDS[what])
rnd.shuffle(all_fields)
# No fields, should return everything
result = _rapi_client.QueryFields(what)
qresult = objects.QueryFieldsResponse.FromDict(result)
AssertEqual(len(qresult.fields), len(all_fields))
# One field
result = _rapi_client.QueryFields(what, fields=[namefield])
qresult = objects.QueryFieldsResponse.FromDict(result)
AssertEqual(len(qresult.fields), 1)
# Specify all fields, order must be correct
result = _rapi_client.QueryFields(what, fields=all_fields)
qresult = objects.QueryFieldsResponse.FromDict(result)
AssertEqual(len(qresult.fields), len(all_fields))
AssertEqual([fdef.name for fdef in qresult.fields], all_fields)
# Unknown field
result = _rapi_client.QueryFields(what, fields=["_unknown!"])
qresult = objects.QueryFieldsResponse.FromDict(result)
AssertEqual(len(qresult.fields), 1)
AssertEqual(qresult.fields[0].name, "_unknown!")
AssertEqual(qresult.fields[0].kind, constants.QFT_UNKNOWN)
# Try once more, this time without the client
_DoTests([
("/2/query/%s/fields" % what, None, "GET", None),
("/2/query/%s/fields?fields=%s,%s,%s" %
(what, namefield, namefield, all_fields[0]), None, "GET", None),
])
# Try missing query argument
try:
_DoTests([
("/2/query/%s" % what, None, "GET", None),
])
except rapi.client.GanetiApiError as err:
AssertEqual(err.code, 400)
else:
raise qa_error.Error(
"Request missing 'fields' parameter didn't fail")
def _Check(exp_fields, data):
qresult = objects.QueryResponse.FromDict(data)
AssertEqual([fdef.name for fdef in qresult.fields], exp_fields)
if not isinstance(qresult.data, list):
raise qa_error.Error("Query did not return a list")
_DoTests([
# Specify fields in query
("/2/query/%s?fields=%s" % (what, ",".join(all_fields)),
compat.partial(_Check, all_fields), "GET", None),
("/2/query/%s?fields=%s" % (what, namefield),
compat.partial(_Check, [namefield]), "GET", None),
# Note the spaces
("/2/query/%s?fields=%s,%%20%s%%09,%s%%20" %
(what, namefield, namefield, namefield),
compat.partial(_Check, [namefield] * 3), "GET", None)
])
if what in constants.QR_VIA_RAPI_PUT:
_DoTests([
# PUT with fields in query
("/2/query/%s?fields=%s" % (what, namefield),
compat.partial(_Check, [namefield]), "PUT", {}),
("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4),
"PUT", {
"fields": [namefield] * 4,
}),
("/2/query/%s" % what, compat.partial(_Check,
all_fields), "PUT", {
"fields": all_fields,
}),
("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4),
"PUT", {
"fields": [namefield] * 4
})
])
if what in constants.QR_VIA_RAPI_PUT:
trivial_filter = {
constants.QR_JOB: [qlang.OP_GE, namefield, 0],
}.get(what, [qlang.OP_REGEXP, namefield, ".*"])
_DoTests([
# With filter
("/2/query/%s" % what, compat.partial(_Check,
all_fields), "PUT", {
"fields": all_fields,
"filter":
trivial_filter
}),
])
if what == constants.QR_NODE:
# Test with filter
(nodes, ) = _DoTests([("/2/query/%s" % what,
compat.partial(_Check, ["name", "master"]),
"PUT", {
"fields": ["name", "master"],
"filter": [qlang.OP_TRUE, "master"],
})])
qresult = objects.QueryResponse.FromDict(nodes)
AssertEqual(qresult.data, [
[[constants.RS_NORMAL, master_name],
[constants.RS_NORMAL, True]],
])
