def TestHooksCleanup(): """Remove the directories created by the tests """ master = GetMasterNode().primary GetCommandOutput(master, "rm %s/*" % MakeNodePath(master, PRE_PATH)) GetCommandOutput(master, "rm %s/*" % MakeNodePath(master, POST_PATH)) GetCommandOutput(master, "rm -rf %s" % _GetHDir())
def _TestGroupModifyISpecs(groupname):
# This test is built on the assumption that the default ipolicy holds for
# the node group under test
old_values = _GetGroupIPolicy(groupname)
samevals = dict((p, 4) for p in constants.ISPECS_PARAMETERS)
base_specs = {
constants.ISPECS_MINMAX: [{
constants.ISPECS_MIN: samevals,
constants.ISPECS_MAX: samevals,
}],
}
mod_values = _TestGroupSetISpecs(groupname, new_specs=base_specs,
old_values=old_values)
for par in constants.ISPECS_PARAMETERS:
# First make sure that the test works with good values
good_specs = {
constants.ISPECS_MINMAX: [{
constants.ISPECS_MIN: {par: 8},
constants.ISPECS_MAX: {par: 8},
}],
}
mod_values = _TestGroupSetISpecs(groupname, diff_specs=good_specs,
old_values=mod_values)
bad_specs = {
constants.ISPECS_MINMAX: [{
constants.ISPECS_MIN: {par: 8},
constants.ISPECS_MAX: {par: 4},
}],
}
_TestGroupSetISpecs(groupname, diff_specs=bad_specs, fail=True,
old_values=mod_values)
AssertCommand(["gnt-group", "modify", "--ipolicy-bounds-specs", "default",
groupname])
AssertEqual(_GetGroupIPolicy(groupname), old_values)
# Get the ipolicy command (from the cluster config)
mnode = qa_config.GetMasterNode()
addcmd = GetCommandOutput(mnode.primary, utils.ShellQuoteArgs([
"gnt-group", "show-ispecs-cmd", "--include-defaults", groupname,
]))
modcmd = ["gnt-group", "modify"]
opts = addcmd.split()
assert opts[0:2] == ["gnt-group", "add"]
for k in range(2, len(opts) - 1):
if opts[k].startswith("--ipolicy-"):
assert k + 2 <= len(opts)
modcmd.extend(opts[k:k + 2])
modcmd.append(groupname)
# Apply the ipolicy to the group and verify the result
AssertCommand(modcmd)
new_addcmd = GetCommandOutput(mnode.primary, utils.ShellQuoteArgs([
"gnt-group", "show-ispecs-cmd", groupname,
]))
AssertEqual(addcmd, new_addcmd)
def _GetJobStatuses():
""" Invokes gnt-job list and extracts an id to status dictionary.
@rtype: dict of string to string
@return: A dictionary mapping job ids to matching statuses
"""
master = qa_config.GetMasterNode()
list_output = GetCommandOutput(
master.primary, "gnt-job list --no-headers --output=id,status")
return dict(map(lambda s: s.split(), list_output.splitlines()))
def TestClusterModifyISpecs():
"""gnt-cluster modify --specs-*"""
params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
(cur_policy, cur_specs) = _GetClusterIPolicy()
# This test assumes that there is only one min/max bound
assert len(cur_specs[constants.ISPECS_MINMAX]) == 1
for par in params:
test_values = [
(True, 0, 4, 12),
(True, 4, 4, 12),
(True, 4, 12, 12),
(True, 4, 4, 4),
(False, 4, 0, 12),
(False, 4, 16, 12),
(False, 4, 4, 0),
(False, 12, 4, 4),
(False, 12, 4, 0),
(False, "a", 4, 12),
(False, 0, "a", 12),
(False, 0, 4, "a"),
# This is to restore the old values
(True,
cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par],
cur_specs[constants.ISPECS_STD][par],
cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par])
]
for (good, mn, st, mx) in test_values:
new_vals = {
constants.ISPECS_MINMAX: [{
constants.ISPECS_MIN: {par: mn},
constants.ISPECS_MAX: {par: mx}
}],
constants.ISPECS_STD: {par: st}
}
cur_state = (cur_policy, cur_specs)
# We update cur_specs, as we've copied the values to restore already
(cur_policy, cur_specs) = TestClusterSetISpecs(
diff_specs=new_vals, fail=not good, old_values=cur_state)
# Get the ipolicy command
mnode = qa_config.GetMasterNode()
initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
modcmd = ["gnt-cluster", "modify"]
opts = initcmd.split()
assert opts[0:2] == ["gnt-cluster", "init"]
for k in range(2, len(opts) - 1):
if opts[k].startswith("--ipolicy-"):
assert k + 2 <= len(opts)
modcmd.extend(opts[k:k + 2])
# Re-apply the ipolicy (this should be a no-op)
AssertCommand(modcmd)
new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
AssertEqual(initcmd, new_initcmd)
def _GetJobStatuses():
""" Invokes gnt-job list and extracts an id to status dictionary.
@rtype: dict of string to string
@return: A dictionary mapping job ids to matching statuses
"""
master = qa_config.GetMasterNode()
list_output = GetCommandOutput(
master.primary, "gnt-job list --no-headers --output=id,status"
)
return dict(map(lambda s: s.split(), list_output.splitlines()))
def AssertClusterVerify(fail=False,
errors=None,
warnings=None,
no_warnings=None):
"""Run cluster-verify and check the result, ignoring warnings by default.
@type fail: bool
@param fail: if cluster-verify is expected to fail instead of succeeding.
@type errors: list of tuples
@param errors: List of CV_XXX errors that are expected; if specified, all the
errors listed must appear in cluster-verify output. A non-empty value
implies C{fail=True}.
@type warnings: list of tuples
@param warnings: List of CV_XXX warnings that are expected to be raised; if
specified, all the errors listed must appear in cluster-verify output.
@type no_warnings: list of tuples
@param no_warnings: List of CV_XXX warnings that we expect NOT to be raised.
"""
cvcmd = "gnt-cluster verify"
mnode = qa_config.GetMasterNode()
if errors or warnings or no_warnings:
cvout = GetCommandOutput(mnode.primary,
cvcmd + " --error-codes",
fail=(fail or errors))
print cvout
(act_errs, act_warns) = _GetCVErrorCodes(cvout)
if errors:
_CheckVerifyErrors(act_errs, errors, "error")
if warnings:
_CheckVerifyErrors(act_warns, warnings, "warning")
if no_warnings:
_CheckVerifyNoWarnings(act_warns, no_warnings)
else:
AssertCommand(cvcmd, fail=fail, node=mnode)
def TestPauseWatcher():
"""Tests and pauses the watcher.
"""
master = qa_config.GetMasterNode()
AssertCommand(["gnt-cluster", "watcher", "pause", "4h"])
cmd = ["gnt-cluster", "watcher", "info"]
output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
AssertMatch(output, r"^.*\bis paused\b.*")
def TestJobCancellation():
"""gnt-job cancel"""
# The delay used for the first command should be large enough for the next
# command and the cancellation command to complete before the first job is
# done. The second delay should be small enough that not too much time is
# spend waiting in the case of a failed cancel and a running command.
FIRST_COMMAND_DELAY = 10.0
AssertCommand(["gnt-debug", "delay", "--submit", str(FIRST_COMMAND_DELAY)])
SECOND_COMMAND_DELAY = 3.0
master = qa_config.GetMasterNode()
# Forcing tty usage does not work on buildbot, so force all output of this
# command to be redirected to stdout
job_id_output = GetCommandOutput(
master.primary,
"gnt-debug delay --submit %s 2>&1" % SECOND_COMMAND_DELAY)
possible_job_ids = re.findall("JobID: ([0-9]+)", job_id_output)
if len(possible_job_ids) != 1:
raise qa_error.Error(
"Cannot parse gnt-debug delay output to find job id")
job_id = possible_job_ids[0]
AssertCommand(["gnt-job", "cancel", job_id])
# Now wait until the second job finishes, and expect the watch to fail due to
# job cancellation
AssertCommand(["gnt-job", "watch", job_id], fail=True)
# Then check for job cancellation
job_status = _GetJobStatus(job_id)
if job_status != constants.JOB_STATUS_CANCELED:
# Try and see if the job is being cancelled, and wait until the status
# changes or we hit a timeout
if job_status == constants.JOB_STATUS_CANCELING:
retry_fn = functools.partial(_RetryingFetchJobStatus,
constants.JOB_STATUS_CANCELING,
job_id)
try:
# The multiplier to use is arbitrary, setting it higher could prevent
# flakiness
WAIT_MULTIPLIER = 4.0
job_status = retry.Retry(retry_fn, 2.0,
WAIT_MULTIPLIER * FIRST_COMMAND_DELAY)
except retry.RetryTimeout:
# The job status remains the same
pass
if job_status != constants.JOB_STATUS_CANCELED:
raise qa_error.Error("Job was not successfully cancelled, status "
"found: %s" % job_status)
def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes):
"""gnt-cluster verify-disks with broken DRBD"""
qa_daemon.TestPauseWatcher()
try:
info = qa_instance.GetInstanceInfo(instance.name)
snode = inst_nodes[1]
for idx, minor in enumerate(info["drbd-minors"][snode.primary]):
if idx % 2 == 0:
break_drbd_cmd = \
"(drbdsetup %d down >/dev/null 2>&1;" \
" drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \
(minor, minor)
else:
break_drbd_cmd = \
"(drbdsetup %d detach >/dev/null 2>&1;" \
" drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \
(minor, minor)
AssertCommand(break_drbd_cmd, node=snode)
verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
"gnt-cluster verify-disks")
activation_msg = "Activating disks for instance '%s'" % instance.name
if activation_msg not in verify_output:
raise qa_error.Error(
"gnt-cluster verify-disks did not activate broken"
" DRBD disks:\n%s" % verify_output)
verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
"gnt-cluster verify-disks")
if activation_msg in verify_output:
raise qa_error.Error(
"gnt-cluster verify-disks wants to activate broken"
" DRBD disks on second attempt:\n%s" % verify_output)
AssertCommand(_CLUSTER_VERIFY)
finally:
qa_daemon.TestResumeWatcher()
def GetOutputFromMaster(cmd, use_multiplexer=True, log_cmd=True):
""" Gets the output of a command executed on master.
"""
if isinstance(cmd, basestring):
cmdstr = cmd
else:
cmdstr = utils.ShellQuoteArgs(cmd)
# Necessary due to the stderr stream not being captured properly on the
# buildbot
cmdstr += " 2>&1"
return GetCommandOutput(qa_config.GetMasterNode().primary, cmdstr,
use_multiplexer=use_multiplexer, log_cmd=log_cmd)
def TestNodeListDrbd(node, is_drbd):
"""gnt-node list-drbd"""
master = qa_config.GetMasterNode()
result_output = GetCommandOutput(master.primary,
"gnt-node list-drbd --no-header %s" %
node.primary)
# Meaningful to note: there is but one instance, and the node is either the
# primary or one of the secondaries
if is_drbd:
# Invoked for both primary and secondary
per_disk_info = result_output.splitlines()
for line in per_disk_info:
try:
drbd_node, _, _, _, _, drbd_peer = line.split()
except ValueError:
raise qa_error.Error("Could not examine list-drbd output: expected a"
" single row of 6 entries, found the following:"
" %s" % line)
AssertIn(node.primary, [drbd_node, drbd_peer],
msg="The output %s does not contain the node" % line)
else:
# Output should be empty, barring newlines
AssertEqual(result_output.strip(), "")
def TestClusterEpo():
"""gnt-cluster epo"""
master = qa_config.GetMasterNode()
# Assert that OOB is unavailable for all nodes
result_output = GetCommandOutput(
master.primary, "gnt-node list --verbose --no-headers -o"
" powered")
AssertEqual(
compat.all(powered == "(unavail)"
for powered in result_output.splitlines()), True)
# Conflicting
AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
# --all doesn't expect arguments
AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
# Unless --all is given master is not allowed to be in the list
AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
# This shouldn't fail
AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
# All instances should have been stopped now
result_output = GetCommandOutput(
master.primary, "gnt-instance list --no-headers -o status")
# ERROR_down because the instance is stopped but not recorded as such
AssertEqual(
compat.all(status == "ERROR_down"
for status in result_output.splitlines()), True)
# Now start everything again
AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
# All instances should have been started now
result_output = GetCommandOutput(
master.primary, "gnt-instance list --no-headers -o status")
AssertEqual(
compat.all(status == "running"
for status in result_output.splitlines()), True)
def TestHookDisappeared():
"""Checks whether the global hooks have been executed (status disappeared).
- Global pre hook should has been executed.
- Global post hook should with status *disappeared* should has been executed.
- Global post hook with other statuses shouldn't have been executed.
"""
master = GetMasterNode().primary
job_id = ExecuteJobProducingCommand("gnt-debug delay --submit 10")
time.sleep(1)
GetCommandOutput(master, "gnt-job cancel --kill --yes-do-it %d" % job_id)
time.sleep(10)
AssertEqual(IsFileExists(master, _GetHookFilePath(job_id, "pre")), True,
"Global pre hook hasn't been executed.")
AssertEqual(IsFileExists(master, _GetHookFilePath(job_id, "post",
constants.POST_HOOKS_STATUS_SUCCESS)), False,
"Global post hook has been executed with status *success*")
AssertEqual(IsFileExists(master, _GetHookFilePath(job_id, "post",
constants.POST_HOOKS_STATUS_ERROR)), False,
"Global post hook has been executed with status *error*")
AssertEqual(IsFileExists(master, _GetHookFilePath(job_id, "post",
constants.POST_HOOKS_STATUS_DISAPPEARED)), True,
"Global post hook hasn't been executed with status *disappeared*")
def TestClusterEpo():
"""gnt-cluster epo"""
master = qa_config.GetMasterNode()
# Assert that OOB is unavailable for all nodes
result_output = GetCommandOutput(master.primary,
"gnt-node list --verbose --no-headers -o"
" powered")
AssertEqual(compat.all(powered == "(unavail)"
for powered in result_output.splitlines()), True)
# Conflicting
AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
# --all doesn't expect arguments
AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
# Unless --all is given master is not allowed to be in the list
AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
# This shouldn't fail
AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
# All instances should have been stopped now
result_output = GetCommandOutput(master.primary,
"gnt-instance list --no-headers -o status")
# ERROR_down because the instance is stopped but not recorded as such
AssertEqual(compat.all(status == "ERROR_down"
for status in result_output.splitlines()), True)
# Now start everything again
AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
# All instances should have been started now
result_output = GetCommandOutput(master.primary,
"gnt-instance list --no-headers -o status")
AssertEqual(compat.all(status == "running"
for status in result_output.splitlines()), True)
def TestHooksInitialize():
"""Creates global hooks on the master node
"""
master = GetMasterNode().primary
hooks_base_dir = MakeNodePath(master, pathutils.HOOKS_BASE_DIR)
pre_path = MakeNodePath(master, PRE_PATH)
post_path = MakeNodePath(master, POST_PATH)
GetCommandOutput(master, "mkdir -p %s" % hooks_base_dir)
GetCommandOutput(master, "mkdir -p %s" % pre_path)
GetCommandOutput(master, "mkdir -p %s" % post_path)
GetCommandOutput(master, "mkdir -p %s" % _GetHDir())
h_name = "/qa_test_hook"
create_hook_common = """
FOUT=%s
echo '#!/bin/sh' > $FOUT
echo 'touch %s/$GANETI_JOB_ID"_"$GANETI_OP_CODE%s' >> $FOUT
chmod +x $FOUT
"""
create_pre = create_hook_common % (pre_path + h_name, _GetHDir(), '"_pre"')
create_post = create_hook_common % (post_path + h_name, _GetHDir(),
'"_post_"$GANETI_POST_STATUS')
GetCommandOutput(master, create_pre)
GetCommandOutput(master, create_post)
def TestClusterModifyISpecs():
"""gnt-cluster modify --specs-*"""
params = [
"memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"
]
(cur_policy, cur_specs) = _GetClusterIPolicy()
# This test assumes that there is only one min/max bound
assert len(cur_specs[constants.ISPECS_MINMAX]) == 1
for par in params:
test_values = [
(True, 0, 4, 12),
(True, 4, 4, 12),
(True, 4, 12, 12),
(True, 4, 4, 4),
(False, 4, 0, 12),
(False, 4, 16, 12),
(False, 4, 4, 0),
(False, 12, 4, 4),
(False, 12, 4, 0),
(False, "a", 4, 12),
(False, 0, "a", 12),
(False, 0, 4, "a"),
# This is to restore the old values
(True,
cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par],
cur_specs[constants.ISPECS_STD][par],
cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par])
]
for (good, mn, st, mx) in test_values:
new_vals = {
constants.ISPECS_MINMAX: [{
constants.ISPECS_MIN: {
par: mn
},
constants.ISPECS_MAX: {
par: mx
}
}],
constants.ISPECS_STD: {
par: st
}
}
cur_state = (cur_policy, cur_specs)
# We update cur_specs, as we've copied the values to restore already
(cur_policy,
cur_specs) = TestClusterSetISpecs(diff_specs=new_vals,
fail=not good,
old_values=cur_state)
# Get the ipolicy command
mnode = qa_config.GetMasterNode()
initcmd = GetCommandOutput(mnode.primary,
"gnt-cluster show-ispecs-cmd")
modcmd = ["gnt-cluster", "modify"]
opts = initcmd.split()
assert opts[0:2] == ["gnt-cluster", "init"]
for k in range(2, len(opts) - 1):
if opts[k].startswith("--ipolicy-"):
assert k + 2 <= len(opts)
modcmd.extend(opts[k:k + 2])
# Re-apply the ipolicy (this should be a no-op)
AssertCommand(modcmd)
new_initcmd = GetCommandOutput(mnode.primary,
"gnt-cluster show-ispecs-cmd")
AssertEqual(initcmd, new_initcmd)
def AssertInGroup(group, nodes):
real_output = GetCommandOutput(master_node,
"gnt-node list --no-headers -o group " +
utils.ShellQuoteArgs(nodes))
AssertEqual(real_output.splitlines(), [group] * len(nodes))