def _CreateRapiUser(rapi_user): """RAPI credentials creation, with the secret auto-generated. """ rapi_secret = utils.GenerateSecret() master = qa_config.GetMasterNode() rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE) rapi_dir = os.path.dirname(rapi_users_path) fh = tempfile.NamedTemporaryFile(mode="w") try: fh.write("%s %s write\n" % (rapi_user, rapi_secret)) fh.flush() tmpru = qa_utils.UploadFile(master.primary, fh.name) try: AssertCommand(["mkdir", "-p", rapi_dir]) AssertCommand(["mv", tmpru, rapi_users_path]) finally: AssertCommand(["rm", "-f", tmpru]) finally: fh.close() # The certificates have to be reloaded now AssertCommand(["service", "ganeti", "restart"]) return rapi_secret
def _ResetWatcherDaemon(): """Removes the watcher daemon's state file. """ path = \ qa_utils.MakeNodePath(qa_config.GetMasterNode(), pathutils.WATCHER_GROUP_STATE_FILE % "*-*-*-*") AssertCommand(["bash", "-c", "rm -vf %s" % path])
def RunCustomSshPortTests(): """Test accessing nodes with custom SSH ports. This requires removing nodes, adding them to a new group, and then undoing the change. """ if not qa_config.TestEnabled("group-custom-ssh-port"): return std_port = netutils.GetDaemonPort(constants.SSH) port = 211 master = qa_config.GetMasterNode() with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes: # Checks if the node(s) could be contacted through IPv6. # If yes, better skip the whole test. for node in nodes: if qa_utils.UsesIPv6Connection(node.primary, std_port): print("Node %s is likely to be reached using IPv6," "skipping the test" % (node.primary, )) return for node in nodes: qa_node.NodeRemove(node) with qa_iptables.RulesContext() as r: with qa_group.NewGroupCtx() as group: qa_group.ModifyGroupSshPort(r, group, nodes, port) for node in nodes: qa_node.NodeAdd(node, group=group) # Make sure that the cluster doesn't have any pre-existing problem qa_cluster.AssertClusterVerify() # Create and allocate instances instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes) try: instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes) try: # cluster-verify checks that disks are allocated correctly qa_cluster.AssertClusterVerify() # Remove instances qa_instance.TestInstanceRemove(instance2) qa_instance.TestInstanceRemove(instance1) finally: instance2.Release() finally: instance1.Release() for node in nodes: qa_node.NodeRemove(node) for node in nodes: qa_node.NodeAdd(node) qa_cluster.AssertClusterVerify()
def TestNodeModify(node): """gnt-node modify""" default_pool_size = 10 nodes = qa_config.GetAllNodes() test_pool_size = len(nodes) - 1 # Reduce the number of master candidates, because otherwise all # subsequent 'gnt-cluster verify' commands fail due to not enough # master candidates. AssertCommand( ["gnt-cluster", "modify", "--candidate-pool-size=%s" % test_pool_size]) # make sure enough master candidates will be available by disabling the # master candidate role first with --auto-promote AssertCommand([ "gnt-node", "modify", "--master-candidate=no", "--auto-promote", node.primary ]) # now it's save to force-remove the master candidate role for flag in ["master-candidate", "drained", "offline"]: for value in ["yes", "no"]: AssertCommand([ "gnt-node", "modify", "--force", "--%s=%s" % (flag, value), node.primary ]) AssertCommand(["gnt-cluster", "verify"]) AssertCommand( ["gnt-node", "modify", "--master-candidate=yes", node.primary]) # Test setting secondary IP address AssertCommand([ "gnt-node", "modify", "--secondary-ip=%s" % node.secondary, node.primary ]) AssertRedirectedCommand(["gnt-cluster", "verify"]) AssertCommand([ "gnt-cluster", "modify", "--candidate-pool-size=%s" % default_pool_size ]) # For test clusters with more nodes than the default pool size, # we now have too many master candidates. To readjust to the original # size, manually demote all nodes and rely on auto-promotion to adjust. if len(nodes) > default_pool_size: master = qa_config.GetMasterNode() for n in nodes: if n.primary != master.primary: AssertCommand([ "gnt-node", "modify", "--master-candidate=no", "--auto-promote", n.primary ])
def MarkNodeAddedAll(): """Mark all nodes as added. This is useful if we don't create the cluster ourselves (in qa). """ master = qa_config.GetMasterNode() for node in qa_config.get("nodes"): if node != master: node.MarkAdded()
def _AssertOobCall(verify_path, expected_args): """Assert the OOB call was performed with expetected args.""" master = qa_config.GetMasterNode() verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path]) output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd, tty=False) AssertEqual(expected_args, output.strip())
def TestResumeWatcher(): """Tests and unpauses the watcher. """ master = qa_config.GetMasterNode() AssertCommand(["gnt-cluster", "watcher", "continue"]) cmd = ["gnt-cluster", "watcher", "info"] output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertMatch(output, r"^.*\bis not paused\b.*")
def _TestGroupModifyISpecs(groupname): # This test is built on the assumption that the default ipolicy holds for # the node group under test old_values = _GetGroupIPolicy(groupname) samevals = dict((p, 4) for p in constants.ISPECS_PARAMETERS) base_specs = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: samevals, constants.ISPECS_MAX: samevals, }], } mod_values = _TestGroupSetISpecs(groupname, new_specs=base_specs, old_values=old_values) for par in constants.ISPECS_PARAMETERS: # First make sure that the test works with good values good_specs = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: {par: 8}, constants.ISPECS_MAX: {par: 8}, }], } mod_values = _TestGroupSetISpecs(groupname, diff_specs=good_specs, old_values=mod_values) bad_specs = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: {par: 8}, constants.ISPECS_MAX: {par: 4}, }], } _TestGroupSetISpecs(groupname, diff_specs=bad_specs, fail=True, old_values=mod_values) AssertCommand(["gnt-group", "modify", "--ipolicy-bounds-specs", "default", groupname]) AssertEqual(_GetGroupIPolicy(groupname), old_values) # Get the ipolicy command (from the cluster config) mnode = qa_config.GetMasterNode() addcmd = GetCommandOutput(mnode.primary, utils.ShellQuoteArgs([ "gnt-group", "show-ispecs-cmd", "--include-defaults", groupname, ])) modcmd = ["gnt-group", "modify"] opts = addcmd.split() assert opts[0:2] == ["gnt-group", "add"] for k in range(2, len(opts) - 1): if opts[k].startswith("--ipolicy-"): assert k + 2 <= len(opts) modcmd.extend(opts[k:k + 2]) modcmd.append(groupname) # Apply the ipolicy to the group and verify the result AssertCommand(modcmd) new_addcmd = GetCommandOutput(mnode.primary, utils.ShellQuoteArgs([ "gnt-group", "show-ispecs-cmd", groupname, ])) AssertEqual(addcmd, new_addcmd)
def TestJobCancellation(): """gnt-job cancel""" # The delay used for the first command should be large enough for the next # command and the cancellation command to complete before the first job is # done. The second delay should be small enough that not too much time is # spend waiting in the case of a failed cancel and a running command. FIRST_COMMAND_DELAY = 10.0 AssertCommand(["gnt-debug", "delay", "--submit", str(FIRST_COMMAND_DELAY)]) SECOND_COMMAND_DELAY = 3.0 master = qa_config.GetMasterNode() # Forcing tty usage does not work on buildbot, so force all output of this # command to be redirected to stdout job_id_output = GetCommandOutput( master.primary, "gnt-debug delay --submit %s 2>&1" % SECOND_COMMAND_DELAY) possible_job_ids = re.findall("JobID: ([0-9]+)", job_id_output) if len(possible_job_ids) != 1: raise qa_error.Error( "Cannot parse gnt-debug delay output to find job id") job_id = possible_job_ids[0] AssertCommand(["gnt-job", "cancel", job_id]) # Now wait until the second job finishes, and expect the watch to fail due to # job cancellation AssertCommand(["gnt-job", "watch", job_id], fail=True) # Then check for job cancellation job_status = qa_job_utils.GetJobStatus(job_id) if job_status != constants.JOB_STATUS_CANCELED: # Try and see if the job is being cancelled, and wait until the status # changes or we hit a timeout if job_status == constants.JOB_STATUS_CANCELING: retry_fn = functools.partial(qa_job_utils.RetryingWhileJobStatus, constants.JOB_STATUS_CANCELING, job_id) try: # The multiplier to use is arbitrary, setting it higher could prevent # flakiness WAIT_MULTIPLIER = 4.0 job_status = retry.Retry(retry_fn, 2.0, WAIT_MULTIPLIER * FIRST_COMMAND_DELAY) except retry.RetryTimeout: # The job status remains the same pass if job_status != constants.JOB_STATUS_CANCELED: raise qa_error.Error("Job was not successfully cancelled, status " "found: %s" % job_status)
def ReloadCertificates(ensure_presence=True): """Reloads the client RAPI certificate with the one present on the node. If the QA is set up to use a specific certificate using the "rapi-files-location" parameter, it will be put in place prior to retrieving it. """ if ensure_presence: _EnsureRapiFilesPresence() if _rapi_username is None or _rapi_password is None: raise qa_error.Error("RAPI username and password have to be set before" " attempting to reload a certificate.") # pylint: disable=W0603 # due to global usage global _rapi_ca global _rapi_client master = qa_config.GetMasterNode() # Load RAPI certificate from master node cmd = [ "openssl", "x509", "-in", qa_utils.MakeNodePath(master, pathutils.RAPI_CERT_FILE) ] # Write to temporary file _rapi_ca = tempfile.NamedTemporaryFile(mode="w") _rapi_ca.write( qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))) _rapi_ca.flush() port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT) cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name, proxy="") if qa_config.UseVirtualCluster(): # TODO: Implement full support for RAPI on virtual clusters print( qa_logging.FormatWarning("RAPI tests are not yet supported on" " virtual clusters and will be disabled")) assert _rapi_client is None else: _rapi_client = rapi.client.GanetiRapiClient(master.primary, port=port, username=_rapi_username, password=_rapi_password, curl_config_fn=cfg_curl) print("RAPI protocol version: %s" % _rapi_client.GetVersion())
def RunInstanceTestsFull(create_fun, inodes, supported_conversions, templ): instance = RunTest(create_fun, inodes) try: RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown, instance) RunTestIf("instance-communication", qa_instance.TestInstanceCommunication, instance, qa_config.GetMasterNode()) RunTestIf("cluster-epo", qa_cluster.TestClusterEpo) RunDaemonTests(instance) for node in inodes: RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node, templ == constants.DT_DRBD8) if len(inodes) > 1: RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit, constants.INITIAL_NODE_GROUP_NAME, inodes[0].primary, inodes[1].primary) # This test will run once but it will cover all the supported # user-provided disk template conversions if qa_config.TestEnabled("instance-convert-disk"): if (len(supported_conversions) > 1 and instance.disk_template in supported_conversions): RunTest(qa_instance.TestInstanceShutdown, instance) RunTest(qa_instance.TestInstanceConvertDiskTemplate, instance, supported_conversions) RunTest(qa_instance.TestInstanceStartup, instance) # At this point we clear the set because the requested conversions # has been tested supported_conversions.clear() else: test_desc = "Converting instance of template %s" % templ ReportTestSkip(test_desc, "conversion feature") RunTestIf("instance-modify-disks", qa_instance.TestInstanceModifyDisks, instance) RunCommonInstanceTests(instance, inodes) if qa_config.TestEnabled("instance-modify-primary"): othernode = qa_config.AcquireNode() RunTest(qa_instance.TestInstanceModifyPrimaryAndBack, instance, inodes[0], othernode) othernode.Release() RunGroupListTests() RunExportImportTests(instance, inodes) RunHardwareFailureTests(instance, inodes) RunRepairDiskSizes() RunTestIf(["rapi", "instance-data-censorship"], qa_rapi.TestInstanceDataCensorship, instance, inodes) RunTest(qa_instance.TestInstanceRemove, instance) finally: instance.Release() del instance qa_cluster.AssertClusterVerify()
def _ReadSsconfInstanceList(): """Reads ssconf_instance_list from the master node. """ master = qa_config.GetMasterNode() ssconf_path = utils.PathJoin(pathutils.DATA_DIR, "ssconf_%s" % constants.SS_INSTANCE_LIST) cmd = ["cat", qa_utils.MakeNodePath(master, ssconf_path)] return qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)).splitlines()
def _InstanceRunning(name): """Checks whether an instance is running. @param name: full name of the instance """ master = qa_config.GetMasterNode() cmd = ( utils.ShellQuoteArgs(["gnt-instance", "list", "-o", "status", name]) + ' | grep running') ret = StartSSH(master.primary, cmd).wait() return ret == 0
def GetObjectInfo(infocmd): """Get and parse information about a Ganeti object. @type infocmd: list of strings @param infocmd: command to be executed, e.g. ["gnt-cluster", "info"] @return: the information parsed, appropriately stored in dictionaries, lists... """ master = qa_config.GetMasterNode() cmdline = utils.ShellQuoteArgs(infocmd) info_out = GetCommandOutput(master.primary, cmdline) return yaml.load(info_out)
def GetOutputFromMaster(cmd, use_multiplexer=True, log_cmd=True): """ Gets the output of a command executed on master. """ if isinstance(cmd, str): cmdstr = cmd else: cmdstr = utils.ShellQuoteArgs(cmd) # Necessary due to the stderr stream not being captured properly on the # buildbot cmdstr += " 2>&1" return GetCommandOutput(qa_config.GetMasterNode().primary, cmdstr, use_multiplexer=use_multiplexer, log_cmd=log_cmd)
def _List(listcmd, fields, names): """Runs a list command. """ master = qa_config.GetMasterNode() cmd = [ listcmd, "list", "--separator=|", "--no-headers", "--output", ",".join(fields) ] if names: cmd.extend(names) return GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)).splitlines()
def _CreateOobScriptStructure(): """Create a simple OOB handling script and its structure.""" master = qa_config.GetMasterNode() data_path = qa_utils.UploadData(master.primary, "") verify_path = qa_utils.UploadData(master.primary, "") exit_code_path = qa_utils.UploadData(master.primary, "") oob_script = (("#!/bin/bash\n" "echo \"$@\" > %s\n" "cat %s\n" "exit $(< %s)\n") % (utils.ShellQuote(verify_path), utils.ShellQuote(data_path), utils.ShellQuote(exit_code_path))) oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0o700) return [oob_path, verify_path, data_path, exit_code_path]
def TestInterClusterInstanceMove(src_instance, dest_instance, inodes, tnode, perform_checks=True): """Test tools/move-instance""" master = qa_config.GetMasterNode() rapi_pw_file = tempfile.NamedTemporaryFile(mode="w") rapi_pw_file.write(_rapi_password) rapi_pw_file.flush() # Needed only if checks are to be performed if perform_checks: dest_instance.SetDiskTemplate(src_instance.disk_template) # TODO: Run some instance tests before moving back if len(inodes) > 1: # No disk template currently requires more than 1 secondary node. If this # changes, either this test must be skipped or the script must be updated. assert len(inodes) == 2 snode = inodes[1] else: # Instance is not redundant, but we still need to pass a node # (which will be ignored) snode = tnode pnode = inodes[0] # pnode:snode are the *current* nodes, and the first move is an # iallocator-guided move outside of pnode. The node lock for the pnode # assures that this happens, and while we cannot be sure where the instance # will land, it is a real move. locks = {locking.LEVEL_NODE: [pnode.primary]} RunWithLocks(_InvokeMoveInstance, locks, 600.0, False, dest_instance.name, src_instance.name, rapi_pw_file.name, master.primary, perform_checks) # And then back to pnode:snode _InvokeMoveInstance(src_instance.name, dest_instance.name, rapi_pw_file.name, master.primary, perform_checks, target_nodes=(pnode.primary, snode.primary))
def GenericQueryFieldsTest(cmd, fields): master = qa_config.GetMasterNode() # Listing fields AssertRedirectedCommand([cmd, "list-fields"]) AssertRedirectedCommand([cmd, "list-fields"] + fields) # Check listed fields (all, must be sorted) realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"] output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(realcmd)).splitlines() AssertEqual([line.split("|", 1)[0] for line in output], utils.NiceSort(fields)) # Check exit code for listing unknown field rcode, _, _ = AssertCommand([cmd, "list-fields", "field/does/not/exist"], fail=True) AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD)
def RemoveFromEtcHosts(hostnames): """Remove hostnames from /etc/hosts. @param hostnames: List of hostnames first used A records, all other CNAMEs """ master = qa_config.GetMasterNode() tmp_hosts = UploadData(master.primary, "", mode=0o644) quoted_tmp_hosts = utils.ShellQuote(tmp_hosts) sed_data = " ".join(hostnames) try: AssertCommand( (r"sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s" r" && mv %s %s") % (sed_data, utils.ShellQuote(pathutils.ETC_HOSTS), quoted_tmp_hosts, quoted_tmp_hosts, utils.ShellQuote(pathutils.ETC_HOSTS))) except Exception: AssertCommand(["rm", "-f", tmp_hosts]) raise
def AddToEtcHosts(hostnames): """Adds hostnames to /etc/hosts. @param hostnames: List of hostnames first used A records, all other CNAMEs """ master = qa_config.GetMasterNode() tmp_hosts = UploadData(master.primary, "", mode=0o644) data = [] for localhost in ("::1", "127.0.0.1"): data.append("%s %s" % (localhost, " ".join(hostnames))) try: AssertCommand( "{ cat %s && echo -e '%s'; } > %s && mv %s %s" % (utils.ShellQuote(pathutils.ETC_HOSTS), "\\n".join(data), utils.ShellQuote(tmp_hosts), utils.ShellQuote(tmp_hosts), utils.ShellQuote(pathutils.ETC_HOSTS))) except Exception: AssertCommand(["rm", "-f", tmp_hosts]) raise
def GetNodeInstances(node, secondaries=False): """Gets a list of instances on a node. """ master = qa_config.GetMasterNode() node_name = ResolveNodeName(node) # Get list of all instances cmd = [ "gnt-instance", "list", "--separator=:", "--no-headers", "--output=name,pnode,snodes" ] output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) instances = [] for line in output.splitlines(): (name, pnode, snodes) = line.split(":", 2) if ((not secondaries and pnode == node_name) or (secondaries and node_name in snodes.split(","))): instances.append(name) return instances
def TestAssignNodesIncludingSplit(orig_group, node1, node2): """gnt-group assign-nodes --force Expects node1 and node2 to be primary and secondary for a common instance. """ assert node1 != node2 (other_group, ) = qa_utils.GetNonexistentGroups(1) master_node = qa_config.GetMasterNode().primary def AssertInGroup(group, nodes): real_output = GetCommandOutput(master_node, "gnt-node list --no-headers -o group " + utils.ShellQuoteArgs(nodes)) AssertEqual(real_output.splitlines(), [group] * len(nodes)) AssertInGroup(orig_group, [node1, node2]) AssertCommand(["gnt-group", "add", other_group]) try: AssertCommand(["gnt-group", "assign-nodes", other_group, node1, node2]) AssertInGroup(other_group, [node1, node2]) # This should fail because moving node1 to orig_group would leave their # common instance split between orig_group and other_group. AssertCommand(["gnt-group", "assign-nodes", orig_group, node1], fail=True) AssertInGroup(other_group, [node1, node2]) AssertCommand(["gnt-group", "assign-nodes", "--force", orig_group, node1]) AssertInGroup(orig_group, [node1]) AssertInGroup(other_group, [node2]) AssertCommand(["gnt-group", "assign-nodes", orig_group, node2]) AssertInGroup(orig_group, [node1, node2]) finally: AssertCommand(["gnt-group", "remove", other_group])
def RunInstanceCheck(instance, running): """Check if instance is running or not. """ instance_name = _GetName(instance, operator.attrgetter("name")) script = qa_config.GetInstanceCheckScript() if not script: return master_node = qa_config.GetMasterNode() # Build command to connect to master node master_ssh = GetSSHCommand(master_node.primary, "--") if running: running_shellval = "1" running_text = "" else: running_shellval = "" running_text = "not " print( FormatInfo("Checking if instance '%s' is %srunning" % (instance_name, running_text))) args = [script, instance_name] env = { "PATH": constants.HOOKS_PATH, "RUN_UUID": _RUN_UUID, "MASTER_SSH": utils.ShellQuoteArgs(master_ssh), "INSTANCE_NAME": instance_name, "INSTANCE_RUNNING": running_shellval, } result = os.spawnve(os.P_WAIT, script, args, env) if result != 0: raise qa_error.Error("Instance check failed with result %s" % result)
def main(): """Main program. """ colors.check_for_colors() parser = optparse.OptionParser(usage="%prog [options] <config-file>") parser.add_option("--yes-do-it", dest="yes_do_it", action="store_true", help="Really execute the tests") (opts, args) = parser.parse_args() if len(args) == 1: (config_file, ) = args else: parser.error("Wrong number of arguments.") if not opts.yes_do_it: print("Executing this script irreversibly destroys any Ganeti\n" "configuration on all nodes involved. If you really want\n" "to start testing, supply the --yes-do-it option.") sys.exit(1) qa_config.Load(config_file) primary = qa_config.GetMasterNode().primary qa_utils.StartMultiplexer(primary) print("SSH command for primary node: %s" % utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, ""))) print("SSH command for other nodes: %s" % utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", ""))) try: RunQa() finally: qa_utils.CloseMultiplexers()
def _LookupRapiSecret(rapi_user): """Find the RAPI secret for the given user on the QA machines. @param rapi_user: Login user @return: Login secret for the user """ CTEXT = "{CLEARTEXT}" master = qa_config.GetMasterNode() cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)] file_content = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) users = ParsePasswordFile(file_content) entry = users.get(rapi_user) if not entry: raise qa_error.Error("User %s not found in RAPI users file" % rapi_user) secret = entry.password if secret.upper().startswith(CTEXT): secret = secret[len(CTEXT):] elif secret.startswith("{"): raise qa_error.Error("Unsupported password schema for RAPI user %s:" " not a clear text password" % rapi_user) return secret
def TestNodeListDrbd(node, is_drbd): """gnt-node list-drbd""" master = qa_config.GetMasterNode() result_output = GetCommandOutput( master.primary, "gnt-node list-drbd --no-header %s" % node.primary) # Meaningful to note: there is but one instance, and the node is either the # primary or one of the secondaries if is_drbd: # Invoked for both primary and secondary per_disk_info = result_output.splitlines() for line in per_disk_info: try: drbd_node, _, _, _, _, drbd_peer = line.split() except ValueError: raise qa_error.Error( "Could not examine list-drbd output: expected a" " single row of 6 entries, found the following:" " %s" % line) AssertIn(node.primary, [drbd_node, drbd_peer], msg="The output %s does not contain the node" % line) else: # Output should be empty, barring newlines AssertEqual(result_output.strip(), "")
def _TestOs(mode, rapi_cb): """Generic function for OS definition testing """ master = qa_config.GetMasterNode() name = _TEMP_OS_NAME variant = "default" fullname = "%s+%s" % (name, variant) dirname = _TEMP_OS_PATH # Ensure OS is usable cmd = ["gnt-os", "modify", "--hidden=no", "--blacklisted=no", name] AssertCommand(cmd) nodes = [] try: for i, node in enumerate(qa_config.get("nodes")): nodes.append(node) if mode == _ALL_INVALID: valid = False elif mode == _ALL_VALID: valid = True elif mode == _PARTIALLY_VALID: valid = bool(i % 2) else: raise AssertionError("Unknown mode %s" % mode) _SetupTempOs(node, dirname, variant, valid) # TODO: Use Python 2.6's itertools.permutations for (hidden, blacklisted) in [(False, False), (True, False), (False, True), (True, True)]: # Change OS' visibility cmd = ["gnt-os", "modify", "--hidden", ["no", "yes"][int(hidden)], "--blacklisted", ["no", "yes"][int(blacklisted)], name] AssertCommand(cmd) # Diagnose, checking exit status AssertCommand(["gnt-os", "diagnose"], fail=(mode != _ALL_VALID)) # Diagnose again, ignoring exit status output = qa_utils.GetCommandOutput(master.primary, "gnt-os diagnose || :") for line in output.splitlines(): if line.startswith("OS: %s [global status:" % name): break else: raise qa_error.Error("Didn't find OS '%s' in 'gnt-os diagnose'" % name) # Check info for all cmd = ["gnt-os", "info"] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertIn("%s:" % name, output.splitlines()) # Check info for OS cmd = ["gnt-os", "info", name] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)).splitlines() AssertIn("%s:" % name, output) for (field, value) in [("valid", mode == _ALL_VALID), ("hidden", hidden), ("blacklisted", blacklisted)]: AssertIn(" - %s: %s" % (field, value), output) # Only valid OSes should be listed cmd = ["gnt-os", "list", "--no-headers"] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) if mode == _ALL_VALID and not (hidden or blacklisted): assert_fn = AssertIn else: assert_fn = AssertNotIn assert_fn(fullname, output.splitlines()) # Check via RAPI if rapi_cb: assert_fn(fullname, rapi_cb()) finally: for node in nodes: _RemoveTempOs(node, dirname)
def RunQa(): """Main QA body. """ RunTestBlock(RunEnvTests) SetupCluster() RunTestBlock(RunClusterTests) RunTestBlock(RunOsTests) RunTestIf("tags", qa_tags.TestClusterTags) RunTestBlock(RunCommonNodeTests) RunTestBlock(RunGroupListTests) RunTestBlock(RunGroupRwTests) RunTestBlock(RunNetworkTests) RunTestBlock(RunFilterTests) # The master shouldn't be readded or put offline; "delay" needs a non-master # node to test pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode()) try: RunTestIf("node-readd", qa_node.TestNodeReadd, pnode) RunTestIf("node-modify", qa_node.TestNodeModify, pnode) RunTestIf("delay", qa_cluster.TestDelay, pnode) finally: pnode.Release() # Make sure the cluster is clean before running instance tests qa_cluster.AssertClusterVerify() pnode = qa_config.AcquireNode() try: RunTestIf("tags", qa_tags.TestNodeTags, pnode) if qa_rapi.Enabled(): RunTest(qa_rapi.TestNode, pnode) if (qa_config.TestEnabled("instance-add-plain-disk") and qa_config.IsTemplateSupported(constants.DT_PLAIN)): # Normal instance allocation via RAPI for use_client in [True, False]: rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode, use_client) try: if qa_config.TestEnabled( "instance-plain-rapi-common-tests"): RunCommonInstanceTests(rapi_instance, [pnode]) RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client) finally: rapi_instance.Release() del rapi_instance # Multi-instance allocation rapi_instance_one, rapi_instance_two = \ RunTest(qa_rapi.TestRapiInstanceMultiAlloc, pnode) try: RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_one, True) RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_two, True) finally: rapi_instance_one.Release() rapi_instance_two.Release() finally: pnode.Release() config_list = [ ("default-instance-tests", lambda: None, lambda _: None), (IsExclusiveStorageInstanceTestEnabled, lambda: qa_cluster.TestSetExclStorCluster(True), qa_cluster.TestSetExclStorCluster), ] for (conf_name, setup_conf_f, restore_conf_f) in config_list: if qa_config.TestEnabled(conf_name): oldconf = setup_conf_f() RunTestBlock(RunInstanceTests) restore_conf_f(oldconf) pnode = qa_config.AcquireNode() try: if qa_config.TestEnabled( ["instance-add-plain-disk", "instance-export"]): for shutdown in [False, True]: instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode]) try: expnode = qa_config.AcquireNode(exclude=pnode) try: if shutdown: # Stop instance before exporting and removing it RunTest(qa_instance.TestInstanceShutdown, instance) RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode) RunTest(qa_instance.TestBackupList, expnode) finally: expnode.Release() finally: instance.Release() del expnode del instance qa_cluster.AssertClusterVerify() finally: pnode.Release() if qa_rapi.Enabled(): RunTestIf("filters", qa_rapi.TestFilters) RunTestIf("cluster-upgrade", qa_cluster.TestUpgrade) RunTestBlock(RunExclusiveStorageTests) RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"], TestIPolicyPlainInstance) RunTestBlock(RunCustomSshPortTests) RunTestIf("instance-add-restricted-by-disktemplates", qa_instance.TestInstanceCreationRestrictedByDiskTemplates) RunTestIf("instance-add-osparams", qa_instance.TestInstanceAddOsParams) RunTestIf("instance-add-osparams", qa_instance.TestSecretOsParams) # Test removing instance with offline drbd secondary if qa_config.TestEnabled( ["instance-remove-drbd-offline", "instance-add-drbd-disk"]): # Make sure the master is not put offline snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode()) try: pnode = qa_config.AcquireNode(exclude=snode) try: instance = qa_instance.TestInstanceAddWithDrbdDisk( [pnode, snode]) set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes") set_online = lambda node: qa_node.MakeNodeOffline(node, "no") RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode, set_offline, set_online) finally: pnode.Release() finally: snode.Release() qa_cluster.AssertClusterVerify() RunTestBlock(RunMonitoringTests) RunPerformanceTests() RunTestIf("cluster-destroy", qa_node.TestNodeRemoveAll) RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
def TestRapiQuery(): """Testing resource queries via remote API. """ # FIXME: the tests are failing if no LVM is enabled, investigate # if it is a bug in the QA or in the code if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG): return master_name = qa_utils.ResolveNodeName(qa_config.GetMasterNode()) rnd = random.Random(7818) for what in constants.QR_VIA_RAPI: namefield = { constants.QR_JOB: "id", constants.QR_EXPORT: "export", constants.QR_FILTER: "uuid", }.get(what, "name") all_fields = list(query.ALL_FIELDS[what]) rnd.shuffle(all_fields) # No fields, should return everything result = _rapi_client.QueryFields(what) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), len(all_fields)) # One field result = _rapi_client.QueryFields(what, fields=[namefield]) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), 1) # Specify all fields, order must be correct result = _rapi_client.QueryFields(what, fields=all_fields) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), len(all_fields)) AssertEqual([fdef.name for fdef in qresult.fields], all_fields) # Unknown field result = _rapi_client.QueryFields(what, fields=["_unknown!"]) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), 1) AssertEqual(qresult.fields[0].name, "_unknown!") AssertEqual(qresult.fields[0].kind, constants.QFT_UNKNOWN) # Try once more, this time without the client _DoTests([ ("/2/query/%s/fields" % what, None, "GET", None), ("/2/query/%s/fields?fields=%s,%s,%s" % (what, namefield, namefield, all_fields[0]), None, "GET", None), ]) # Try missing query argument try: _DoTests([ ("/2/query/%s" % what, None, "GET", None), ]) except rapi.client.GanetiApiError as err: AssertEqual(err.code, 400) else: raise qa_error.Error( "Request missing 'fields' parameter didn't fail") def _Check(exp_fields, data): qresult = objects.QueryResponse.FromDict(data) AssertEqual([fdef.name for fdef in qresult.fields], exp_fields) if not isinstance(qresult.data, list): raise qa_error.Error("Query did not return a list") _DoTests([ # Specify fields in query ("/2/query/%s?fields=%s" % (what, ",".join(all_fields)), compat.partial(_Check, all_fields), "GET", None), ("/2/query/%s?fields=%s" % (what, namefield), compat.partial(_Check, [namefield]), "GET", None), # Note the spaces ("/2/query/%s?fields=%s,%%20%s%%09,%s%%20" % (what, namefield, namefield, namefield), compat.partial(_Check, [namefield] * 3), "GET", None) ]) if what in constants.QR_VIA_RAPI_PUT: _DoTests([ # PUT with fields in query ("/2/query/%s?fields=%s" % (what, namefield), compat.partial(_Check, [namefield]), "PUT", {}), ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", { "fields": [namefield] * 4, }), ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", { "fields": all_fields, }), ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", { "fields": [namefield] * 4 }) ]) if what in constants.QR_VIA_RAPI_PUT: trivial_filter = { constants.QR_JOB: [qlang.OP_GE, namefield, 0], }.get(what, [qlang.OP_REGEXP, namefield, ".*"]) _DoTests([ # With filter ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", { "fields": all_fields, "filter": trivial_filter }), ]) if what == constants.QR_NODE: # Test with filter (nodes, ) = _DoTests([("/2/query/%s" % what, compat.partial(_Check, ["name", "master"]), "PUT", { "fields": ["name", "master"], "filter": [qlang.OP_TRUE, "master"], })]) qresult = objects.QueryResponse.FromDict(nodes) AssertEqual(qresult.data, [ [[constants.RS_NORMAL, master_name], [constants.RS_NORMAL, True]], ])