def SetupRapi(): """Sets up the RAPI certificate and usernames for the client. """ if not Enabled(): return (None, None) # pylint: disable=W0603 # due to global usage global _rapi_username global _rapi_password _rapi_username = qa_config.get("rapi-user", "ganeti-qa") if qa_config.TestEnabled("create-cluster") and \ qa_config.get("rapi-files-location") is None: # For a new cluster, we have to invent a secret and a user, unless it has # been provided separately _rapi_password = _CreateRapiUser(_rapi_username) else: _EnsureRapiFilesPresence() _rapi_password = _GetRapiSecret(_rapi_username) # Once a username and password have been set, we can fetch the certs and # get all we need for a working RAPI client. ReloadCertificates(ensure_presence=False)
def _GenInstanceAllocationDict(node, instance): """Creates an instance allocation dict to be used with the RAPI""" instance.SetDiskTemplate(constants.DT_PLAIN) disks = [{ "size": utils.ParseUnit(d.get("size")), "name": str(d.get("name")) } for d in qa_config.GetDiskOptions()] nic0_mac = instance.GetNicMacAddr(0, constants.VALUE_GENERATE) nics = [{ constants.INIC_MAC: nic0_mac, }] beparams = { constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)), constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)), } return _rapi_client.InstanceAllocation(constants.INSTANCE_CREATE, instance.name, constants.DT_PLAIN, disks, nics, os=qa_config.get("os"), pnode=node.primary, beparams=beparams)
def GetGenericAddParameters(inst, disk_template, force_mac=None): params = ["-B"] params.append("%s=%s,%s=%s" % (constants.BE_MINMEM, qa_config.get(constants.BE_MINMEM), constants.BE_MAXMEM, qa_config.get(constants.BE_MAXMEM))) if disk_template != constants.DT_DISKLESS: for idx, disk in enumerate(qa_config.GetDiskOptions()): size = disk.get("size") name = disk.get("name") diskparams = "%s:size=%s" % (idx, size) if name: diskparams += ",name=%s" % name if qa_config.AreSpindlesSupported(): spindles = disk.get("spindles") if spindles is None: raise qa_error.Error("'spindles' is a required parameter for disks" " when you enable exclusive storage tests") diskparams += ",spindles=%s" % spindles params.extend(["--disk", diskparams]) # Set static MAC address if configured if force_mac: nic0_mac = force_mac else: nic0_mac = inst.GetNicMacAddr(0, None) if nic0_mac: params.extend(["--net", "0:mac=%s" % nic0_mac]) return params
def TestIcmpPing(): """ICMP ping each node. """ nodes = qa_config.get("nodes") pingprimary = pingsecondary = "fping" if qa_config.get("primary_ip_version") == 6: pingprimary = "fping6" pricmd = [pingprimary, "-e"] seccmd = [pingsecondary, "-e"] for i in nodes: pricmd.append(i.primary) if i.secondary: seccmd.append(i.secondary) pristr = utils.ShellQuoteArgs(pricmd) if seccmd: cmdall = "%s && %s" % (pristr, utils.ShellQuoteArgs(seccmd)) else: cmdall = pristr for node in nodes: AssertCommand(cmdall, node=node)
def TestRapiInstanceAdd(node, use_client): """Test adding a new instance via RAPI""" if not qa_config.IsTemplateSupported(constants.DT_PLAIN): return instance = qa_config.AcquireInstance() instance.SetDiskTemplate(constants.DT_PLAIN) try: disks = [{ "size": utils.ParseUnit(d.get("size")), "name": str(d.get("name")) } for d in qa_config.GetDiskOptions()] nic0_mac = instance.GetNicMacAddr(0, constants.VALUE_GENERATE) nics = [{ constants.INIC_MAC: nic0_mac, }] beparams = { constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)), constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)), } if use_client: job_id = _rapi_client.CreateInstance(constants.INSTANCE_CREATE, instance.name, constants.DT_PLAIN, disks, nics, os=qa_config.get("os"), pnode=node.primary, beparams=beparams) else: body = { "__version__": 1, "mode": constants.INSTANCE_CREATE, "name": instance.name, "os_type": qa_config.get("os"), "disk_template": constants.DT_PLAIN, "pnode": node.primary, "beparams": beparams, "disks": disks, "nics": nics, } (job_id, ) = _DoTests([ ("/2/instances", _VerifyReturnsJob, "POST", body), ]) _WaitForRapiJob(job_id) return instance except: instance.Release() raise
def _SubmitInstanceCreationJob(instance, disk_template=None): """Submit an instance creation job. @type instance: L{qa_config._QaInstance} @param instance: instance to submit a create command for @type disk_template: string @param disk_template: disk template for the new instance or C{None} which causes the default disk template to be used @rtype: int @return: job id of the submitted creation job """ if disk_template is None: disk_template = qa_config.GetDefaultDiskTemplate() try: cmd = (["gnt-instance", "add", "--submit", "--opportunistic-locking", "--os-type=%s" % qa_config.get("os"), "--disk-template=%s" % disk_template] + GetGenericAddParameters(instance, disk_template)) cmd.append(instance.name) instance.SetDiskTemplate(disk_template) return _ExecuteJobSubmittingCmd(cmd) except: instance.Release() raise
def GetNonexistentEntityNames(count, name_config, name_prefix): """Gets entity names which shouldn't exist on the cluster. The actualy names can refer to arbitrary entities (for example groups, networks). @param count: Number of names to get @rtype: integer @param name_config: name of the leaf in the config containing this entity's configuration, including a 'inexistent-' element @rtype: string @param name_prefix: prefix of the entity's names, used to compose the default values; for example for groups, the prefix is 'group' and the generated names are then group1, group2, ... @rtype: string """ entities = qa_config.get(name_config, {}) default = [name_prefix + str(i) for i in range(count)] assert count <= len(default) name_config_inexistent = "inexistent-" + name_config candidates = entities.get(name_config_inexistent, default)[:count] if len(candidates) < count: raise Exception("At least %s non-existent %s are needed" % (count, name_config)) return candidates
def ModifyGroupSshPort(ipt_rules, group, nodes, ssh_port): """Modifies the node group settings and sets up iptable rules. For each pair of nodes add two rules that affect SSH connections from one to the other one. The first one redirects port 22 to some unused port so that connecting through 22 fails. The second redirects port `ssh_port` to port 22. Together this results in master seeing the SSH daemons on the nodes on `ssh_port` instead of 22. """ default_ssh_port = netutils.GetDaemonPort(constants.SSH) all_nodes = qa_config.get("nodes") AssertCommand(["gnt-group", "modify", "--node-parameters=ssh_port=" + str(ssh_port), group]) for node in nodes: ipt_rules.RedirectPort(node.primary, "localhost", default_ssh_port, 65535) ipt_rules.RedirectPort(node.primary, "localhost", ssh_port, default_ssh_port) for node2 in all_nodes: ipt_rules.RedirectPort(node2.primary, node.primary, default_ssh_port, 65535) ipt_rules.RedirectPort(node2.primary, node.primary, ssh_port, default_ssh_port)
def TestGanetiCommands(): """Test availibility of Ganeti commands. """ cmds = (["gnt-backup", "--version"], ["gnt-cluster", "--version"], ["gnt-debug", "--version"], ["gnt-instance", "--version"], ["gnt-job", "--version"], ["gnt-network", "--version"], ["gnt-node", "--version"], ["gnt-os", "--version"], ["gnt-storage", "--version"], ["gnt-filter", "--version"], ["ganeti-noded", "--version"], ["ganeti-rapi", "--version"], ["ganeti-watcher", "--version"], ["ganeti-confd", "--version"], ["ganeti-luxid", "--version"], ["ganeti-wconfd", "--version"], ) cmd = " && ".join([utils.ShellQuoteArgs(i) for i in cmds]) for node in qa_config.get("nodes"): AssertCommand(cmd, node=node)
def TestParallelModify(instances): """PERFORMANCE: Parallel instance modify. @type instances: list of L{qa_config._QaInstance} @param instances: list of instances to issue modify commands against """ job_driver = _JobQueueDriver() # set min mem to same value as max mem new_min_mem = qa_config.get(constants.BE_MAXMEM) for instance in instances: cmd = (["gnt-instance", "modify", "--submit", "-B", "%s=%s" % (constants.BE_MINMEM, new_min_mem)]) cmd.append(instance.name) job_driver.AddJob(_ExecuteJobSubmittingCmd(cmd)) cmd = (["gnt-instance", "modify", "--submit", "-O", "fake_os_param=fake_value"]) cmd.append(instance.name) job_driver.AddJob(_ExecuteJobSubmittingCmd(cmd)) cmd = (["gnt-instance", "modify", "--submit", "-O", "fake_os_param=fake_value", "-B", "%s=%s" % (constants.BE_MINMEM, new_min_mem)]) cmd.append(instance.name) job_driver.AddJob(_ExecuteJobSubmittingCmd(cmd)) job_driver.WaitForCompletion()
def IsExclusiveStorageInstanceTestEnabled(): test_name = "exclusive-storage-instance-tests" if qa_config.TestEnabled(test_name): vgname = qa_config.get("vg-name", constants.DEFAULT_VG) vgscmd = utils.ShellQuoteArgs([ "vgs", "--noheadings", "-o", "pv_count", vgname, ]) nodes = qa_config.GetConfig()["nodes"] for node in nodes: try: pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd)) except Exception as e: msg = ( "Cannot get the number of PVs on %s, needed by '%s': %s" % (node.primary, test_name, e)) raise qa_error.Error(msg) if pvnum < 2: raise qa_error.Error( "Node %s has not enough PVs (%s) to run '%s'" % (node.primary, pvnum, test_name)) res = True else: res = False return res
def TestNetworkConnect(): """gnt-network connect/disconnect""" (group1, ) = qa_utils.GetNonexistentGroups(1) (network1, ) = GetNonexistentNetworks(1) default_mode = "bridged" default_link = "xen-br0" nicparams = qa_config.get("default-nicparams") if nicparams: mode = nicparams.get("mode", default_mode) link = nicparams.get("link", default_link) else: mode = default_mode link = default_link nicparams = "mode=%s,link=%s" % (mode, link) AssertCommand(["gnt-group", "add", group1]) AssertCommand( ["gnt-network", "add", "--network", "192.0.2.0/24", network1]) AssertCommand([ "gnt-network", "connect", "--nic-parameters", nicparams, network1, group1 ]) TestNetworkList() AssertCommand(["gnt-network", "disconnect", network1, group1]) AssertCommand(["gnt-group", "remove", group1]) AssertCommand(["gnt-network", "remove", network1])
def _EnsureRapiFilesPresence(): """Ensures that the specified RAPI files are present on the cluster, if any. """ rapi_files_location = qa_config.get("rapi-files-location", None) if rapi_files_location is None: # No files to be had return print( qa_logging.FormatWarning("Replacing the certificate and users file on" " the node with the ones provided in %s" % rapi_files_location)) # The RAPI files AssertCommand(["mkdir", "-p", pathutils.RAPI_DATA_DIR]) for filename in _FILES_TO_COPY: basename = os.path.split(filename)[-1] AssertCommand( ["cp", os.path.join(rapi_files_location, basename), filename]) AssertCommand(["gnt-cluster", "copyfile", filename]) # The certificates have to be reloaded now AssertCommand(["service", "ganeti", "restart"])
def ConfigureGroups(): """Configures groups and nodes for tests such as custom SSH ports. """ defgroup = GetDefaultGroup() nodes = qa_config.get("nodes") options = qa_config.get("options", {}) # Clear any old configuration qa_iptables.CleanRules(nodes) # Custom SSH ports: ssh_port = options.get("ssh-port") default_ssh_port = netutils.GetDaemonPort(constants.SSH) if (ssh_port is not None) and (ssh_port != default_ssh_port): ModifyGroupSshPort(qa_iptables.GLOBAL_RULES, defgroup, nodes, ssh_port)
def _FilterTags(seq): """Removes unwanted tags from a sequence. """ ignore_re = qa_config.get("ignore-tags-re", None) if ignore_re: return itertools.filterfalse(re.compile(ignore_re).match, seq) else: return seq
def MarkNodeAddedAll(): """Mark all nodes as added. This is useful if we don't create the cluster ourselves (in qa). """ master = qa_config.GetMasterNode() for node in qa_config.get("nodes"): if node != master: node.MarkAdded()
def ReloadCertificates(ensure_presence=True): """Reloads the client RAPI certificate with the one present on the node. If the QA is set up to use a specific certificate using the "rapi-files-location" parameter, it will be put in place prior to retrieving it. """ if ensure_presence: _EnsureRapiFilesPresence() if _rapi_username is None or _rapi_password is None: raise qa_error.Error("RAPI username and password have to be set before" " attempting to reload a certificate.") # pylint: disable=W0603 # due to global usage global _rapi_ca global _rapi_client master = qa_config.GetMasterNode() # Load RAPI certificate from master node cmd = [ "openssl", "x509", "-in", qa_utils.MakeNodePath(master, pathutils.RAPI_CERT_FILE) ] # Write to temporary file _rapi_ca = tempfile.NamedTemporaryFile(mode="w") _rapi_ca.write( qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))) _rapi_ca.flush() port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT) cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name, proxy="") if qa_config.UseVirtualCluster(): # TODO: Implement full support for RAPI on virtual clusters print( qa_logging.FormatWarning("RAPI tests are not yet supported on" " virtual clusters and will be disabled")) assert _rapi_client is None else: _rapi_client = rapi.client.GanetiRapiClient(master.primary, port=port, username=_rapi_username, password=_rapi_password, curl_config_fn=cfg_curl) print("RAPI protocol version: %s" % _rapi_client.GetVersion())
def RunInstanceTests(): """Create and exercise instances.""" requested_conversions = qa_config.get("convert-disk-templates", []) supported_conversions = \ set(requested_conversions).difference(constants.DTS_NOT_CONVERTIBLE_TO) for (test_name, templ, create_fun, num_nodes) in \ qa_instance.available_instance_tests: if (qa_config.TestEnabled(test_name) and qa_config.IsTemplateSupported(templ)): inodes = qa_config.AcquireManyNodes(num_nodes) try: # run instance tests with default hvparams print( _FormatHeader( "Starting instance tests with default HVparams")) RunInstanceTestsFull(create_fun, inodes, supported_conversions, templ) # iterate through alternating hvparam values (if enabled) if qa_config.TestEnabled("instance-iterate-hvparams"): hvparam_iterations = qa_cluster.PrepareHvParameterSets() for param, test_data in hvparam_iterations.items(): for value in test_data["values"]: print( _FormatHeader( "Starting reduced number of instance tests " "with hypervisor parameter %s=%s" % (param, value))) qa_cluster.AssertClusterHvParameterModify( param, value) RunInstanceTestsReduced(create_fun, inodes) qa_cluster.AssertClusterHvParameterModify( param, test_data["reset_value"]) else: test_desc = "Iterating through hypervisor parameter values" ReportTestSkip(test_desc, "instance-iterate-hvparams") finally: qa_config.ReleaseManyNodes(inodes) else: test_desc = "Creating instances of template %s" % templ if not qa_config.TestEnabled(test_name): ReportTestSkip(test_desc, test_name) else: ReportTestSkip(test_desc, "disk template %s" % templ)
def _GetRapiSecret(rapi_user): """Returns the secret to be used for RAPI access. Where exactly this secret can be found depends on the QA configuration options, and this function invokes additional tools as needed. It can look up a local secret, a remote one, or create a user with a new secret. @param rapi_user: Login user @return: Login secret for the user """ password_file_path = qa_config.get("rapi-password-file", None) if password_file_path is not None: # If the password file is specified, we use the password within. # The file must be present on the QA runner. return _ReadRapiSecret(password_file_path) else: # On an existing cluster, just find out the user's secret return _LookupRapiSecret(rapi_user)
def SetupCluster(): """Initializes the cluster. """ RunTestIf("create-cluster", qa_cluster.TestClusterInit) if not qa_config.TestEnabled("create-cluster"): # If the cluster is already in place, we assume that exclusive-storage is # already set according to the configuration qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False)) qa_rapi.SetupRapi() qa_group.ConfigureGroups() # Test on empty cluster RunTestIf("node-list", qa_node.TestNodeList) RunTestIf("instance-list", qa_instance.TestInstanceList) RunTestIf("job-list", qa_job.TestJobList) RunTestIf("create-cluster", qa_node.TestNodeAddAll) if not qa_config.TestEnabled("create-cluster"): # consider the nodes are already there qa_node.MarkNodeAddedAll() RunTestIf("test-jobqueue", qa_cluster.TestJobqueue) RunTestIf("test-jobqueue", qa_job.TestJobCancellation) # enable the watcher (unconditionally) RunTest(qa_daemon.TestResumeWatcher) RunTestIf("node-list", qa_node.TestNodeList) # Test listing fields RunTestIf("node-list", qa_node.TestNodeListFields) RunTestIf("instance-list", qa_instance.TestInstanceListFields) RunTestIf("job-list", qa_job.TestJobListFields) RunTestIf("instance-export", qa_instance.TestBackupListFields) RunTestIf("node-info", qa_node.TestNodeInfo)
def _CreateInstanceByDiskTemplateRaw(nodes_spec, disk_template, fail=False): """Creates an instance with the given disk template on the given nodes(s). Note that this function does not check if enough nodes are given for the respective disk template. @type nodes_spec: string @param nodes_spec: string specification of one node (by node name) or several nodes according to the requirements of the disk template @type disk_template: string @param disk_template: the disk template to be used by the instance @return: the created instance """ instance = qa_config.AcquireInstance() try: cmd = (["gnt-instance", "add", "--os-type=%s" % qa_config.get("os"), "--disk-template=%s" % disk_template, "--node=%s" % nodes_spec] + GetGenericAddParameters(instance, disk_template)) cmd.append(instance.name) AssertCommand(cmd, fail=fail) if not fail: CheckSsconfInstanceList(instance.name) instance.SetDiskTemplate(disk_template) return instance except: instance.Release() raise # Handle the case where creation is expected to fail assert fail instance.Release() return None
def RunInstanceTests(): """Create and exercise instances.""" requested_conversions = qa_config.get("convert-disk-templates", []) supported_conversions = \ set(requested_conversions).difference(constants.DTS_NOT_CONVERTIBLE_TO) for (test_name, templ, create_fun, num_nodes) in \ qa_instance.available_instance_tests: if (qa_config.TestEnabled(test_name) and qa_config.IsTemplateSupported(templ)): inodes = qa_config.AcquireManyNodes(num_nodes) try: instance = RunTest(create_fun, inodes) try: RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown, instance) RunTestIf("instance-communication", qa_instance.TestInstanceCommunication, instance, qa_config.GetMasterNode()) RunTestIf("cluster-epo", qa_cluster.TestClusterEpo) RunDaemonTests(instance) for node in inodes: RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node, templ == constants.DT_DRBD8) if len(inodes) > 1: RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit, constants.INITIAL_NODE_GROUP_NAME, inodes[0].primary, inodes[1].primary) # This test will run once but it will cover all the supported # user-provided disk template conversions if qa_config.TestEnabled("instance-convert-disk"): if (len(supported_conversions) > 1 and instance.disk_template in supported_conversions): RunTest(qa_instance.TestInstanceShutdown, instance) RunTest( qa_instance.TestInstanceConvertDiskTemplate, instance, supported_conversions) RunTest(qa_instance.TestInstanceStartup, instance) # At this point we clear the set because the requested conversions # has been tested supported_conversions.clear() else: test_desc = "Converting instance of template %s" % templ ReportTestSkip(test_desc, "conversion feature") RunTestIf("instance-modify-disks", qa_instance.TestInstanceModifyDisks, instance) RunCommonInstanceTests(instance, inodes) if qa_config.TestEnabled("instance-modify-primary"): othernode = qa_config.AcquireNode() RunTest(qa_instance.TestInstanceModifyPrimaryAndBack, instance, inodes[0], othernode) othernode.Release() RunGroupListTests() RunExportImportTests(instance, inodes) RunHardwareFailureTests(instance, inodes) RunRepairDiskSizes() RunTestIf(["rapi", "instance-data-censorship"], qa_rapi.TestInstanceDataCensorship, instance, inodes) RunTest(qa_instance.TestInstanceRemove, instance) finally: instance.Release() del instance finally: qa_config.ReleaseManyNodes(inodes) qa_cluster.AssertClusterVerify() else: test_desc = "Creating instances of template %s" % templ if not qa_config.TestEnabled(test_name): ReportTestSkip(test_desc, test_name) else: ReportTestSkip(test_desc, "disk template %s" % templ)
def GetDefaultGroup(): """Returns the default node group. """ groups = qa_config.get("groups", {}) return groups.get("group-with-nodes", constants.INITIAL_NODE_GROUP_NAME)
def TestNodeAddAll(): """Adding all nodes to cluster.""" master = qa_config.GetMasterNode() for node in qa_config.get("nodes"): if node != master: NodeAdd(node, readd=False)
def TestSshConnection(): """Test SSH connection. """ for node in qa_config.get("nodes"): AssertCommand("exit", node=node)
def _TestOs(mode, rapi_cb): """Generic function for OS definition testing """ master = qa_config.GetMasterNode() name = _TEMP_OS_NAME variant = "default" fullname = "%s+%s" % (name, variant) dirname = _TEMP_OS_PATH # Ensure OS is usable cmd = ["gnt-os", "modify", "--hidden=no", "--blacklisted=no", name] AssertCommand(cmd) nodes = [] try: for i, node in enumerate(qa_config.get("nodes")): nodes.append(node) if mode == _ALL_INVALID: valid = False elif mode == _ALL_VALID: valid = True elif mode == _PARTIALLY_VALID: valid = bool(i % 2) else: raise AssertionError("Unknown mode %s" % mode) _SetupTempOs(node, dirname, variant, valid) # TODO: Use Python 2.6's itertools.permutations for (hidden, blacklisted) in [(False, False), (True, False), (False, True), (True, True)]: # Change OS' visibility cmd = ["gnt-os", "modify", "--hidden", ["no", "yes"][int(hidden)], "--blacklisted", ["no", "yes"][int(blacklisted)], name] AssertCommand(cmd) # Diagnose, checking exit status AssertCommand(["gnt-os", "diagnose"], fail=(mode != _ALL_VALID)) # Diagnose again, ignoring exit status output = qa_utils.GetCommandOutput(master.primary, "gnt-os diagnose || :") for line in output.splitlines(): if line.startswith("OS: %s [global status:" % name): break else: raise qa_error.Error("Didn't find OS '%s' in 'gnt-os diagnose'" % name) # Check info for all cmd = ["gnt-os", "info"] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertIn("%s:" % name, output.splitlines()) # Check info for OS cmd = ["gnt-os", "info", name] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)).splitlines() AssertIn("%s:" % name, output) for (field, value) in [("valid", mode == _ALL_VALID), ("hidden", hidden), ("blacklisted", blacklisted)]: AssertIn(" - %s: %s" % (field, value), output) # Only valid OSes should be listed cmd = ["gnt-os", "list", "--no-headers"] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) if mode == _ALL_VALID and not (hidden or blacklisted): assert_fn = AssertIn else: assert_fn = AssertNotIn assert_fn(fullname, output.splitlines()) # Check via RAPI if rapi_cb: assert_fn(fullname, rapi_cb()) finally: for node in nodes: _RemoveTempOs(node, dirname)
def TestNodeRemoveAll(): """Removing all nodes from cluster.""" master = qa_config.GetMasterNode() for node in qa_config.get("nodes"): if node != master: NodeRemove(node)