def check_memory(self, vm=None): """ Check is guest memory is really match assgined to VM. :param vm: VM object, get VM object from env if vm is None. """ error_context.context("Verify memory info", logging.info) if not vm: vm = self.env.get_vm(self.params["main_vm"]) vm.verify_alive() threshold = float(self.params.get("threshold", 0.10)) timeout = float(self.params.get("wait_resume_timeout", 60)) # Notes: # some sub test will pause VM, here need to wait VM resume # then check memory info in guest. utils_misc.wait_for(lambda: not vm.is_paused(), timeout=timeout) utils_misc.verify_dmesg() self.os_type = self.params.get("os_type") guest_mem_size = super(MemoryHotplugTest, self).get_guest_total_mem(vm) vm_mem_size = self.get_vm_mem(vm) if abs(guest_mem_size - vm_mem_size) > vm_mem_size * threshold: msg = ("Assigned '%s MB' memory to '%s'" "but, '%s MB' memory detect by OS" % (vm_mem_size, vm.name, guest_mem_size)) raise exceptions.TestFail(msg)
def run(test, params, env): """ Run htm cases: Case one 1) Download unit test suite and configure it 2) Run kvm test on host 3) Check host is still available Case two 1) Download test application in the guest 2) Run it in the guest :param test: QEMU test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ if params["unit_test"] == "yes": error_context.context("Prepare unit test on host", logging.info) cmds = [params["get_htm_dir"], params["compile_htm"]] for cmd in cmds: s, o = process.getstatusoutput(cmd, timeout=3600) if s: test.error("Failed to run cmd '%s', output: %s" % (cmd, o)) error_context.context("Run htm unit test on host", logging.info) s, o = process.getstatusoutput(params["run_htm_test"], timeout=3600) if s: test.fail("Run htm unit test failed, output: %s" % o) # Make sure if host is available by do commands on host status, output = process.getstatusoutput("rm -rf %s" % params["htm_dir"]) if status: test.fail("Please check host's status: %s" % output) utils_misc.verify_dmesg() else: check_exist_cmd = params["check_htm_env"] s, o = process.getstatusoutput(check_exist_cmd) if s: test.error( "Please check htm is supported or not by '%s', output: %s" % (check_exist_cmd, o)) vm = env.get_vm(params["main_vm"]) session = vm.wait_for_login() pkgs = params["depends_pkgs"].split() if not utils_package.package_install(pkgs, session): test.error("Install dependency packages failed") session.cmd(params["get_htm_dir"]) download_htm_demo = params["download_htm_demo"] status = session.cmd_status("wget %s" % download_htm_demo) if status: test.error( "Failed to download test file, please configure it in cfg : %s" % download_htm_demo) else: status, output = session.cmd_status_output( params["test_htm_command"]) if not re.search(params["expected_htm_test_result"], output): test.fail("Test failed and please check : %s" % output) vm.verify_kernel_crash()
def run(test, params, env): """ Test rbd disk device. 1.Prepare test environment,destroy or suspend a VM. 2.Prepare disk image. 3.Edit disks xml and start the domain. 4.Perform test operation. 5.Recover test environment. """ vm_name = params.get("main_vm") vm = env.get_vm(vm_name) virsh_dargs = {'debug': True, 'ignore_status': True} # Global variable to store max/current memory, # it may change after attach/detach new_max_mem = None new_cur_mem = None def get_vm_memtotal(session): """ Get guest total memory """ proc_meminfo = session.cmd_output("cat /proc/meminfo") # verify format and units are expected return int( re.search(r'MemTotal:\s+(\d+)\s+[kK]B', proc_meminfo).group(1)) def consume_vm_mem(size=1000, timeout=360): """ To consume guest memory, default size is 1000M """ session = vm.wait_for_login() # Mount tmpfs on /mnt and write to a file on it, # it is the memory operation sh_cmd = ("swapoff -a; mount -t tmpfs -o size={0}M tmpfs " "/mnt; dd if=/dev/urandom of=/mnt/test bs=1M" " count={0}".format(size)) session.cmd(sh_cmd, timeout=timeout) session.close() def check_qemu_cmd(): """ Check qemu command line options. """ cmd = ("ps -ef | grep %s | grep -v grep " % vm_name) if max_mem_rt: cmd += (" | grep 'slots=%s,maxmem=%sk'" % (max_mem_slots, max_mem_rt)) if tg_size: size = int(tg_size) * 1024 cmd_str = 'memdimm.\|memory-backend-ram,id=ram-node.' cmd += (" | grep 'memory-backend-ram,id=%s' | grep 'size=%s" % (cmd_str, size)) if pg_size: cmd += ",host-nodes=%s" % node_mask if numa_memnode: for node in numa_memnode: if ('nodeset' in node and node['nodeset'] in node_mask): cmd += ",policy=%s" % node['mode'] cmd += ".*pc-dimm,node=%s" % tg_node if mem_addr: cmd += (".*slot=%s,addr=%s" % (mem_addr['slot'], int(mem_addr['base'], 16))) cmd += "'" # Run the command process.run(cmd, shell=True) def check_guest_meminfo(old_mem, check_option): """ Check meminfo on guest. """ assert old_mem is not None session = vm.wait_for_login() # Hot-plugged memory should be online by udev rules udev_file = "/lib/udev/rules.d/80-hotplug-cpu-mem.rules" udev_rules = ('SUBSYSTEM=="memory", ACTION=="add", TEST=="state",' ' ATTR{state}=="offline", ATTR{state}="online"') cmd = ("grep memory %s || echo '%s' >> %s" % (udev_file, udev_rules, udev_file)) session.cmd(cmd) # Wait a while for new memory to be detected. utils_misc.wait_for(lambda: get_vm_memtotal(session) != int(old_mem), 20, first=15.0) new_mem = get_vm_memtotal(session) session.close() logging.debug("Memtotal on guest: %s", new_mem) no_of_times = 1 if at_times: no_of_times = at_times if check_option == "attach": if new_mem != int(old_mem) + (int(tg_size) * no_of_times): test.fail("Total memory on guest couldn't changed after " "attach memory device") if check_option == "detach": if new_mem != int(old_mem) - (int(tg_size) * no_of_times): test.fail("Total memory on guest couldn't changed after " "detach memory device") def check_dom_xml(at_mem=False, dt_mem=False): """ Check domain xml options. """ # Global variable to store max/current memory global new_max_mem global new_cur_mem if attach_option.count("config"): dom_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) else: dom_xml = vm_xml.VMXML.new_from_dumpxml(vm_name) try: xml_max_mem_rt = int(dom_xml.max_mem_rt) xml_max_mem = int(dom_xml.max_mem) xml_cur_mem = int(dom_xml.current_mem) assert int(max_mem_rt) == xml_max_mem_rt # Check attached/detached memory if at_mem: if at_times: assert int(max_mem) + (int(tg_size) * at_times) == xml_max_mem else: assert int(max_mem) + int(tg_size) == xml_max_mem # Bug 1220702, skip the check for current memory if at_times: assert int(cur_mem) + (int(tg_size) * at_times) == xml_cur_mem else: assert int(cur_mem) + int(tg_size) == xml_cur_mem new_max_mem = xml_max_mem new_cur_mem = xml_cur_mem mem_dev = dom_xml.get_devices("memory") memory_devices = 1 if at_times: memory_devices = at_times if len(mem_dev) != memory_devices: test.fail("Found wrong number of memory device") assert int(tg_size) == int(mem_dev[0].target.size) assert int(tg_node) == int(mem_dev[0].target.node) elif dt_mem: if at_times: assert int(new_max_mem) - (int(tg_size) * at_times) == xml_max_mem assert int(new_cur_mem) - (int(tg_size) * at_times) == xml_cur_mem else: assert int(new_max_mem) - int(tg_size) == xml_max_mem # Bug 1220702, skip the check for current memory assert int(new_cur_mem) - int(tg_size) == xml_cur_mem except AssertionError: utils_misc.log_last_traceback() test.fail("Found unmatched memory setting from domain xml") def check_save_restore(): """ Test save and restore operation """ save_file = os.path.join(data_dir.get_tmp_dir(), "%s.save" % vm_name) ret = virsh.save(vm_name, save_file, **virsh_dargs) libvirt.check_exit_status(ret) ret = virsh.restore(save_file, **virsh_dargs) libvirt.check_exit_status(ret) if os.path.exists(save_file): os.remove(save_file) # Login to check vm status vm.wait_for_login().close() def create_mem_xml(): """ Create memory device xml. """ mem_xml = memory.Memory() mem_model = params.get("mem_model", "dimm") mem_xml.mem_model = mem_model if tg_size: tg_xml = memory.Memory.Target() tg_xml.size = int(tg_size) tg_xml.size_unit = tg_sizeunit # There is support for non-numa node if numa_cells: tg_xml.node = int(tg_node) mem_xml.target = tg_xml if pg_size: src_xml = memory.Memory.Source() src_xml.pagesize = int(pg_size) src_xml.pagesize_unit = pg_unit src_xml.nodemask = node_mask mem_xml.source = src_xml if mem_addr: mem_xml.address = mem_xml.new_mem_address(**{"attrs": mem_addr}) logging.debug("Memory device xml: %s", mem_xml) return mem_xml.copy() def add_device(dev_xml, at_error=False): """ Add memory device by attachment or modify domain xml. """ if attach_device: ret = virsh.attach_device(vm_name, dev_xml.xml, flagstr=attach_option) libvirt.check_exit_status(ret, at_error) else: vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) if numa_cells: del vmxml.max_mem del vmxml.current_mem vmxml.add_device(dev_xml) vmxml.sync() def modify_domain_xml(): """ Modify domain xml and define it. """ vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) mem_unit = params.get("mem_unit", "KiB") vcpu = params.get("vcpu", "4") if max_mem_rt: vmxml.max_mem_rt = int(max_mem_rt) vmxml.max_mem_rt_slots = max_mem_slots vmxml.max_mem_rt_unit = mem_unit if vcpu: vmxml.vcpu = int(vcpu) vcpu_placement = params.get("vcpu_placement", "static") vmxml.placement = vcpu_placement if numa_memnode: vmxml.numa_memory = {} vmxml.numa_memnode = numa_memnode else: try: del vmxml.numa_memory del vmxml.numa_memnode except Exception: # Not exists pass if numa_cells: cells = [ast.literal_eval(x) for x in numa_cells] # Rounding the numa memory values if align_mem_values: for cell in range(cells.__len__()): memory_value = str( utils_numeric.align_value(cells[cell]["memory"], align_to_value)) cells[cell]["memory"] = memory_value cpu_xml = vm_xml.VMCPUXML() cpu_xml.xml = "<cpu><numa/></cpu>" cpu_mode = params.get("cpu_mode") model_fallback = params.get("model_fallback") if cpu_mode: cpu_xml.mode = cpu_mode if model_fallback: cpu_xml.fallback = model_fallback cpu_xml.numa_cell = cells vmxml.cpu = cpu_xml # Delete memory and currentMemory tag, # libvirt will fill it automatically del vmxml.max_mem del vmxml.current_mem # hugepages setting if huge_pages: membacking = vm_xml.VMMemBackingXML() hugepages = vm_xml.VMHugepagesXML() pagexml_list = [] for i in range(len(huge_pages)): pagexml = hugepages.PageXML() pagexml.update(huge_pages[i]) pagexml_list.append(pagexml) hugepages.pages = pagexml_list membacking.hugepages = hugepages vmxml.mb = membacking logging.debug("vm xml: %s", vmxml) vmxml.sync() pre_vm_state = params.get("pre_vm_state", "running") attach_device = "yes" == params.get("attach_device", "no") detach_device = "yes" == params.get("detach_device", "no") attach_error = "yes" == params.get("attach_error", "no") start_error = "yes" == params.get("start_error", "no") detach_error = "yes" == params.get("detach_error", "no") maxmem_error = "yes" == params.get("maxmem_error", "no") attach_option = params.get("attach_option", "") test_qemu_cmd = "yes" == params.get("test_qemu_cmd", "no") test_managedsave = "yes" == params.get("test_managedsave", "no") test_save_restore = "yes" == params.get("test_save_restore", "no") test_mem_binding = "yes" == params.get("test_mem_binding", "no") restart_libvirtd = "yes" == params.get("restart_libvirtd", "no") add_mem_device = "yes" == params.get("add_mem_device", "no") test_dom_xml = "yes" == params.get("test_dom_xml", "no") max_mem = params.get("max_mem") max_mem_rt = params.get("max_mem_rt") max_mem_slots = params.get("max_mem_slots", "16") cur_mem = params.get("current_mem") numa_cells = params.get("numa_cells", "").split() set_max_mem = params.get("set_max_mem") align_mem_values = "yes" == params.get("align_mem_values", "no") align_to_value = int(params.get("align_to_value", "65536")) known_unplug_errors = [] known_unplug_errors.append(params.get("known_unplug_errors")) # params for attached device tg_size = params.get("tg_size") tg_sizeunit = params.get("tg_sizeunit", 'KiB') tg_node = params.get("tg_node", 0) pg_size = params.get("page_size") pg_unit = params.get("page_unit", "KiB") node_mask = params.get("node_mask", "0") mem_addr = ast.literal_eval(params.get("memory_addr", "{}")) huge_pages = [ ast.literal_eval(x) for x in params.get("huge_pages", "").split() ] numa_memnode = [ ast.literal_eval(x) for x in params.get("numa_memnode", "").split() ] at_times = int(params.get("attach_times", 1)) # Back up xml file. vmxml_backup = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) if not libvirt_version.version_compare(1, 2, 14): test.cancel("Memory hotplug not supported in current libvirt version.") if align_mem_values: # Rounding the following values to 'align' max_mem = utils_numeric.align_value(max_mem, align_to_value) max_mem_rt = utils_numeric.align_value(max_mem_rt, align_to_value) cur_mem = utils_numeric.align_value(cur_mem, align_to_value) tg_size = utils_numeric.align_value(tg_size, align_to_value) try: # Drop caches first for host has enough memory drop_caches() # Destroy domain first if vm.is_alive(): vm.destroy(gracefully=False) modify_domain_xml() # Start the domain any way if attach memory device old_mem_total = None if attach_device: vm.start() session = vm.wait_for_login() old_mem_total = get_vm_memtotal(session) logging.debug("Memtotal on guest: %s", old_mem_total) session.close() dev_xml = None # To attach the memory device. if add_mem_device: at_times = int(params.get("attach_times", 1)) dev_xml = create_mem_xml() for x in xrange(at_times): # If any error excepted, command error status should be # checked in the last time if x == at_times - 1: add_device(dev_xml, attach_error) else: add_device(dev_xml) # Check domain xml after attach device. if test_dom_xml: check_dom_xml(at_mem=attach_device) # Set domain state if pre_vm_state == "transient": logging.info("Creating %s...", vm_name) vmxml_for_test = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) if vm.is_alive(): vm.destroy(gracefully=False) vm.undefine() if virsh.create(vmxml_for_test.xml, **virsh_dargs).exit_status: vmxml_backup.define() test.fail("Cann't create the domain") elif vm.is_dead(): try: vm.start() vm.wait_for_login().close() except virt_vm.VMStartError as detail: if start_error: pass else: test.fail(detail) # Set memory operation if set_max_mem: max_mem_option = params.get("max_mem_option", "") ret = virsh.setmaxmem(vm_name, set_max_mem, flagstr=max_mem_option) libvirt.check_exit_status(ret, maxmem_error) # Check domain xml after start the domain. if test_dom_xml: check_dom_xml(at_mem=attach_device) # Check qemu command line if test_qemu_cmd: check_qemu_cmd() # Check guest meminfo after attachment if (attach_device and not attach_option.count("config") and not any([attach_error, start_error])): check_guest_meminfo(old_mem_total, check_option="attach") # Consuming memory on guest, # to verify memory changes by numastat if test_mem_binding: pid = vm.get_pid() old_numastat = read_from_numastat(pid, "Total") logging.debug("Numastat: %s", old_numastat) consume_vm_mem() new_numastat = read_from_numastat(pid, "Total") logging.debug("Numastat: %s", new_numastat) # Only check total memory which is the last element if float(new_numastat[-1]) - float(old_numastat[-1]) < 0: test.fail("Numa memory can't be consumed on guest") # Run managedsave command to check domain xml. if test_managedsave: ret = virsh.managedsave(vm_name, **virsh_dargs) libvirt.check_exit_status(ret) vm.start() vm.wait_for_login().close() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Run save and restore command to check domain xml if test_save_restore: check_save_restore() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Check domain xml after restarting libvirtd if restart_libvirtd: libvirtd = utils_libvirtd.Libvirtd() libvirtd.restart() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Detach the memory device unplug_failed_with_known_error = False if detach_device: if not dev_xml: dev_xml = create_mem_xml() for x in xrange(at_times): ret = virsh.detach_device(vm_name, dev_xml.xml, flagstr=attach_option) try: libvirt.check_exit_status(ret, detach_error) except Exception as detail: dmesg_file = tempfile.mktemp(dir=data_dir.get_tmp_dir()) try: session = vm.wait_for_login() utils_misc.verify_dmesg(dmesg_log_file=dmesg_file, ignore_result=True, session=session, level_check=5) except Exception: session.close() test.fail("After memory unplug Unable to connect to VM" " or unable to collect dmesg") session.close() if os.path.exists(dmesg_file): with open(dmesg_file, 'r') as f: flag = re.findall( r'memory memory\d+?: Offline failed', f.read()) if not flag: # The attached memory is used by vm, and it could not be unplugged # The result is expected os.remove(dmesg_file) test.fail(detail) unplug_failed_with_known_error = True os.remove(dmesg_file) # Check whether a known error occured or not dmesg_file = tempfile.mktemp(dir=data_dir.get_tmp_dir()) try: session = vm.wait_for_login() utils_misc.verify_dmesg(dmesg_log_file=dmesg_file, ignore_result=True, session=session, level_check=4) except Exception: session.close() test.fail("After memory unplug Unable to connect to VM" " or unable to collect dmesg") session.close() if known_unplug_errors and os.path.exists(dmesg_file): for known_error in known_unplug_errors: if (known_error[0] == known_error[-1]) and \ known_error.startswith(("'")): known_error = known_error[1:-1] with open(dmesg_file, 'r') as f: if known_error in f.read(): unplug_failed_with_known_error = True logging.debug( "Known error occured, while hot unplug" ": %s", known_error) if test_dom_xml and not unplug_failed_with_known_error: check_dom_xml(dt_mem=detach_device) # Remove dmesg temp file if os.path.exists(dmesg_file): os.remove(dmesg_file) finally: # Delete snapshots. snapshot_lists = virsh.snapshot_list(vm_name) if len(snapshot_lists) > 0: libvirt.clean_up_snapshots(vm_name, snapshot_lists) for snap in snapshot_lists: virsh.snapshot_delete(vm_name, snap, "--metadata") # Recover VM. if vm.is_alive(): vm.destroy(gracefully=False) logging.info("Restoring vm...") vmxml_backup.sync()
def dmesg(self): utils_misc.verify_dmesg(session=self.session) utils_misc.verify_dmesg()
def run(test, params, env): """ Test rbd disk device. 1.Prepare test environment,destroy or suspend a VM. 2.Prepare disk image. 3.Edit disks xml and start the domain. 4.Perform test operation. 5.Recover test environment. """ vm_name = params.get("main_vm") vm = env.get_vm(vm_name) virsh_dargs = {'debug': True, 'ignore_status': True} # Global variable to store max/current memory, # it may change after attach/detach new_max_mem = None new_cur_mem = None def consume_vm_mem(size=1000, timeout=360): """ To consume guest memory, default size is 1000M """ session = vm.wait_for_login() # Mount tmpfs on /mnt and write to a file on it, # it is the memory operation sh_cmd = ("swapoff -a; mount -t tmpfs -o size={0}M tmpfs " "/mnt; dd if=/dev/urandom of=/mnt/test bs=1M" " count={0}".format(size)) session.cmd(sh_cmd, timeout=timeout) session.close() def mount_hugepages(page_size): """ To mount hugepages :param page_size: unit is kB, it can be 4,2048,1048576,etc """ if page_size == 4: perm = "" else: perm = "pagesize=%dK" % page_size tlbfs_status = utils_misc.is_mounted("hugetlbfs", "/dev/hugepages", "hugetlbfs") if tlbfs_status: utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs") utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs", perm) def setup_hugepages(page_size=2048, shp_num=2000): """ To setup hugepages :param page_size: unit is kB, it can be 4,2048,1048576,etc :param shp_num: number of hugepage, string type """ mount_hugepages(page_size) utils_memory.set_num_huge_pages(shp_num) config.hugetlbfs_mount = ["/dev/hugepages"] utils_libvirtd.libvirtd_restart() def restore_hugepages(page_size=4): """ To recover hugepages :param page_size: unit is kB, it can be 4,2048,1048576,etc """ mount_hugepages(page_size) config.restore() utils_libvirtd.libvirtd_restart() def check_qemu_cmd(max_mem_rt, tg_size): """ Check qemu command line options. :param max_mem_rt: size of max memory :param tg_size: Target hotplug memory size :return: None """ cmd = ("ps -ef | grep %s | grep -v grep " % vm_name) if discard: if libvirt_version.version_compare(7, 3, 0): cmd = cmd + " | grep " + '\\"discard-data\\":true' else: cmd += " | grep 'discard-data=yes'" elif max_mem_rt: cmd += (" | grep 'slots=%s,maxmem=%sk'" % (max_mem_slots, max_mem_rt)) if tg_size: size = int(tg_size) * 1024 if huge_pages or discard or cold_plug_discard: cmd_str = 'memdimm.\|memory-backend-file,id=ram-node.' cmd += ( " | grep 'memory-backend-file,id=%s' | grep 'size=%s" % (cmd_str, size)) else: cmd_str = 'mem.\|memory-backend-ram,id=ram-node.' cmd += ( " | grep 'memory-backend-ram,id=%s' | grep 'size=%s" % (cmd_str, size)) if pg_size: cmd += ",host-nodes=%s" % node_mask if numa_memnode: for node in numa_memnode: if ('nodeset' in node and node['nodeset'] in node_mask): cmd += ",policy=%s" % node['mode'] cmd += ".*pc-dimm,node=%s" % tg_node if mem_addr: cmd += (".*slot=%s" % (mem_addr['slot'])) cmd += "'" if cold_plug_discard: cmd += " | grep 'discard-data=yes'" # Run the command result = process.run(cmd, shell=True, verbose=True, ignore_status=True) if result.exit_status: test.fail('Qemu command check fail.') def check_guest_meminfo(old_mem, check_option): """ Check meminfo on guest. """ assert old_mem is not None session = vm.wait_for_login() # Hot-plugged memory should be online by udev rules udev_file = "/lib/udev/rules.d/80-hotplug-cpu-mem.rules" udev_rules = ('SUBSYSTEM=="memory", ACTION=="add", TEST=="state",' ' ATTR{state}=="offline", ATTR{state}="online"') cmd = ("grep memory %s || echo '%s' >> %s" % (udev_file, udev_rules, udev_file)) session.cmd(cmd) # Wait a while for new memory to be detected. utils_misc.wait_for( lambda: vm.get_totalmem_sys(online) != int(old_mem), 30, first=20.0) new_mem = vm.get_totalmem_sys(online) session.close() logging.debug("Memtotal on guest: %s", new_mem) no_of_times = 1 if at_times: no_of_times = at_times if check_option == "attach": if new_mem != int(old_mem) + (int(tg_size) * no_of_times): test.fail("Total memory on guest couldn't changed after " "attach memory device") if check_option == "detach": if new_mem != int(old_mem) - (int(tg_size) * no_of_times): test.fail("Total memory on guest couldn't changed after " "detach memory device") def check_dom_xml(at_mem=False, dt_mem=False): """ Check domain xml options. """ # Global variable to store max/current memory global new_max_mem global new_cur_mem if attach_option.count("config"): dom_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) else: dom_xml = vm_xml.VMXML.new_from_dumpxml(vm_name) try: xml_max_mem_rt = int(dom_xml.max_mem_rt) xml_max_mem = int(dom_xml.max_mem) xml_cur_mem = int(dom_xml.current_mem) assert int(max_mem_rt) == xml_max_mem_rt # Check attached/detached memory logging.info("at_mem=%s,dt_mem=%s", at_mem, dt_mem) logging.info("detach_device is %s", detach_device) if at_mem: if at_times: assert int(max_mem) + (int(tg_size) * at_times) == xml_max_mem else: assert int(max_mem) + int(tg_size) == xml_max_mem # Bug 1220702, skip the check for current memory if at_times: assert int(cur_mem) + (int(tg_size) * at_times) == xml_cur_mem else: assert int(cur_mem) + int(tg_size) == xml_cur_mem new_max_mem = xml_max_mem new_cur_mem = xml_cur_mem mem_dev = dom_xml.get_devices("memory") memory_devices = 1 if at_times: memory_devices = at_times if len(mem_dev) != memory_devices: test.fail("Found wrong number of memory device") assert int(tg_size) == int(mem_dev[0].target.size) assert int(tg_node) == int(mem_dev[0].target.node) elif dt_mem: if at_times: assert int(new_max_mem) - (int(tg_size) * at_times) == xml_max_mem assert int(new_cur_mem) - (int(tg_size) * at_times) == xml_cur_mem else: assert int(new_max_mem) - int(tg_size) == xml_max_mem # Bug 1220702, skip the check for current memory assert int(new_cur_mem) - int(tg_size) == xml_cur_mem except AssertionError: utils_misc.log_last_traceback() test.fail("Found unmatched memory setting from domain xml") def check_mem_align(): """ Check if set memory align to 256 """ dom_xml = vm_xml.VMXML.new_from_dumpxml(vm_name) dom_mem = {} dom_mem['maxMemory'] = int(dom_xml.max_mem_rt) dom_mem['memory'] = int(dom_xml.memory) dom_mem['currentMemory'] = int(dom_xml.current_mem) cpuxml = dom_xml.cpu numa_cell = cpuxml.numa_cell dom_mem['numacellMemory'] = int(numa_cell[0]['memory']) sum_numa_mem = sum([int(cell['memory']) for cell in numa_cell]) attached_mem = dom_xml.get_devices(device_type='memory')[0] dom_mem['attached_mem'] = attached_mem.target.size all_align = True for key in dom_mem: logging.info('%-20s:%15d', key, dom_mem[key]) if dom_mem[key] % 262144: logging.error('%s not align to 256', key) if key == 'currentMemory': continue all_align = False if not all_align: test.fail('Memory not align to 256') if dom_mem['memory'] == sum_numa_mem + dom_mem['attached_mem']: logging.info( 'Check Pass: Memory is equal to (all numa memory + memory device)' ) else: test.fail( 'Memory is not equal to (all numa memory + memory device)') return dom_mem def check_save_restore(): """ Test save and restore operation """ save_file = os.path.join(data_dir.get_tmp_dir(), "%s.save" % vm_name) ret = virsh.save(vm_name, save_file, **virsh_dargs) libvirt.check_exit_status(ret) def _wait_for_restore(): try: virsh.restore(save_file, debug=True, ignore_status=False) return True except Exception as e: logging.error(e) utils_misc.wait_for(_wait_for_restore, 30, step=5) if os.path.exists(save_file): os.remove(save_file) # Login to check vm status vm.wait_for_login().close() def add_device(dev_xml, attach, at_error=False): """ Add memory device by attachment or modify domain xml. """ if attach: ret = virsh.attach_device(vm_name, dev_xml.xml, flagstr=attach_option, debug=True) libvirt.check_exit_status(ret, at_error) else: vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) if numa_cells: del vmxml.max_mem del vmxml.current_mem vmxml.add_device(dev_xml) vmxml.sync() def modify_domain_xml(): """ Modify domain xml and define it. """ vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) mem_unit = params.get("mem_unit", "KiB") vcpu = params.get("vcpu", "4") if max_mem_rt: vmxml.max_mem_rt = int(max_mem_rt) vmxml.max_mem_rt_slots = max_mem_slots vmxml.max_mem_rt_unit = mem_unit if max_mem: vmxml.max_mem = int(max_mem) if cur_mem: vmxml.current_mem = int(cur_mem) if memory_val: vmxml.memory = int(memory_val) if vcpu: vmxml.vcpu = int(vcpu) vcpu_placement = params.get("vcpu_placement", "static") vmxml.placement = vcpu_placement if numa_memnode: vmxml.numa_memory = {} vmxml.numa_memnode = numa_memnode else: try: del vmxml.numa_memory del vmxml.numa_memnode except Exception: # Not exists pass if numa_cells: cells = [ast.literal_eval(x) for x in numa_cells] # Rounding the numa memory values if align_mem_values: for cell in range(cells.__len__()): memory_value = str( utils_numeric.align_value(cells[cell]["memory"], align_to_value)) cells[cell]["memory"] = memory_value cpu_xml = vm_xml.VMCPUXML() cpu_xml.xml = "<cpu mode='host-model'><numa/></cpu>" cpu_mode = params.get("cpu_mode") model_fallback = params.get("model_fallback") if cpu_mode: cpu_xml.mode = cpu_mode if model_fallback: cpu_xml.fallback = model_fallback cpu_xml.numa_cell = cpu_xml.dicts_to_cells(cells) vmxml.cpu = cpu_xml # Delete memory and currentMemory tag, # libvirt will fill it automatically del vmxml.max_mem del vmxml.current_mem # hugepages setting if huge_pages or discard or cold_plug_discard: membacking = vm_xml.VMMemBackingXML() membacking.discard = True membacking.source = '' membacking.source_type = 'file' if huge_pages: hugepages = vm_xml.VMHugepagesXML() pagexml_list = [] for i in range(len(huge_pages)): pagexml = hugepages.PageXML() pagexml.update(huge_pages[i]) pagexml_list.append(pagexml) hugepages.pages = pagexml_list membacking.hugepages = hugepages vmxml.mb = membacking logging.debug("vm xml: %s", vmxml) vmxml.sync() pre_vm_state = params.get("pre_vm_state", "running") attach_device = "yes" == params.get("attach_device", "no") detach_device = "yes" == params.get("detach_device", "no") detach_alias = "yes" == params.get("detach_alias", "no") detach_alias_options = params.get("detach_alias_options") attach_error = "yes" == params.get("attach_error", "no") start_error = "yes" == params.get("start_error", "no") define_error = "yes" == params.get("define_error", "no") detach_error = "yes" == params.get("detach_error", "no") maxmem_error = "yes" == params.get("maxmem_error", "no") attach_option = params.get("attach_option", "") test_qemu_cmd = "yes" == params.get("test_qemu_cmd", "no") wait_before_save_secs = int(params.get("wait_before_save_secs", 0)) test_managedsave = "yes" == params.get("test_managedsave", "no") test_save_restore = "yes" == params.get("test_save_restore", "no") test_mem_binding = "yes" == params.get("test_mem_binding", "no") restart_libvirtd = "yes" == params.get("restart_libvirtd", "no") add_mem_device = "yes" == params.get("add_mem_device", "no") test_dom_xml = "yes" == params.get("test_dom_xml", "no") max_mem = params.get("max_mem") max_mem_rt = params.get("max_mem_rt") max_mem_slots = params.get("max_mem_slots", "16") memory_val = params.get('memory_val', '') mem_align = 'yes' == params.get('mem_align', 'no') hot_plug = 'yes' == params.get('hot_plug', 'no') cur_mem = params.get("current_mem") numa_cells = params.get("numa_cells", "").split() set_max_mem = params.get("set_max_mem") align_mem_values = "yes" == params.get("align_mem_values", "no") align_to_value = int(params.get("align_to_value", "65536")) hot_reboot = "yes" == params.get("hot_reboot", "no") rand_reboot = "yes" == params.get("rand_reboot", "no") guest_known_unplug_errors = [] guest_known_unplug_errors.append(params.get("guest_known_unplug_errors")) host_known_unplug_errors = [] host_known_unplug_errors.append(params.get("host_known_unplug_errors")) discard = "yes" == params.get("discard", "no") cold_plug_discard = "yes" == params.get("cold_plug_discard", "no") if cold_plug_discard or discard: mem_discard = 'yes' else: mem_discard = 'no' # params for attached device mem_model = params.get("mem_model", "dimm") tg_size = params.get("tg_size") tg_sizeunit = params.get("tg_sizeunit", 'KiB') tg_node = params.get("tg_node", 0) pg_size = params.get("page_size") pg_unit = params.get("page_unit", "KiB") huge_page_num = int(params.get('huge_page_num', 2000)) node_mask = params.get("node_mask", "0") mem_addr = ast.literal_eval(params.get("memory_addr", "{}")) huge_pages = [ ast.literal_eval(x) for x in params.get("huge_pages", "").split() ] numa_memnode = [ ast.literal_eval(x) for x in params.get("numa_memnode", "").split() ] at_times = int(params.get("attach_times", 1)) online = params.get("mem_online", "no") config = utils_config.LibvirtQemuConfig() setup_hugepages_flag = params.get("setup_hugepages") if (setup_hugepages_flag == "yes"): cpu_arch = cpu_util.get_family() if hasattr(cpu_util, 'get_family')\ else cpu_util.get_cpu_arch() if cpu_arch == 'power8': pg_size = '16384' huge_page_num = 200 elif cpu_arch == 'power9': pg_size = '2048' huge_page_num = 2000 [x.update({'size': pg_size}) for x in huge_pages] setup_hugepages(int(pg_size), shp_num=huge_page_num) # Back up xml file. vmxml_backup = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) if not libvirt_version.version_compare(1, 2, 14): test.cancel("Memory hotplug not supported in current libvirt version.") if 'align_256m' in params.get('name', ''): arch = platform.machine() if arch.lower() != 'ppc64le': test.cancel('This case is for ppc64le only.') if align_mem_values: # Rounding the following values to 'align' max_mem = utils_numeric.align_value(max_mem, align_to_value) max_mem_rt = utils_numeric.align_value(max_mem_rt, align_to_value) cur_mem = utils_numeric.align_value(cur_mem, align_to_value) tg_size = utils_numeric.align_value(tg_size, align_to_value) try: # Drop caches first for host has enough memory drop_caches() # Destroy domain first if vm.is_alive(): vm.destroy(gracefully=False) modify_domain_xml() numa_info = utils_misc.NumaInfo() logging.debug(numa_info.get_all_node_meminfo()) # Start the domain any way if attach memory device old_mem_total = None if attach_device: vm.start() session = vm.wait_for_login() old_mem_total = vm.get_totalmem_sys(online) logging.debug("Memtotal on guest: %s", old_mem_total) session.close() elif discard: vm.start() session = vm.wait_for_login() check_qemu_cmd(max_mem_rt, tg_size) dev_xml = None # To attach the memory device. if (add_mem_device and not hot_plug) or cold_plug_discard: at_times = int(params.get("attach_times", 1)) randvar = 0 if rand_reboot: rand_value = random.randint(15, 25) logging.debug("reboots at %s", rand_value) for x in xrange(at_times): # If any error excepted, command error status should be # checked in the last time device_alias = "ua-" + str(uuid.uuid4()) dev_xml = utils_hotplug.create_mem_xml( tg_size, pg_size, mem_addr, tg_sizeunit, pg_unit, tg_node, node_mask, mem_model, mem_discard, device_alias) randvar = randvar + 1 logging.debug("attaching device count = %s", x) if x == at_times - 1: add_device(dev_xml, attach_device, attach_error) else: add_device(dev_xml, attach_device) if hot_reboot: vm.reboot() vm.wait_for_login() if rand_reboot and randvar == rand_value: vm.reboot() vm.wait_for_login() randvar = 0 rand_value = random.randint(15, 25) logging.debug("reboots at %s", rand_value) # Check domain xml after attach device. if test_dom_xml: check_dom_xml(at_mem=attach_device) # Set domain state if pre_vm_state == "transient": logging.info("Creating %s...", vm_name) vmxml_for_test = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) if vm.is_alive(): vm.destroy(gracefully=False) vm.undefine() if virsh.create(vmxml_for_test.xml, **virsh_dargs).exit_status: vmxml_backup.define() test.fail("Can't create the domain") elif vm.is_dead(): try: vm.start() vm.wait_for_login().close() except virt_vm.VMStartError as detail: if start_error: pass else: except_msg = "memory hotplug isn't supported by this QEMU binary" if except_msg in detail.reason: test.cancel(detail) test.fail(detail) # Set memory operation if set_max_mem: max_mem_option = params.get("max_mem_option", "") ret = virsh.setmaxmem(vm_name, set_max_mem, flagstr=max_mem_option) libvirt.check_exit_status(ret, maxmem_error) # Hotplug memory device if add_mem_device and hot_plug: process.run('ps -ef|grep qemu', shell=True, verbose=True) session = vm.wait_for_login() original_mem = vm.get_totalmem_sys() dev_xml = utils_hotplug.create_mem_xml(tg_size, pg_size, mem_addr, tg_sizeunit, pg_unit, tg_node, node_mask, mem_model) add_device(dev_xml, True) mem_after = vm.get_totalmem_sys() params['delta'] = mem_after - original_mem # Check domain xml after start the domain. if test_dom_xml: check_dom_xml(at_mem=attach_device) if mem_align: dom_mem = check_mem_align() check_qemu_cmd(dom_mem['maxMemory'], dom_mem['attached_mem']) if hot_plug and params['delta'] != dom_mem['attached_mem']: test.fail( 'Memory after attach not equal to original mem + attached mem' ) # Check qemu command line if test_qemu_cmd: check_qemu_cmd(max_mem_rt, tg_size) # Check guest meminfo after attachment if (attach_device and not attach_option.count("config") and not any([attach_error, start_error])): check_guest_meminfo(old_mem_total, check_option="attach") # Consuming memory on guest, # to verify memory changes by numastat if test_mem_binding: pid = vm.get_pid() old_numastat = read_from_numastat(pid, "Total") logging.debug("Numastat: %s", old_numastat) # Increase the memory consumed to 1500 consume_vm_mem(1500) new_numastat = read_from_numastat(pid, "Total") logging.debug("Numastat: %s", new_numastat) # Only check total memory which is the last element if float(new_numastat[-1]) - float(old_numastat[-1]) < 0: test.fail("Numa memory can't be consumed on guest") # Run managedsave command to check domain xml. if test_managedsave: # Wait 10s for vm to be ready before managedsave time.sleep(wait_before_save_secs) ret = virsh.managedsave(vm_name, **virsh_dargs) libvirt.check_exit_status(ret) def _wait_for_vm_start(): try: vm.start() return True except Exception as e: logging.error(e) utils_misc.wait_for(_wait_for_vm_start, timeout=30, step=5) vm.wait_for_login().close() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Run save and restore command to check domain xml if test_save_restore: # Wait 10s for vm to be ready before save time.sleep(wait_before_save_secs) check_save_restore() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Check domain xml after restarting libvirtd if restart_libvirtd: libvirtd = utils_libvirtd.Libvirtd() libvirtd.restart() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Detach the memory device unplug_failed_with_known_error = False if detach_device: dev_xml = utils_hotplug.create_mem_xml(tg_size, pg_size, mem_addr, tg_sizeunit, pg_unit, tg_node, node_mask, mem_model, mem_discard) for x in xrange(at_times): if not detach_alias: ret = virsh.detach_device(vm_name, dev_xml.xml, flagstr=attach_option, debug=True) else: ret = virsh.detach_device_alias(vm_name, device_alias, detach_alias_options, debug=True) if ret.stderr and host_known_unplug_errors: for known_error in host_known_unplug_errors: if (known_error[0] == known_error[-1]) and \ known_error.startswith(("'")): known_error = known_error[1:-1] if known_error in ret.stderr: unplug_failed_with_known_error = True logging.debug( "Known error occurred in Host, while" " hot unplug: %s", known_error) if unplug_failed_with_known_error: break try: libvirt.check_exit_status(ret, detach_error) except Exception as detail: dmesg_file = tempfile.mktemp(dir=data_dir.get_tmp_dir()) try: session = vm.wait_for_login() utils_misc.verify_dmesg(dmesg_log_file=dmesg_file, ignore_result=True, session=session, level_check=5) except Exception: session.close() test.fail("After memory unplug Unable to connect to VM" " or unable to collect dmesg") session.close() if os.path.exists(dmesg_file): with open(dmesg_file, 'r') as f: flag = re.findall( r'memory memory\d+?: Offline failed', f.read()) if not flag: # The attached memory is used by vm, and it could # not be unplugged.The result is expected os.remove(dmesg_file) test.fail(detail) unplug_failed_with_known_error = True os.remove(dmesg_file) # Check whether a known error occurred or not dmesg_file = tempfile.mktemp(dir=data_dir.get_tmp_dir()) try: session = vm.wait_for_login() utils_misc.verify_dmesg(dmesg_log_file=dmesg_file, ignore_result=True, session=session, level_check=4) except Exception: session.close() test.fail("After memory unplug Unable to connect to VM" " or unable to collect dmesg") session.close() if guest_known_unplug_errors and os.path.exists(dmesg_file): for known_error in guest_known_unplug_errors: if (known_error[0] == known_error[-1]) and \ known_error.startswith(("'")): known_error = known_error[1:-1] with open(dmesg_file, 'r') as f: if known_error in f.read(): unplug_failed_with_known_error = True logging.debug( "Known error occurred, while hot" " unplug: %s", known_error) if test_dom_xml and not unplug_failed_with_known_error: check_dom_xml(dt_mem=detach_device) # Remove dmesg temp file if os.path.exists(dmesg_file): os.remove(dmesg_file) except xcepts.LibvirtXMLError: if define_error: pass finally: # Delete snapshots. snapshot_lists = virsh.snapshot_list(vm_name) if len(snapshot_lists) > 0: libvirt.clean_up_snapshots(vm_name, snapshot_lists) for snap in snapshot_lists: virsh.snapshot_delete(vm_name, snap, "--metadata") # Recover VM. if vm.is_alive(): vm.destroy(gracefully=False) logging.info("Restoring vm...") if (setup_hugepages_flag == "yes"): restore_hugepages() vmxml_backup.sync()
def run(test, params, env): """ Qemu hugepage memory stress test. Steps: 1) System setup hugepages on host. 2) Mount this hugepage to /mnt/kvm_hugepage. 3) HugePages didn't leak when using non-existent mem-path. 4) Run memory heavy stress inside guest. 5) Check guest call trace in dmesg log. :params test: QEMU test object. :params params: Dictionary with the test parameters. :params env: Dictionary with test environment. """ def heavyload_install(): if session.cmd_status(test_installed_cmd) != 0: logging.warning("Could not find installed heavyload in guest, will" " install it via winutils.iso ") winutil_drive = utils_misc.get_winutils_vol(session) if not winutil_drive: test.cancel("WIN_UTILS CDROM not found.") install_cmd = params["install_cmd"] % winutil_drive session.cmd(install_cmd) if params.get_boolean("non_existent_point"): dir = tempfile.mkdtemp(prefix='hugepage_') error_context.context("This path %s, doesn't mount hugepage." % dir, logging.info) params["extra_params"] = " -mem-path %s" % dir params["start_vm"] = "yes" env_process.preprocess_vm(test, params, env, params["main_vm"]) os_type = params["os_type"] stress_duration = params.get_numeric("stress_duration", 60) vm = env.get_vm(params["main_vm"]) vm.verify_alive() session = vm.wait_for_login() try: error_context.context("Run memory heavy stress in guest", logging.info) if os_type == "linux": stress_args = params["stress_custom_args"] % ( params.get_numeric("mem") / 512) stress_test = utils_test.VMStress(vm, "stress", params, stress_args=stress_args) try: stress_test.load_stress_tool() utils_misc.wait_for(lambda: (stress_test.app_running is False), 30) stress_test.unload_stress() utils_misc.verify_dmesg(session=session) finally: stress_test.clean() else: install_path = params["install_path"] test_installed_cmd = 'dir "%s" | findstr /I heavyload' % install_path heavyload_install() error_context.context("Run heavyload inside guest.", logging.info) heavyload_bin = r'"%s\heavyload.exe" ' % install_path heavyload_options = ["/MEMORY %d" % (params.get_numeric("mem") / 512), "/DURATION %d" % (stress_duration // 60), "/AUTOEXIT", "/START"] start_cmd = heavyload_bin + " ".join(heavyload_options) stress_tool = BackgroundTest(session.cmd, (start_cmd, stress_duration, stress_duration)) stress_tool.start() if not utils_misc.wait_for(stress_tool.is_alive, stress_duration): test.error("Failed to start heavyload process.") stress_tool.join(stress_duration) if params.get_boolean("non_existent_point"): error_context.context("Check large memory pages free on host.", logging.info) if utils_memory.get_num_huge_pages() != utils_memory.get_num_huge_pages_free(): test.fail("HugePages leaked.") finally: session.close()
def run(test, params, env): """ Run Pktgen test between host/guest 1) Boot the main vm, or just grab it if it's already booted. 2) Configure pktgen server(only linux) 3) Run pktgen test, finish when timeout or env["pktgen_run"] != True :param test: QEMU test object. :param params: Dictionary with the test parameters. :param env: Dictionary with test environment. """ login_timeout = float(params.get("login_timeout", 360)) error_context.context("Init the VM, and try to login", logging.info) external_host = params.get("external_host") if not external_host: get_host_cmd = "ip route | awk '/default/ {print $3}'" external_host = process.system_output(get_host_cmd, shell=True).decode() vm = env.get_vm(params["main_vm"]) vm.verify_alive() session = vm.wait_for_login(timeout=login_timeout) error_context.context("Pktgen server environment prepare", logging.info) # pktgen server only support linux, since pktgen is a linux kernel module pktgen_server = params.get("pktgen_server", "localhost") params_server = params.object_params("pktgen_server") s_shell_client = params_server.get("shell_client", "ssh") s_shell_port = params_server.get("shell_port", "22") s_username = params_server.get("username", "root") s_passwd = params_server.get("password", "123456") s_shell_prompt = params_server.get("shell_prompt") server_session = "" # pktgen server is autotest virtual guest(only linux) if pktgen_server in params.get("vms", "vm1 vm2"): vm_pktgen = env.get_vm(pktgen_server) vm_pktgen.verify_alive() server_session = vm_pktgen.wait_for_login(timeout=login_timeout) runner = server_session.cmd pktgen_ip = vm_pktgen.get_address() pktgen_mac = vm_pktgen.get_mac_address() server_interface = utils_net.get_linux_ifname(server_session, pktgen_mac) # pktgen server is a external host assigned elif re.match(r"((\d){1,3}\.){3}(\d){1,3}", pktgen_server): pktgen_ip = pktgen_server server_session = remote.wait_for_login(s_shell_client, pktgen_ip, s_shell_port, s_username, s_passwd, s_shell_prompt) runner = server_session.cmd server_interface = params.get("server_interface") if not server_interface: test.cancel("Must config server interface before test") else: # using host as a pktgen server server_interface = params.get("netdst", "switch") host_nic = utils_net.Interface(server_interface) pktgen_ip = host_nic.get_ip() pktgen_mac = host_nic.get_mac() runner = process.system # copy pktgen_test scipt to the test server. local_path = os.path.join(data_dir.get_root_dir(), "shared/scripts/pktgen.sh") remote_path = "/tmp/pktgen.sh" remote.scp_to_remote(pktgen_ip, s_shell_port, s_username, s_passwd, local_path, remote_path) error_context.context("Run pktgen test", logging.info) run_threads = params.get("pktgen_threads", 1) pktgen_stress_timeout = float(params.get("pktgen_test_timeout", 600)) exec_cmd = "%s %s %s %s %s" % (remote_path, vm.get_address(), vm.get_mac_address(), server_interface, run_threads) try: env["pktgen_run"] = True try: # Set a run flag in env, when other case call this case as a sub # backgroud process, can set run flag to False to stop this case. start_time = time.time() stop_time = start_time + pktgen_stress_timeout while (env["pktgen_run"] and time.time() < stop_time): runner(exec_cmd, timeout=pktgen_stress_timeout) # using ping to kill the pktgen stress except aexpect.ShellTimeoutError: session.cmd("ping %s" % pktgen_ip, ignore_all_errors=True) finally: env["pktgen_run"] = False error_context.context("Verify Host and guest kernel no error " "and call trace", logging.info) vm.verify_kernel_crash() utils_misc.verify_dmesg() error_context.context("Ping external host after pktgen test", logging.info) session_ping = vm.wait_for_login(timeout=login_timeout) status, output = utils_test.ping(dest=external_host, session=session_ping, timeout=240, count=20) loss_ratio = utils_test.get_loss_ratio(output) if (loss_ratio > int(params.get("packet_lost_ratio", 5)) or loss_ratio == -1): logging.debug("Ping %s output: %s", external_host, output) test.fail("Guest network connction unusable, " "packet lost ratio is '%d%%'" % loss_ratio) if server_session: server_session.close() if session: session.close() if session_ping: session_ping.close()
def dmesg(self): session = self.vm.wait_for_login(timeout=self.timeout) utils_misc.verify_dmesg(session=session)#check guest dmesg utils_misc.verify_dmesg()#check host dmesg
def run(test, params, env): """ Run Pktgen test between host/guest 1) Boot the main vm, or just grab it if it's already booted. 2) Configure pktgen on guest or host 3) Run pktgen test, finish when timeout :param test: QEMU test object. :param params: Dictionary with the test parameters. :param env: Dictionary with test environment. """ def _pin_vm_threads(node): """ pin guest vcpu and vhost threads to cpus of a numa node repectively :param node: which numa node to pin """ if node: if not isinstance(node, utils_misc.NumaNode): node = utils_misc.NumaNode(int(node)) utils_test.qemu.pin_vm_threads(vm, node) timeout = float(params.get("pktgen_test_timeout", "240")) run_threads = params.get("pktgen_threads", 1) record_list = params.get("record_list") error_context.context("Init the VM, and try to login", logging.info) vm = env.get_vm(params["main_vm"]) vm.verify_alive() session_serial = vm.wait_for_serial_login(restart_network=True) session = vm.wait_for_login() # print numa information on host and pinning vhost and vcpus to cpus process.system_output("numactl --hardware") process.system_output("numactl --show") _pin_vm_threads(params.get("numa_node")) # get parameter from dictionary category = params.get("category") pkt_size = params.get("pkt_size") kvm_ver_chk_cmd = params.get("kvm_ver_chk_cmd") guest_ver_cmd = params["guest_ver_cmd"] # get qemu, guest kernel and kvm version info and write them into result result_path = utils_misc.get_path(test.resultsdir, "pktgen_perf.RHS") result_file = open(result_path, "w") kvm_ver = process.system_output(kvm_ver_chk_cmd, shell=True).decode() host_ver = os.uname()[2] guest_ver = session.cmd_output(guest_ver_cmd, timeout) result_file.write("### kvm-userspace-ver : %s\n" % kvm_ver) result_file.write("### kvm_version : %s\n" % host_ver) result_file.write("### guest-kernel-ver :%s\n" % guest_ver) # get record_list record_line = "" for record in record_list.split(): record_line += "%s|" % format_result(record) def install_package(ver, session=None): """ check module pktgen, install kernel-modules-internal package """ output_cmd = _system_output kernel_ver = "kernel-modules-internal-%s" % ver cmd_download = "cd /tmp && brew download-build %s --rpm" % kernel_ver cmd_install = "cd /tmp && rpm -ivh %s.rpm --force --nodeps" % kernel_ver output_cmd(cmd_download).decode() cmd_clean = "rm -rf /tmp/%s.rpm" % kernel_ver if session: output_cmd = session.cmd_output local_path = "/tmp/%s.rpm" % kernel_ver remote_path = "/tmp/" vm.copy_files_to(local_path, remote_path) output_cmd(cmd_install) output_cmd(cmd_clean) check_cmd = "uname -r |grep el8" if process.run(check_cmd, shell=True): install_package(host_ver) if session.cmd(check_cmd): install_package(guest_ver.strip(), session=session) # get result tested by each scenario for pkt_cate in category.split(): result_file.write("Category:%s\n" % pkt_cate) result_file.write("%s\n" % record_line.rstrip("|")) # copy pktgen_test script to test server local_path = os.path.join(data_dir.get_shared_dir(), "scripts/pktgen_perf.sh") remote_path = "/tmp/pktgen_perf.sh" if pkt_cate == "tx": vm.copy_files_to(local_path, remote_path) elif pkt_cate == "rx": process.run("cp %s %s" % (local_path, remote_path)) for size in pkt_size.split(): if pkt_cate == "tx": error_context.context("test guest tx pps performance", logging.info) guest_mac = vm.get_mac_address(0) pktgen_interface = utils_net.get_linux_ifname( session, guest_mac) dsc_dev = utils_net.Interface(vm.get_ifname(0)) dsc = dsc_dev.get_mac() runner = session.cmd pktgen_ip = vm.wait_for_get_address(0, timeout=5) pkt_cate_r = run_test(session_serial, runner, remote_path, pktgen_ip, dsc, pktgen_interface, run_threads, size, timeout) elif pkt_cate == "rx": error_context.context("test guest rx pps performance", logging.info) host_bridge = params.get("netdst", "switch") host_nic = utils_net.Interface(host_bridge) pktgen_ip = host_nic.get_ip() dsc = vm.wait_for_get_address(0, timeout=5) pktgen_interface = vm.get_ifname(0) runner = _system_output pkt_cate_r = run_test(session_serial, runner, remote_path, pktgen_ip, dsc, pktgen_interface, run_threads, size, timeout) line = "%s|" % format_result(size) line += "%s" % format_result(pkt_cate_r) result_file.write(("%s\n" % line)) error_context.context( "Verify Host and guest kernel no error\ and call trace", logging.info) vm.verify_kernel_crash() utils_misc.verify_dmesg() result_file.close() session_serial.close() session.close()
def run(test, params, env): """ Run Pktgen test between host/guest 1) Boot the main vm, or just grab it if it's already booted. 2) Configure pktgen server(only linux) 3) Run pktgen test, finish when timeout or env["pktgen_run"] != True :param test: QEMU test object. :param params: Dictionary with the test parameters. :param env: Dictionary with test environment. """ login_timeout = float(params.get("login_timeout", 360)) error.context("Init the VM, and try to login", logging.info) external_host = params.get("external_host") if not external_host: get_host_cmd = "ip route | awk '/default/ {print $3}'" external_host = utils.system_output(get_host_cmd) vm = env.get_vm(params["main_vm"]) vm.verify_alive() session = vm.wait_for_login(timeout=login_timeout) error.context("Pktgen server environment prepare", logging.info) # pktgen server only support linux, since pktgen is a linux kernel module pktgen_server = params.get("pktgen_server", "localhost") params_server = params.object_params("pktgen_server") s_shell_client = params_server.get("shell_client", "ssh") s_shell_port = params_server.get("shell_port", "22") s_username = params_server.get("username", "root") s_passwd = params_server.get("password", "123456") s_shell_prompt = params_server.get("shell_prompt") server_session = "" # pktgen server is autotest virtual guest(only linux) if pktgen_server in params.get("vms", "vm1 vm2"): vm_pktgen = env.get_vm(pktgen_server) vm_pktgen.verify_alive() server_session = vm_pktgen.wait_for_login(timeout=login_timeout) runner = server_session.cmd_output_safe pktgen_ip = vm_pktgen.get_address() pktgen_mac = vm_pktgen.get_mac_address() server_interface = utils_net.get_linux_ifname(server_session, pktgen_mac) # pktgen server is a external host assigned elif re.match(r"((\d){1,3}\.){3}(\d){1,3}", pktgen_server): pktgen_ip = pktgen_server server_session = remote.wait_for_login(s_shell_client, pktgen_ip, s_shell_port, s_username, s_passwd, s_shell_prompt) runner = server_session.cmd_output_safe server_interface = params.get("server_interface") if not server_interface: raise error.TestNAError("Must config server interface before test") else: # using host as a pktgen server server_interface = params.get("netdst", "switch") host_nic = utils_net.Interface(server_interface) pktgen_ip = host_nic.get_ip() pktgen_mac = host_nic.get_mac() runner = utils.system # copy pktgen_test scipt to the test server. local_path = os.path.join(data_dir.get_root_dir(), "shared/scripts/pktgen.sh") remote_path = "/tmp/pktgen.sh" remote.scp_to_remote(pktgen_ip, s_shell_port, s_username, s_passwd, local_path, remote_path) error.context("Run pktgen test", logging.info) run_threads = params.get("pktgen_threads", 1) pktgen_stress_timeout = float(params.get("pktgen_test_timeout", 600)) exec_cmd = "%s %s %s %s %s" % (remote_path, vm.get_address(), vm.get_mac_address(), server_interface, run_threads) try: env["pktgen_run"] = True try: # Set a run flag in env, when other case call this case as a sub # backgroud process, can set run flag to False to stop this case. start_time = time.time() stop_time = start_time + pktgen_stress_timeout while (env["pktgen_run"] and time.time() < stop_time): runner(exec_cmd, timeout=pktgen_stress_timeout) # using ping to kill the pktgen stress except aexpect.ShellTimeoutError: session.cmd("ping %s" % pktgen_ip, ignore_all_errors=True) finally: env["pktgen_run"] = False error.context("Verify Host and guest kernel no error and call trace", logging.info) vm.verify_kernel_crash() utils_misc.verify_dmesg() error.context("Ping external host after pktgen test", logging.info) session_ping = vm.wait_for_login(timeout=login_timeout) status, output = utils_test.ping(dest=external_host, session=session_ping, timeout=240, count=20) loss_ratio = utils_test.get_loss_ratio(output) if (loss_ratio > int(params.get("packet_lost_ratio", 5)) or loss_ratio == -1): logging.debug("Ping %s output: %s" % (external_host, output)) raise error.TestFail("Guest network connction unusable," + "packet lost ratio is '%d%%'" % loss_ratio) if server_session: server_session.close() if session: session.close() if session_ping: session_ping.close()
def run(test, params, env): """ Run Pktgen test between host/guest 1) Boot the main vm, or just grab it if it's already booted. 2) Configure pktgen on guest or host 3) Run pktgen test, finish when timeout :param test: QEMU test object. :param params: Dictionary with the test parameters. :param env: Dictionary with test environment. """ def _pin_vm_threads(node): """ pin guest vcpu and vhost threads to cpus of a numa node repectively :param node: which numa node to pin """ if node: if not isinstance(node, utils_misc.NumaNode): node = utils_misc.NumaNode(int(node)) utils_test.qemu.pin_vm_threads(vm, node) timeout = float(params.get("pktgen_test_timeout", "240")) run_threads = params.get("pktgen_threads", 1) record_list = params.get("record_list") error_context.context("Init the VM, and try to login", logging.info) vm = env.get_vm(params["main_vm"]) vm.verify_alive() session = vm.wait_for_login() session_serial = vm.wait_for_serial_login() # print numa information on host and pinning vhost and vcpus to cpus process.system_output("numactl --hardware") process.system_output("numactl --show") _pin_vm_threads(params.get("numa_node")) # get parameter from dictionary category = params.get("category") pkt_size = params.get("pkt_size") kvm_ver_chk_cmd = params.get("kvm_ver_chk_cmd") guest_ver_cmd = params["guest_ver_cmd"] # get qemu, guest kernel and kvm version info and write them into result result_path = utils_misc.get_path(test.resultsdir, "pktgen_perf.RHS") result_file = open(result_path, "w") kvm_ver = process.system_output(kvm_ver_chk_cmd, shell=True) host_ver = os.uname()[2] guest_ver = session.cmd_output(guest_ver_cmd, timeout) result_file.write("### kvm-userspace-ver : %s\n" % kvm_ver) result_file.write("### kvm_version : %s\n" % host_ver) result_file.write("### guest-kernel-ver :%s\n" % guest_ver) # get record_list record_line = "" for record in record_list.split(): record_line += "%s|" % format_result(record) # get result tested by each scenario for pkt_cate in category.split(): result_file.write("Category:%s\n" % pkt_cate) result_file.write("%s\n" % record_line.rstrip("|")) # copy pktgen_test script to test server local_path = os.path.join(data_dir.get_shared_dir(), "scripts/pktgen_perf.sh") remote_path = "/tmp/pktgen_perf.sh" if pkt_cate == "tx": vm.copy_files_to(local_path, remote_path) elif pkt_cate == "rx": process.run("cp %s %s" % (local_path, remote_path)) for size in pkt_size.split(): if pkt_cate == "tx": error_context.context("test guest tx pps performance", logging.info) guest_mac = vm.get_mac_address(0) pktgen_interface = utils_net.get_linux_ifname(session, guest_mac) dsc_dev = utils_net.Interface(vm.get_ifname(0)) dsc = dsc_dev.get_mac() runner = session.cmd pktgen_ip = vm.wait_for_get_address(0, timeout=5) pkt_cate_r = run_test(session_serial, runner, remote_path, pktgen_ip, dsc, pktgen_interface, run_threads, size, timeout) elif pkt_cate == "rx": error_context.context("test guest rx pps performance", logging.info) host_bridge = params.get("netdst", "switch") host_nic = utils_net.Interface(host_bridge) pktgen_ip = host_nic.get_ip() dsc = vm.wait_for_get_address(0, timeout=5) pktgen_interface = vm.get_ifname(0) runner = _system_output pkt_cate_r = run_test(session_serial, runner, remote_path, pktgen_ip, dsc, pktgen_interface, run_threads, size, timeout) line = "%s|" % format_result(size) line += "%s" % format_result(pkt_cate_r) result_file.write(("%s\n" % line)) error_context.context("Verify Host and guest kernel no error\ and call trace", logging.info) vm.verify_kernel_crash() utils_misc.verify_dmesg() result_file.close() session_serial.close() session.close()
def run(test, params, env): """ Test rbd disk device. 1.Prepare test environment,destroy or suspend a VM. 2.Prepare disk image. 3.Edit disks xml and start the domain. 4.Perform test operation. 5.Recover test environment. """ vm_name = params.get("main_vm") vm = env.get_vm(vm_name) virsh_dargs = {'debug': True, 'ignore_status': True} # Global variable to store max/current memory, # it may change after attach/detach new_max_mem = None new_cur_mem = None def consume_vm_mem(size=1000, timeout=360): """ To consume guest memory, default size is 1000M """ session = vm.wait_for_login() # Mount tmpfs on /mnt and write to a file on it, # it is the memory operation sh_cmd = ("swapoff -a; mount -t tmpfs -o size={0}M tmpfs " "/mnt; dd if=/dev/urandom of=/mnt/test bs=1M" " count={0}".format(size)) session.cmd(sh_cmd, timeout=timeout) session.close() def mount_hugepages(page_size): """ To mount hugepages :param page_size: unit is kB, it can be 4,2048,1048576,etc """ if page_size == 4: perm = "" else: perm = "pagesize=%dK" % page_size tlbfs_status = utils_misc.is_mounted("hugetlbfs", "/dev/hugepages", "hugetlbfs") if tlbfs_status: utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs") utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs", perm) def setup_hugepages(page_size=2048, shp_num=2000): """ To setup hugepages :param page_size: unit is kB, it can be 4,2048,1048576,etc :param shp_num: number of hugepage, string type """ mount_hugepages(page_size) utils_memory.set_num_huge_pages(shp_num) config.hugetlbfs_mount = ["/dev/hugepages"] utils_libvirtd.libvirtd_restart() def restore_hugepages(page_size=4): """ To recover hugepages :param page_size: unit is kB, it can be 4,2048,1048576,etc """ mount_hugepages(page_size) config.restore() utils_libvirtd.libvirtd_restart() def check_qemu_cmd(max_mem_rt, tg_size): """ Check qemu command line options. :param max_mem_rt: size of max memory :param tg_size: Target hotplug memory size :return: None """ cmd = ("ps -ef | grep %s | grep -v grep " % vm_name) if max_mem_rt: cmd += (" | grep 'slots=%s,maxmem=%sk'" % (max_mem_slots, max_mem_rt)) if tg_size: size = int(tg_size) * 1024 cmd_str = 'memdimm.\|memory-backend-ram,id=ram-node.' cmd += (" | grep 'memory-backend-ram,id=%s' | grep 'size=%s" % (cmd_str, size)) if pg_size: cmd += ",host-nodes=%s" % node_mask if numa_memnode: for node in numa_memnode: if ('nodeset' in node and node['nodeset'] in node_mask): cmd += ",policy=%s" % node['mode'] cmd += ".*pc-dimm,node=%s" % tg_node if mem_addr: cmd += (".*slot=%s,addr=%s" % (mem_addr['slot'], int(mem_addr['base'], 16))) cmd += "'" # Run the command result = process.run(cmd, shell=True, verbose=True, ignore_status=True) if result.exit_status: test.fail('Qemu command check fail.') def check_guest_meminfo(old_mem, check_option): """ Check meminfo on guest. """ assert old_mem is not None session = vm.wait_for_login() # Hot-plugged memory should be online by udev rules udev_file = "/lib/udev/rules.d/80-hotplug-cpu-mem.rules" udev_rules = ('SUBSYSTEM=="memory", ACTION=="add", TEST=="state",' ' ATTR{state}=="offline", ATTR{state}="online"') cmd = ("grep memory %s || echo '%s' >> %s" % (udev_file, udev_rules, udev_file)) session.cmd(cmd) # Wait a while for new memory to be detected. utils_misc.wait_for( lambda: vm.get_totalmem_sys(online) != int(old_mem), 30, first=20.0) new_mem = vm.get_totalmem_sys(online) session.close() logging.debug("Memtotal on guest: %s", new_mem) no_of_times = 1 if at_times: no_of_times = at_times if check_option == "attach": if new_mem != int(old_mem) + (int(tg_size) * no_of_times): test.fail("Total memory on guest couldn't changed after " "attach memory device") if check_option == "detach": if new_mem != int(old_mem) - (int(tg_size) * no_of_times): test.fail("Total memory on guest couldn't changed after " "detach memory device") def check_dom_xml(at_mem=False, dt_mem=False): """ Check domain xml options. """ # Global variable to store max/current memory global new_max_mem global new_cur_mem if attach_option.count("config"): dom_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) else: dom_xml = vm_xml.VMXML.new_from_dumpxml(vm_name) try: xml_max_mem_rt = int(dom_xml.max_mem_rt) xml_max_mem = int(dom_xml.max_mem) xml_cur_mem = int(dom_xml.current_mem) assert int(max_mem_rt) == xml_max_mem_rt # Check attached/detached memory if at_mem: if at_times: assert int(max_mem) + (int(tg_size) * at_times) == xml_max_mem else: assert int(max_mem) + int(tg_size) == xml_max_mem # Bug 1220702, skip the check for current memory if at_times: assert int(cur_mem) + (int(tg_size) * at_times) == xml_cur_mem else: assert int(cur_mem) + int(tg_size) == xml_cur_mem new_max_mem = xml_max_mem new_cur_mem = xml_cur_mem mem_dev = dom_xml.get_devices("memory") memory_devices = 1 if at_times: memory_devices = at_times if len(mem_dev) != memory_devices: test.fail("Found wrong number of memory device") assert int(tg_size) == int(mem_dev[0].target.size) assert int(tg_node) == int(mem_dev[0].target.node) elif dt_mem: if at_times: assert int(new_max_mem) - (int(tg_size) * at_times) == xml_max_mem assert int(new_cur_mem) - (int(tg_size) * at_times) == xml_cur_mem else: assert int(new_max_mem) - int(tg_size) == xml_max_mem # Bug 1220702, skip the check for current memory assert int(new_cur_mem) - int(tg_size) == xml_cur_mem except AssertionError: utils_misc.log_last_traceback() test.fail("Found unmatched memory setting from domain xml") def check_mem_align(): """ Check if set memory align to 256 """ dom_xml = vm_xml.VMXML.new_from_dumpxml(vm_name) dom_mem = {} dom_mem['maxMemory'] = int(dom_xml.max_mem_rt) dom_mem['memory'] = int(dom_xml.memory) dom_mem['currentMemory'] = int(dom_xml.current_mem) cpuxml = dom_xml.cpu numa_cell = cpuxml.numa_cell dom_mem['numacellMemory'] = int(numa_cell[0]['memory']) sum_numa_mem = sum([int(cell['memory']) for cell in numa_cell]) attached_mem = dom_xml.get_devices(device_type='memory')[0] dom_mem['attached_mem'] = attached_mem.target.size all_align = True for key in dom_mem: logging.info('%-20s:%15d', key, dom_mem[key]) if dom_mem[key] % 256: logging.error('%s not align to 256', key) all_align = False if not all_align: test.fail('Memory not align to 256') if dom_mem['memory'] == sum_numa_mem + dom_mem['attached_mem']: logging.info('Check Pass: Memory is equal to (all numa memory + memory device)') else: test.fail('Memory is not equal to (all numa memory + memory device)') return dom_mem def check_save_restore(): """ Test save and restore operation """ save_file = os.path.join(data_dir.get_tmp_dir(), "%s.save" % vm_name) ret = virsh.save(vm_name, save_file, **virsh_dargs) libvirt.check_exit_status(ret) ret = virsh.restore(save_file, **virsh_dargs) libvirt.check_exit_status(ret) if os.path.exists(save_file): os.remove(save_file) # Login to check vm status vm.wait_for_login().close() def add_device(dev_xml, attach, at_error=False): """ Add memory device by attachment or modify domain xml. """ if attach: ret = virsh.attach_device(vm_name, dev_xml.xml, flagstr=attach_option, debug=True) libvirt.check_exit_status(ret, at_error) else: vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) if numa_cells: del vmxml.max_mem del vmxml.current_mem vmxml.add_device(dev_xml) vmxml.sync() def modify_domain_xml(): """ Modify domain xml and define it. """ vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) mem_unit = params.get("mem_unit", "KiB") vcpu = params.get("vcpu", "4") if max_mem_rt: vmxml.max_mem_rt = int(max_mem_rt) vmxml.max_mem_rt_slots = max_mem_slots vmxml.max_mem_rt_unit = mem_unit if memory_val: vmxml.memory = int(memory_val) if vcpu: vmxml.vcpu = int(vcpu) vcpu_placement = params.get("vcpu_placement", "static") vmxml.placement = vcpu_placement if numa_memnode: vmxml.numa_memory = {} vmxml.numa_memnode = numa_memnode else: try: del vmxml.numa_memory del vmxml.numa_memnode except Exception: # Not exists pass if numa_cells: cells = [ast.literal_eval(x) for x in numa_cells] # Rounding the numa memory values if align_mem_values: for cell in range(cells.__len__()): memory_value = str(utils_numeric.align_value( cells[cell]["memory"], align_to_value)) cells[cell]["memory"] = memory_value cpu_xml = vm_xml.VMCPUXML() cpu_xml.xml = "<cpu><numa/></cpu>" cpu_mode = params.get("cpu_mode") model_fallback = params.get("model_fallback") if cpu_mode: cpu_xml.mode = cpu_mode if model_fallback: cpu_xml.fallback = model_fallback cpu_xml.numa_cell = cells vmxml.cpu = cpu_xml # Delete memory and currentMemory tag, # libvirt will fill it automatically del vmxml.max_mem del vmxml.current_mem # hugepages setting if huge_pages: membacking = vm_xml.VMMemBackingXML() hugepages = vm_xml.VMHugepagesXML() pagexml_list = [] for i in range(len(huge_pages)): pagexml = hugepages.PageXML() pagexml.update(huge_pages[i]) pagexml_list.append(pagexml) hugepages.pages = pagexml_list membacking.hugepages = hugepages vmxml.mb = membacking logging.debug("vm xml: %s", vmxml) vmxml.sync() pre_vm_state = params.get("pre_vm_state", "running") attach_device = "yes" == params.get("attach_device", "no") detach_device = "yes" == params.get("detach_device", "no") attach_error = "yes" == params.get("attach_error", "no") start_error = "yes" == params.get("start_error", "no") detach_error = "yes" == params.get("detach_error", "no") maxmem_error = "yes" == params.get("maxmem_error", "no") attach_option = params.get("attach_option", "") test_qemu_cmd = "yes" == params.get("test_qemu_cmd", "no") test_managedsave = "yes" == params.get("test_managedsave", "no") test_save_restore = "yes" == params.get("test_save_restore", "no") test_mem_binding = "yes" == params.get("test_mem_binding", "no") restart_libvirtd = "yes" == params.get("restart_libvirtd", "no") add_mem_device = "yes" == params.get("add_mem_device", "no") test_dom_xml = "yes" == params.get("test_dom_xml", "no") max_mem = params.get("max_mem") max_mem_rt = params.get("max_mem_rt") max_mem_slots = params.get("max_mem_slots", "16") memory_val = params.get('memory_val', '') mem_align = 'yes' == params.get('mem_align', 'no') hot_plug = 'yes' == params.get('hot_plug', 'no') cur_mem = params.get("current_mem") numa_cells = params.get("numa_cells", "").split() set_max_mem = params.get("set_max_mem") align_mem_values = "yes" == params.get("align_mem_values", "no") align_to_value = int(params.get("align_to_value", "65536")) hot_reboot = "yes" == params.get("hot_reboot", "no") rand_reboot = "yes" == params.get("rand_reboot", "no") guest_known_unplug_errors = [] guest_known_unplug_errors.append(params.get("guest_known_unplug_errors")) host_known_unplug_errors = [] host_known_unplug_errors.append(params.get("host_known_unplug_errors")) # params for attached device mem_model = params.get("mem_model", "dimm") tg_size = params.get("tg_size") tg_sizeunit = params.get("tg_sizeunit", 'KiB') tg_node = params.get("tg_node", 0) pg_size = params.get("page_size") pg_unit = params.get("page_unit", "KiB") node_mask = params.get("node_mask", "0") mem_addr = ast.literal_eval(params.get("memory_addr", "{}")) huge_pages = [ast.literal_eval(x) for x in params.get("huge_pages", "").split()] numa_memnode = [ast.literal_eval(x) for x in params.get("numa_memnode", "").split()] at_times = int(params.get("attach_times", 1)) online = params.get("mem_online", "no") config = utils_config.LibvirtQemuConfig() setup_hugepages_flag = params.get("setup_hugepages") if (setup_hugepages_flag == "yes"): setup_hugepages(int(pg_size)) # Back up xml file. vmxml_backup = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) if not libvirt_version.version_compare(1, 2, 14): test.cancel("Memory hotplug not supported in current libvirt version.") if 'align_256m' in params.get('name', ''): arch = platform.machine() if arch.lower() != 'ppc64le': test.cancel('This case is for ppc64le only.') if align_mem_values: # Rounding the following values to 'align' max_mem = utils_numeric.align_value(max_mem, align_to_value) max_mem_rt = utils_numeric.align_value(max_mem_rt, align_to_value) cur_mem = utils_numeric.align_value(cur_mem, align_to_value) tg_size = utils_numeric.align_value(tg_size, align_to_value) try: # Drop caches first for host has enough memory drop_caches() # Destroy domain first if vm.is_alive(): vm.destroy(gracefully=False) modify_domain_xml() # Start the domain any way if attach memory device old_mem_total = None if attach_device: vm.start() session = vm.wait_for_login() old_mem_total = vm.get_totalmem_sys(online) logging.debug("Memtotal on guest: %s", old_mem_total) session.close() dev_xml = None # To attach the memory device. if add_mem_device and not hot_plug: at_times = int(params.get("attach_times", 1)) dev_xml = utils_hotplug.create_mem_xml(tg_size, pg_size, mem_addr, tg_sizeunit, pg_unit, tg_node, node_mask, mem_model) randvar = 0 rand_value = random.randint(15, 25) logging.debug("reboots at %s", rand_value) for x in xrange(at_times): # If any error excepted, command error status should be # checked in the last time randvar = randvar + 1 logging.debug("attaching device count = %s", x) if x == at_times - 1: add_device(dev_xml, attach_device, attach_error) else: add_device(dev_xml, attach_device) if hot_reboot: vm.reboot() vm.wait_for_login() if rand_reboot and randvar == rand_value: randvar = 0 rand_value = random.randint(15, 25) logging.debug("reboots at %s", rand_value) vm.reboot() vm.wait_for_login() # Check domain xml after attach device. if test_dom_xml: check_dom_xml(at_mem=attach_device) # Set domain state if pre_vm_state == "transient": logging.info("Creating %s...", vm_name) vmxml_for_test = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) if vm.is_alive(): vm.destroy(gracefully=False) vm.undefine() if virsh.create(vmxml_for_test.xml, **virsh_dargs).exit_status: vmxml_backup.define() test.fail("Cann't create the domain") elif vm.is_dead(): try: vm.start() vm.wait_for_login().close() except virt_vm.VMStartError as detail: if start_error: pass else: except_msg = "memory hotplug isn't supported by this QEMU binary" if except_msg in detail.reason: test.cancel(detail) test.fail(detail) # Set memory operation if set_max_mem: max_mem_option = params.get("max_mem_option", "") ret = virsh.setmaxmem(vm_name, set_max_mem, flagstr=max_mem_option) libvirt.check_exit_status(ret, maxmem_error) # Hotplug memory device if add_mem_device and hot_plug: process.run('ps -ef|grep qemu', shell=True, verbose=True) session = vm.wait_for_login() original_mem = vm.get_totalmem_sys() dev_xml = utils_hotplug.create_mem_xml(tg_size, pg_size, mem_addr, tg_sizeunit, pg_unit, tg_node, node_mask, mem_model) add_device(dev_xml, True) mem_after = vm.get_totalmem_sys() params['delta'] = mem_after - original_mem # Check domain xml after start the domain. if test_dom_xml: check_dom_xml(at_mem=attach_device) if mem_align: dom_mem = check_mem_align() check_qemu_cmd(dom_mem['maxMemory'], dom_mem['attached_mem']) if hot_plug and params['delta'] != dom_mem['attached_mem']: test.fail('Memory after attach not equal to original mem + attached mem') # Check qemu command line if test_qemu_cmd: check_qemu_cmd(max_mem_rt, tg_size) # Check guest meminfo after attachment if (attach_device and not attach_option.count("config") and not any([attach_error, start_error])): check_guest_meminfo(old_mem_total, check_option="attach") # Consuming memory on guest, # to verify memory changes by numastat if test_mem_binding: pid = vm.get_pid() old_numastat = read_from_numastat(pid, "Total") logging.debug("Numastat: %s", old_numastat) consume_vm_mem() new_numastat = read_from_numastat(pid, "Total") logging.debug("Numastat: %s", new_numastat) # Only check total memory which is the last element if float(new_numastat[-1]) - float(old_numastat[-1]) < 0: test.fail("Numa memory can't be consumed on guest") # Run managedsave command to check domain xml. if test_managedsave: ret = virsh.managedsave(vm_name, **virsh_dargs) libvirt.check_exit_status(ret) vm.start() vm.wait_for_login().close() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Run save and restore command to check domain xml if test_save_restore: check_save_restore() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Check domain xml after restarting libvirtd if restart_libvirtd: libvirtd = utils_libvirtd.Libvirtd() libvirtd.restart() if test_dom_xml: check_dom_xml(at_mem=attach_device) # Detach the memory device unplug_failed_with_known_error = False if detach_device: if not dev_xml: dev_xml = utils_hotplug.create_mem_xml(tg_size, pg_size, mem_addr, tg_sizeunit, pg_unit, tg_node, node_mask, mem_model) for x in xrange(at_times): ret = virsh.detach_device(vm_name, dev_xml.xml, flagstr=attach_option) if ret.stderr and host_known_unplug_errors: for known_error in host_known_unplug_errors: if (known_error[0] == known_error[-1]) and \ known_error.startswith(("'")): known_error = known_error[1:-1] if known_error in ret.stderr: unplug_failed_with_known_error = True logging.debug("Known error occured in Host, while" " hot unplug: %s", known_error) if unplug_failed_with_known_error: break try: libvirt.check_exit_status(ret, detach_error) except Exception as detail: dmesg_file = tempfile.mktemp(dir=data_dir.get_tmp_dir()) try: session = vm.wait_for_login() utils_misc.verify_dmesg(dmesg_log_file=dmesg_file, ignore_result=True, session=session, level_check=5) except Exception: session.close() test.fail("After memory unplug Unable to connect to VM" " or unable to collect dmesg") session.close() if os.path.exists(dmesg_file): with open(dmesg_file, 'r') as f: flag = re.findall( r'memory memory\d+?: Offline failed', f.read()) if not flag: # The attached memory is used by vm, and it could not be unplugged # The result is expected os.remove(dmesg_file) test.fail(detail) unplug_failed_with_known_error = True os.remove(dmesg_file) # Check whether a known error occured or not dmesg_file = tempfile.mktemp(dir=data_dir.get_tmp_dir()) try: session = vm.wait_for_login() utils_misc.verify_dmesg(dmesg_log_file=dmesg_file, ignore_result=True, session=session, level_check=4) except Exception: session.close() test.fail("After memory unplug Unable to connect to VM" " or unable to collect dmesg") session.close() if guest_known_unplug_errors and os.path.exists(dmesg_file): for known_error in guest_known_unplug_errors: if (known_error[0] == known_error[-1]) and \ known_error.startswith(("'")): known_error = known_error[1:-1] with open(dmesg_file, 'r') as f: if known_error in f.read(): unplug_failed_with_known_error = True logging.debug("Known error occured, while hot unplug" ": %s", known_error) if test_dom_xml and not unplug_failed_with_known_error: check_dom_xml(dt_mem=detach_device) # Remove dmesg temp file if os.path.exists(dmesg_file): os.remove(dmesg_file) finally: # Delete snapshots. snapshot_lists = virsh.snapshot_list(vm_name) if len(snapshot_lists) > 0: libvirt.clean_up_snapshots(vm_name, snapshot_lists) for snap in snapshot_lists: virsh.snapshot_delete(vm_name, snap, "--metadata") # Recover VM. if vm.is_alive(): vm.destroy(gracefully=False) logging.info("Restoring vm...") if (setup_hugepages_flag == "yes"): restore_hugepages() vmxml_backup.sync()