def test(self):
self.disk_path = None
while self.disk_path is None or os.path.exists(self.disk_path):
self.disk_path = (
"%s/disk_%s" %
(test.tmpdir, data_factory.generate_random_string(3)))
disk_size = int(
utils_misc.normalize_data_size(params.get("disk_size", "10M"),
"M"))
exp_str = r".*gzip: stdout: No space left on device.*"
vm_guest = env.get_vm("virt_test_vm1_guest")
process.run("mkdir -p %s" % (mount_path))
vm_guest.verify_alive()
vm_guest.wait_for_login(timeout=login_timeout)
create_file_disk(self.disk_path, disk_size)
mount(self.disk_path, mount_path, "-o loop")
vm_guest.migrate(mig_timeout,
mig_protocol,
not_wait_for_migration=True,
migration_exec_cmd_src=migration_exec_cmd_src,
env=env)
if not utils_misc.wait_for(
lambda: process_output_check(vm_guest.process, exp_str),
timeout=60,
first=1):
test.fail("The migration to destination with low "
"storage space didn't fail as it should.")
def _memory_stats_compare(self, keyname, memory_stat_qmp):
"""
Check whether memory statistics from qmp is same with guest memory.
:param keyname: key name of the output of the 'qom-get' property.
:param memory_stat_qmp: memory stat values from qmp.
"""
check_mem_ratio = float(self.params.get("check_mem_ratio", 0.1))
check_mem_diff = float(self.params.get("check_mem_diff", 150))
error_context.context("Get memory from guest", logging.info)
if keyname == "stat-free-memory":
guest_mem = self.get_guest_free_mem(self.vm)
elif keyname == "stat-total-memory":
guest_mem = self.get_vm_mem(self.vm)
memory_stat_qmp = "%sB" % memory_stat_qmp
memory_stat_qmp = int(float(utils_misc.normalize_data_size(
memory_stat_qmp, order_magnitude="M")))
mem_diff = float(abs(guest_mem - memory_stat_qmp))
if ((mem_diff / guest_mem) > check_mem_ratio and
mem_diff > check_mem_diff):
self.test.fail("%s of guest %s is not equal to %s in qmp,the"
"acceptable ratio/diff is %s/%s" % (keyname,
guest_mem,
memory_stat_qmp,
check_mem_ratio,
check_mem_diff))
def get_image_size(self, image):
"""
Get image size for given image.
:param image: image file.
:return: image size.
"""
force_share = False
pause_vm = False
params = self.params.object_params(image)
qemu_image = qemu_storage.QemuImg(params, self.data_dir, image)
if self.vm:
pids = process.getoutput(
"lsof %s |grep -v PID|awk '{print $2}'" % qemu_image.image_filename)
force_share = str(self.vm.get_pid()) in pids
if force_share and not self.vm.is_paused():
self.vm.pause()
pause_vm = True
image_info = qemu_image.info(force_share=force_share)
if self.vm and pause_vm:
self.vm.resume()
if not image_info:
self.test.error("Get image info failed.")
image_size = re.findall(r"disk size: (\d\.?\d*?.*)", image_info)[0]
image_size = int(float(utils_misc.normalize_data_size(image_size, "B")))
logging.info("Image size of %s is %s", image, image_size)
return image_size
def check_ksm(mem, stress=False):
"""
:param mem: Boot guest with given memory, in KB
:param stress: Load stress or not
"""
params['mem'] = mem // 1024
params['start_vm'] = 'yes'
vm_name = params['main_vm']
env_process.preprocess_vm(test, params, env, vm_name)
vm = env.get_vm(vm_name)
vm.wait_for_login()
if stress:
params['stress_args'] = ('--cpu 4 --io 4 --vm 2 --vm-bytes %sM' %
(int(params['mem']) // 2))
stress_test = VMStress(vm, "stress", params)
stress_test.load_stress_tool()
time.sleep(30)
qemu_pid = vm.get_pid()
qemu_used_page = utils_misc.normalize_data_size(process.getoutput(
params['cmd_get_qemu_used_mem'] % qemu_pid, shell=True) + 'K', 'B')
pagesize = utils_memory.getpagesize()
qemu_used_mem = int(float(qemu_used_page)) * pagesize
free_mem_host = utils_memory.freememtotal()
ksm_status = process.getoutput(params['cmd_check_ksm_status'])
vm.destroy()
logging.info('The ksm threshold is %s, the memory allocated by qemu is'
' %s, and the total free memory on host is %s.' %
(ksm_thres, qemu_used_mem, free_mem_host))
if free_mem_host >= ksm_thres:
if ksm_status != '0':
test.fail('Ksm should not start.')
if stress:
test.error('The host resource is not consumed as expected.')
elif ksm_status == '0':
test.fail('Ksm should start but it does not.')
def _memory_stats_compare(self, keyname, memory_stat_qmp):
"""
Check whether memory statistics from qmp is same with guest memory.
:param keyname: key name of the output of the 'qom-get' property.
:param memory_stat_qmp: memory stat values from qmp.
"""
check_mem_ratio = float(self.params.get("check_mem_ratio", 0.1))
check_mem_diff = float(self.params.get("check_mem_diff", 150))
error_context.context("Get memory from guest", logging.info)
if keyname == "stat-free-memory":
guest_mem = self.get_guest_free_mem(self.vm)
elif keyname == "stat-total-memory":
guest_mem = self.get_vm_mem(self.vm)
memory_stat_qmp = "%sB" % memory_stat_qmp
memory_stat_qmp = int(
float(
utils_misc.normalize_data_size(memory_stat_qmp,
order_magnitude="M")))
mem_diff = float(abs(guest_mem - memory_stat_qmp))
if ((mem_diff / guest_mem) > check_mem_ratio
and mem_diff > check_mem_diff):
self.test.fail("%s of guest %s is not equal to %s in qmp,the"
"acceptable ratio/diff is %s/%s" %
(keyname, guest_mem, memory_stat_qmp,
check_mem_ratio, check_mem_diff))
def rbd_image_info(ceph_monitor, rbd_pool_name, rbd_image_name):
"""
Get information of a rbd image
:params ceph_monitor: The specified monitor to connect to
:params rbd_pool_name: The name of rbd pool
:params rbd_image_name: The name of rbd image
"""
cmd = "rbd info %s/%s -m %s" % (rbd_pool_name, rbd_image_name,
ceph_monitor)
output = process.system(cmd)
info_pattern = "rbd image \'%s\':.*?$" % rbd_image_name
rbd_image_info_str = re.findall(info_pattern, output, re.S)[0]
rbd_image_info = {}
for rbd_image_line in rbd_image_info_str.splitlines():
if ":" not in rbd_image_line:
if "size" in rbd_image_line:
size_str = re.findall("size\s+(\d+\s+\w+)\s+",
rbd_image_line)[0]
size = utils_misc.normalize_data_size(size_str)
rbd_image_info['size'] = size
if "order" in rbd_image_line:
rbd_image_info['order'] = int(
re.findall("order\s+(\d+)", rbd_image_line))
else:
tmp_str = rbd_image_line.strip().split(":")
rbd_image_info[tmp_str[0]] = tmp_str[1]
return rbd_image_info
def rbd_image_info(ceph_monitor, rbd_pool_name, rbd_image_name):
"""
Get information of a rbd image
:params ceph_monitor: The specified monitor to connect to
:params rbd_pool_name: The name of rbd pool
:params rbd_image_name: The name of rbd image
"""
cmd = "rbd info %s/%s -m %s" % (rbd_pool_name, rbd_image_name,
ceph_monitor)
output = process.system(cmd)
info_pattern = "rbd image \'%s\':.*?$" % rbd_image_name
rbd_image_info_str = re.findall(info_pattern, output, re.S)[0]
rbd_image_info = {}
for rbd_image_line in rbd_image_info_str.splitlines():
if ":" not in rbd_image_line:
if "size" in rbd_image_line:
size_str = re.findall("size\s+(\d+\s+\w+)\s+",
rbd_image_line)[0]
size = utils_misc.normalize_data_size(size_str)
rbd_image_info['size'] = size
if "order" in rbd_image_line:
rbd_image_info['order'] = int(re.findall("order\s+(\d+)",
rbd_image_line))
else:
tmp_str = rbd_image_line.strip().split(":")
rbd_image_info[tmp_str[0]] = tmp_str[1]
return rbd_image_info
def rbd_image_create(ceph_monitor,
rbd_pool_name,
rbd_image_name,
rbd_image_size,
force_create=False):
"""
Create a rbd image.
:params ceph_monitor: The specified monitor to connect to
:params rbd_pool_name: The name of rbd pool
:params rbd_image_name: The name of rbd image
:params rbd_image_size: The size of rbd image
:params force_create: Force create the image or not
"""
if rbd_image_exist(ceph_monitor, rbd_pool_name, rbd_image_name):
create_image = False
image_info = rbd_image_info(ceph_monitor, rbd_pool_name,
rbd_image_name)
try:
int(rbd_image_size)
compare_str = rbd_image_size
except ValueError:
compare_str = utils_misc.normalize_data_size(rbd_image_size)
if image_info['size'] != compare_str or force_create:
rbd_image_rm(ceph_monitor, rbd_pool_name, rbd_image_name)
create_image = True
if create_image:
cmd = "rbd create %s/%s -m %s" % (rbd_pool_name, rbd_image_name,
ceph_monitor)
process.system(cmd, verbose=True)
else:
logging.debug("Image already exist skip the create.")
def rbd_image_create(ceph_monitor, rbd_pool_name, rbd_image_name,
rbd_image_size, force_create=False):
"""
Create a rbd image.
:params ceph_monitor: The specified monitor to connect to
:params rbd_pool_name: The name of rbd pool
:params rbd_image_name: The name of rbd image
:params rbd_image_size: The size of rbd image
:params force_create: Force create the image or not
"""
if rbd_image_exist(ceph_monitor, rbd_pool_name, rbd_image_name):
create_image = False
image_info = rbd_image_info(ceph_monitor, rbd_pool_name,
rbd_image_name)
try:
int(rbd_image_size)
compare_str = rbd_image_size
except ValueError:
compare_str = utils_misc.normalize_data_size(rbd_image_size)
if image_info['size'] != compare_str or force_create:
rbd_image_rm(ceph_monitor, rbd_pool_name, rbd_image_name)
create_image = True
if create_image:
cmd = "rbd create %s/%s -m %s" % (rbd_pool_name, rbd_image_name,
ceph_monitor)
process.system(cmd, verbose=True)
else:
logging.debug("Image already exist skip the create.")
def test(self):
self.disk_path = None
while self.disk_path is None or os.path.exists(self.disk_path):
self.disk_path = (
"%s/disk_%s" %
(test.tmpdir, data_factory.generate_random_string(3)))
disk_size = int(utils_misc.normalize_data_size(
params.get("disk_size", "10M"), "M"))
exp_str = r".*gzip: stdout: No space left on device.*"
vm_guest = env.get_vm("virt_test_vm1_guest")
process.run("mkdir -p %s" % (mount_path))
vm_guest.verify_alive()
vm_guest.wait_for_login(timeout=login_timeout)
create_file_disk(self.disk_path, disk_size)
mount(self.disk_path, mount_path, "-o loop")
vm_guest.migrate(mig_timeout, mig_protocol,
not_wait_for_migration=True,
migration_exec_cmd_src=migration_exec_cmd_src,
env=env)
if not utils_misc.wait_for(lambda: process_output_check(
vm_guest.process, exp_str),
timeout=60, first=1):
test.fail("The migration to destination with low "
"storage space didn't fail as it should.")
def get_memory_boundary(self, balloon_type=''):
"""
Get the legal memory boundary for balloon operation.
:param balloon_type: evict or enlarge
:type balloon_type: string
:return: min and max size of the memory
:rtype: tuple
"""
max_size = self.ori_mem
min_size = self.params.get("minmem", "512M")
min_size = int(float(utils_misc.normalize_data_size(min_size)))
balloon_buffer = self.params.get("balloon_buffer", 300)
if self.params.get('os_type') == 'windows':
logging.info("Get windows miminum balloon value:")
self.vm.balloon(1)
balloon_timeout = self.params.get("balloon_timeout", 900)
self.wait_for_balloon_complete(balloon_timeout)
used_size = min((self.get_ballooned_memory() + balloon_buffer),
max_size)
self.vm.balloon(max_size)
self.wait_for_balloon_complete(balloon_timeout)
self.ori_gmem = self.get_memory_status()
else:
vm_total = self.get_memory_status()
vm_mem_free = self.get_free_mem()
used_size = min((self.ori_mem - vm_mem_free + balloon_buffer),
max_size)
if balloon_type == "enlarge":
min_size = self.current_mmem
elif balloon_type == "evict":
max_size = self.current_mmem
min_size = max(used_size, min_size)
return min_size, max_size
def get_memory_boundary(self, balloon_type=''):
"""
Get the legal memory boundary for balloon operation.
:param balloon_type: evict or enlarge
:type balloon_type: string
:return: min and max size of the memory
:rtype: tuple
"""
max_size = self.ori_mem
min_size = self.params.get("minmem", "512M")
min_size = int(float(utils_misc.normalize_data_size(min_size)))
balloon_buffer = self.params.get("balloon_buffer", 300)
if self.params.get('os_type') == 'windows':
logging.info("Get windows miminum balloon value:")
self.vm.balloon(1)
balloon_timeout = self.params.get("balloon_timeout", 900)
self.wait_for_balloon_complete(balloon_timeout)
used_size = min((self.get_ballooned_memory() + balloon_buffer), max_size)
self.vm.balloon(max_size)
self.wait_for_balloon_complete(balloon_timeout)
self.ori_gmem = self.get_memory_status()
else:
vm_total = self.get_memory_status()
vm_mem_free = self.get_free_mem()
used_size = min((self.ori_mem - vm_mem_free + balloon_buffer), max_size)
if balloon_type == "enlarge":
min_size = self.current_mmem
elif balloon_type == "evict":
max_size = self.current_mmem
min_size = max(used_size, min_size)
return min_size, max_size
def memory_check(vm, get_polling_output, keyname):
"""
Check memory status.
:param vm: VM object.
:param get_polling_output: output of get polling in qmp.
:param keyname: key name of the output of the 'qom-get' property.
"""
check_mem_ratio = float(params.get("check_mem_ratio", 0.1))
error_context.context("Get memory from guest", logging.info)
mem_base = MemoryBaseTest(test, params, env)
if keyname == "stat-free-memory":
guest_mem = mem_base.get_guest_free_mem(vm)
elif keyname == "stat-total-memory":
guest_mem = mem_base.get_vm_mem(vm)
error_context.context("Get memory from qmp", logging.info)
stat_memory_qmp = get_polling_output['stats'][keyname]
stat_memory_qmp = "%sB" % stat_memory_qmp
stat_memory_qmp = int(float(utils_misc.normalize_data_size(
(stat_memory_qmp), order_magnitude="M")))
error_context.context("Compare memory from guest with qmp",
logging.info)
if (abs(guest_mem - stat_memory_qmp)) > (guest_mem * check_mem_ratio):
raise exceptions.TestFail("%s of guest %s is not equal to %s in"
" qmp, the acceptable ratio is %s" %
(keyname, guest_mem, stat_memory_qmp,
check_mem_ratio))
def verify_info(self, params=None):
"""
verify option is applied to image file correctly
"""
error_context.context("verify option of converted image", logging.info)
image_filename = storage.get_image_filename(params, self.data_dir)
info = utils_test.get_image_info(image_filename)
avalue = evalue = ""
for option in params.objects("option_verified"):
avalue = info.get(option)
if option == "format":
evalue = params.get("image_format")
elif option == "lcounts":
if params.get("lazy_refcounts") == "on":
evalue = "true"
elif params.get("lazy_refcounts") == "off":
evalue = "false"
elif option == "csize":
csize = params.get("cluster_size")
evalue = int(float(utils_misc.normalize_data_size(csize, "B")))
elif option == "sparse_size":
if info.get("dsize") < info.get("vsize"):
avalue = info.get("dsize")
evalue = info.get("vsize")
else:
evalue = params.get(option)
if avalue is not None and avalue != evalue:
msg = "Get wrong %s from image %s!" % (option, image_filename)
msg += "Expect: %s, actual: %s" % (evalue, avalue)
self.test.fail(msg)
def memory_check(vm, get_polling_output, keyname):
"""
Check memory status.
:param vm: VM object.
:param get_polling_output: output of get polling in qmp.
:param keyname: key name of the output of the 'qom-get' property.
"""
error_context.context("Check whether memory status as expected",
logging.info)
check_mem_ratio = float(params.get("check_mem_ratio", 0.1))
mem_base = MemoryBaseTest(test, params, env)
if keyname == "stat-free-memory":
guest_mem = mem_base.get_guest_free_mem(vm)
elif keyname == "stat-total-memory":
guest_mem = mem_base.get_vm_mem(vm)
stat_memory_qmp = get_polling_output['stats'][keyname]
stat_memory_qmp = "%sB" % stat_memory_qmp
stat_memory_qmp = int(float(utils_misc.normalize_data_size(
(stat_memory_qmp), order_magnitude="M")))
if (abs(guest_mem - stat_memory_qmp)) > (guest_mem * check_mem_ratio):
raise exceptions.TestFail("%s of guest %s is not equal to %s in"
" qmp, the ratio is %s" % (keyname,
guest_mem,
stat_memory_qmp,
check_mem_ratio))
def verify_info(self, params=None):
"""
verify option is applied to image file correctly
"""
error_context.context("verify option of converted image", logging.info)
image_filename = storage.get_image_filename(params, self.data_dir)
info = utils_test.get_image_info(image_filename)
avalue = evalue = ""
for option in params.objects("option_verified"):
avalue = info.get(option)
if option == "format":
evalue = params.get("image_format")
elif option == "lcounts":
if params.get("lazy_refcounts") == "on":
evalue = "true"
elif params.get("lazy_refcounts") == "off":
evalue = "false"
elif option == "csize":
csize = params.get("image_cluster_size")
evalue = int(float(utils_misc.normalize_data_size(csize, "B")))
elif option == "sparse_size":
if info.get("dsize") < info.get("vsize"):
avalue = info.get("dsize")
evalue = info.get("vsize")
elif option == "compat":
evalue = params.get("qcow2_compatible")
else:
evalue = params.get(option)
if avalue is not None and avalue != evalue:
msg = "Get wrong %s from image %s!" % (option, image_filename)
msg += "Expect: %s, actual: %s" % (evalue, avalue)
self.test.fail(msg)
def get_memory_boundary(self, balloon_type=''):
"""
Get the legal memory boundary for balloon operation.
:param balloon_type: evict or enlarge
:type balloon_type: string
:return: min and max size of the memory
:rtype: tuple
"""
max_size = self.ori_mem
min_size = self.params.get("minmem", "512M")
min_size = int(float(utils_misc.normalize_data_size(min_size)))
if self.params.get('os_type') == 'windows':
logging.info("Get windows miminum balloon value:")
self.vm.balloon(1)
time.sleep(90)
used_size = int(self.get_ballooned_memory() + self.ratio * self.ori_mem)
self.vm.balloon(max_size)
time.sleep(90)
self.ori_gmem = self.get_memory_status()
else:
vm_total = self.get_memory_status()
vm_mem_free = self.get_free_mem()
used_size = vm_total - vm_mem_free + 16
if balloon_type == "enlarge":
min_size = self.current_mmem
elif balloon_type == "evict":
max_size = self.current_mmem
min_size = max(used_size, min_size)
return min_size, max_size
def speed2byte(speed):
"""
convert speed to Bytes/s
"""
if str(speed).isdigit():
speed = "%sB" % speed
speed = utils_misc.normalize_data_size(speed, "B")
return int(float(speed))
def speed2byte(speed):
"""
convert speed to Bytes/s
"""
if str(speed).isdigit():
speed = "%sB" % speed
speed = utils_misc.normalize_data_size(speed, "B")
return int(float(speed))
def _get_size_value(size_str):
"""
Get size value form size string
:param size_str: data size string
:return: the size numeric value, measured by MB
"""
size_str = utils_misc.normalize_data_size(size_str)
size = float(size_str)
return size
def get_image_info(image_file):
"""
Get image information and put it into a dict. Image information like this:
*******************************
image: /path/vm1_6.3.img
file format: raw
virtual size: 10G (10737418240 bytes)
disk size: 888M
....
....
*******************************
And the image info dict will be like this
image_info_dict = { 'format':'raw',
'vsize' : '10737418240'
'dsize' : '931135488'
}
TODO: Add more information to dict
"""
try:
cmd = "qemu-img info %s" % image_file
image_info = utils.run(cmd, ignore_status=False).stdout.strip()
image_info_dict = {}
if image_info:
for line in image_info.splitlines():
if line.find("format") != -1:
image_info_dict['format'] = line.split(':')[-1].strip()
elif line.find("virtual size") != -1:
vsize = line.split(":")[-1].strip().split(" ")[0]
vsize = utils_misc.normalize_data_size(vsize,
order_magnitude="B",
factor=1024)
image_info_dict['vsize'] = int(float(vsize))
elif line.find("disk size") != -1:
dsize = line.split(':')[-1].strip()
dsize = utils_misc.normalize_data_size(dsize,
order_magnitude="B",
factor=1024)
image_info_dict['dsize'] = int(float(dsize))
return image_info_dict
except (KeyError, IndexError, ValueError, error.CmdError), detail:
raise error.TestError("Fail to get information of %s:\n%s" %
(image_file, detail))
def get_image_info(image_file):
"""
Get image information and put it into a dict. Image information like this:
*******************************
image: /path/vm1_6.3.img
file format: raw
virtual size: 10G (10737418240 bytes)
disk size: 888M
....
....
*******************************
And the image info dict will be like this
image_info_dict = { 'format':'raw',
'vsize' : '10737418240'
'dsize' : '931135488'
}
TODO: Add more information to dict
"""
try:
cmd = "qemu-img info %s" % image_file
image_info = utils.run(cmd, ignore_status=False).stdout.strip()
image_info_dict = {}
if image_info:
for line in image_info.splitlines():
if line.find("format") != -1:
image_info_dict['format'] = line.split(':')[-1].strip()
elif line.find("virtual size") != -1:
vsize = line.split(":")[-1].strip().split(" ")[0]
vsize = utils_misc.normalize_data_size(vsize,
order_magnitude="B",
factor=1024)
image_info_dict['vsize'] = int(float(vsize))
elif line.find("disk size") != -1:
dsize = line.split(':')[-1].strip()
dsize = utils_misc.normalize_data_size(dsize,
order_magnitude="B",
factor=1024)
image_info_dict['dsize'] = int(float(dsize))
return image_info_dict
except (KeyError, IndexError, ValueError, error.CmdError), detail:
raise error.TestError("Fail to get information of %s:\n%s" %
(image_file, detail))
def get_ksmstat():
"""
Return sharing memory by ksm in MB
:return: memory in MB
"""
fpages = open('/sys/kernel/mm/ksm/pages_sharing')
ksm_pages = int(fpages.read())
fpages.close()
sharing_mem = ksm_pages * pagesize
return int(float(utils_misc.normalize_data_size("%sK" % sharing_mem)))
def set_vm_for_dump(test, params):
"""
Update vm mem and image size params to match dump generate and analysed.
:param test: kvm test object
:param params: Params object
"""
host_free_mem = utils_misc.get_mem_info(attr='MemFree')
host_avail_disk = int(process.getoutput(params["get_avail_disk"]))
sys_image_size = int(
float(utils_misc.normalize_data_size(params["image_size"], "G")))
if host_avail_disk < (host_free_mem // 1024**2) * 1.2 + sys_image_size:
params["mem"] = (host_avail_disk - sys_image_size) * 0.8 // 2.4 * 1024
image_size_stg = int(
float(utils_misc.normalize_data_size(params["mem"] + "M", "G")) * 1.4)
params["image_size_stg"] = str(image_size_stg) + "G"
params["force_create_image_stg"] = "yes"
image_params = params.object_params("stg")
env_process.preprocess_image(test, image_params, "stg")
def run_test_virtio_mem():
"""
Update memory device for virtio-mem device
"""
mem_device_attrs = eval(params.get('mem_device_attrs', '{}'))
vm.start()
vm_session = vm.wait_for_login()
cmdRes = vm_session.cmd_output('free -m')
vm_mem_before = int(re.findall(r'Mem:\s+(\d+)\s+\d+\s+', cmdRes)[-1])
test.log.debug("VM's memory before updating requested size: %s",
vm_mem_before)
test.log.info(
"TEST_STEP1: Update requested size for virtio-mem device.")
vmxml_cur = vm_xml.VMXML.new_from_dumpxml(vm_name)
mem_dev = vmxml_cur.devices.by_device_tag("memory")[0]
mem_dev_alias = mem_dev.fetch_attrs()['alias']['name']
virsh_opts = params.get('virsh_opts') % mem_dev_alias
virsh.update_memory_device(vm.name,
options=virsh_opts,
wait_for_event=True,
**VIRSH_ARGS)
test.log.info("TEST_STEP2: Check requested and current size changes.")
mem_dev = vm_xml.VMXML.new_from_dumpxml(vm_name).devices.\
by_device_tag("memory")[0]
expr_requested_size = int(
float(
utils_misc.normalize_data_size(params.get(
"requested_size", '80Mib'),
order_magnitude='K')))
for check_item in ['requested_size', 'current_size']:
if getattr(mem_dev.target, check_item) != expr_requested_size:
test.fail("Incorrect %s! It should be %s, but got %s." %
(check_item, expr_requested_size,
getattr(mem_dev.target, check_item)))
test.log.info("TEST_STEP3: Check 'MEMORY_DEVICE_SIZE_CHANGE' in "
"libvirtd/virtqemud log")
log_file = utils_misc.get_path(test.debugdir, "libvirtd.log")
check_log_str = params.get("check_log_str",
"MEMORY_DEVICE_SIZE_CHANGE")
libvirt.check_logfile(check_log_str, log_file)
test.log.info("TEST STEP4: Check memory in the VM.")
cmdRes = vm_session.cmd_output('free -m')
vm_mem_after = int(re.findall(r'Mem:\s+(\d+)\s+\d+\s+', cmdRes)[-1])
mem_request_decrease = (mem_device_attrs['target']['requested_size'] -
expr_requested_size) / 1024
vm_mem_decrease = vm_mem_before - vm_mem_after
if mem_request_decrease != vm_mem_decrease:
test.fail(
"VM mem change comparison failed! Expect %d, but got %d." %
(mem_request_decrease, vm_mem_decrease))
def normalize_mem_size(cls, str_size):
"""
Convert memory size unit
:param str_size: memory size string, like: 1GB
:return: memory size value in MB
"""
args = (str_size, cls.UNIT, 1024)
try:
size = utils_misc.normalize_data_size(*args)
return int(float(size))
except ValueError, details:
logging.debug("Convert memory size error('%s')" % details)
def setup(self, force_start=False):
"""
Setup test NFS share.
:param force_start: Whether to make NFS service start anyway.
"""
error_context.context("Setting up test NFS share", logging.info)
for d in [self.nfs_dir, self.mnt_dir]:
try:
os.makedirs(d)
except OSError:
pass
error_context.context("Checking available space to export",
logging.info)
stat = os.statvfs(self.nfs_dir)
free = str(stat.f_bsize * stat.f_bfree) + 'B'
available_size = float(
utils_misc.normalize_data_size(free, order_magnitude="M"))
required_size = float(
utils_misc.normalize_data_size(self.required_size,
order_magnitude="M"))
if available_size < required_size:
raise NFSCorruptError("Space available: %fM, space needed: %fM" %
(available_size, required_size))
if force_start:
self.start_service()
else:
if not self.is_service_active():
self.start_service()
process.run("exportfs %s:%s -o rw,no_root_squash" %
(self.nfs_ip, self.nfs_dir),
shell=True)
process.run("mount %s:%s %s -o rw,soft,timeo=30,retrans=1,vers=3" %
(self.nfs_ip, self.nfs_dir, self.mnt_dir),
shell=True)
def normalize_mem_size(cls, str_size):
"""
Convert memory size unit
:param str_size: memory size string, like: 1GB
:return: memory size value in MB
"""
args = (str_size, cls.UNIT, 1024)
try:
size = utils_misc.normalize_data_size(*args)
return int(float(size))
except ValueError, details:
logging.debug("Convert memory size error('%s')" % details)
def test(self):
self.copier_pid = None
if params.get("nettype") != "bridge":
test.cancel("Unable start test without params nettype=bridge.")
self.disk_serial = params.get("drive_serial_image2_vm1",
"nfs-disk-image2-vm1")
self.disk_serial_src = params.get("drive_serial_image1_vm1",
"root-image1-vm1")
self.guest_mount_path = params.get("guest_disk_mount_path", "/mnt")
self.copy_timeout = int(params.get("copy_timeout", "1024"))
self.copy_block_size = int(
utils_misc.normalize_data_size(
params.get("copy_block_size", "100M"), "M"))
self.disk_size = "%sM" % int(self.copy_block_size * 1.4)
self.server_recover_timeout = (int(
params.get("server_recover_timeout", "240")))
process.run("mkdir -p %s" % (mount_path))
self.test_params()
self.config()
self.vm_guest_params = params.copy()
self.vm_guest_params["images_base_dir_image2_vm1"] = mount_path
self.vm_guest_params[
"image_name_image2_vm1"] = "ni_mount_%s/test" % (test_rand)
self.vm_guest_params["image_size_image2_vm1"] = self.disk_size
self.vm_guest_params = self.vm_guest_params.object_params("vm1")
self.image2_vm_guest_params = (
self.vm_guest_params.object_params("image2"))
env_process.preprocess_image(test, self.image2_vm_guest_params,
env)
self.vm_guest.create(params=self.vm_guest_params)
self.vm_guest.verify_alive()
self.vm_guest.wait_for_login(timeout=login_timeout)
self.workload()
self.restart_server()
self.vm_guest.migrate(mig_timeout, mig_protocol, env=env)
try:
self.vm_guest.verify_alive()
self.vm_guest.wait_for_login(timeout=login_timeout)
except aexpect.ExpectTimeoutError:
test.fail("Migration should be successful.")
def create_luks_vol(vol_name, sec_uuid, params, test):
"""
Create a luks volume
:param vol_name: the name of the volume
:param sec_uuid: secret's uuid to be used for luks encryption
:param params: detailed params to create volume
:param test: test object
"""
pool_name = params.get("pool_name")
extra_option = params.get("extra_option", "")
unprivileged_user = params.get('unprivileged_user')
uri = params.get("virsh_uri")
vol_arg = {}
for key in list(params.keys()):
if (key.startswith('vol_') and not key.startswith('vol_new')):
if key[4:] in ['capacity', 'allocation']:
vol_arg[key[4:]] = int(
float(
utils_misc.normalize_data_size(params[key], "B",
1024)))
elif key[4:] in ['owner', 'group']:
vol_arg[key[4:]] = int(params[key])
else:
vol_arg[key[4:]] = params[key]
if vol_arg[key[
4:]] == "qcow2" and not libvirt_version.version_compare(
6, 10, 0):
test.cancel("Qcow2 format with luks encryption is not"
" supported in current libvirt version")
vol_arg['name'] = vol_name
volxml = libvirt_xml.VolXML()
newvol = volxml.new_vol(**vol_arg)
luks_encryption_params = {}
luks_encryption_params.update({"format": "luks"})
luks_encryption_params.update(
{"secret": {
"type": "passphrase",
"uuid": sec_uuid
}})
newvol.encryption = volxml.new_encryption(**luks_encryption_params)
vol_xml = newvol['xml']
if params.get('setup_libvirt_polkit') == 'yes':
process.run("chmod 666 %s" % vol_xml, ignore_status=True, shell=True)
logging.debug("Create volume from XML: %s" % newvol.xmltreefile)
cmd_result = virsh.vol_create(pool_name,
vol_xml,
extra_option,
unprivileged_user=unprivileged_user,
uri=uri,
ignore_status=True,
debug=True)
def test(self):
self.copier_pid = None
if params.get("nettype") != "bridge":
test.cancel("Unable start test without params nettype=bridge.")
self.disk_serial = params.get("drive_serial_image2_vm1",
"nfs-disk-image2-vm1")
self.disk_serial_src = params.get("drive_serial_image1_vm1",
"root-image1-vm1")
self.guest_mount_path = params.get("guest_disk_mount_path", "/mnt")
self.copy_timeout = int(params.get("copy_timeout", "1024"))
self.copy_block_size = int(utils_misc.normalize_data_size(
params.get("copy_block_size", "100M"), "M"))
self.disk_size = "%sM" % int(self.copy_block_size * 1.4)
self.server_recover_timeout = (
int(params.get("server_recover_timeout", "240")))
process.run("mkdir -p %s" % (mount_path))
self.test_params()
self.config()
self.vm_guest_params = params.copy()
self.vm_guest_params["images_base_dir_image2_vm1"] = mount_path
self.vm_guest_params["image_name_image2_vm1"] = "ni_mount_%s/test" % (test_rand)
self.vm_guest_params["image_size_image2_vm1"] = self.disk_size
self.vm_guest_params = self.vm_guest_params.object_params("vm1")
self.image2_vm_guest_params = (self.vm_guest_params.
object_params("image2"))
env_process.preprocess_image(test,
self.image2_vm_guest_params,
env)
self.vm_guest.create(params=self.vm_guest_params)
self.vm_guest.verify_alive()
self.vm_guest.wait_for_login(timeout=login_timeout)
self.workload()
self.restart_server()
self.vm_guest.migrate(mig_timeout, mig_protocol, env=env)
try:
self.vm_guest.verify_alive()
self.vm_guest.wait_for_login(timeout=login_timeout)
except aexpect.ExpectTimeoutError:
test.fail("Migration should be successful.")
def get_win_mon_free_mem(self, session):
"""
Get Performance Monitored Free memory.
:param session: shell Object
:return string: freespace M-bytes
"""
cmd = r'typeperf "\Memory\Free & Zero Page List Bytes" -sc 1'
status, output = session.cmd_status_output(cmd)
if status == 0:
free = "%s" % re.findall(r"\d+\.\d+", output)[2]
free = float(utils_misc.normalize_data_size(free, order_magnitude="M"))
return int(free)
else:
self.test.fail("Failed to get windows guest free memory")
def setup(self, force_start=False):
"""
Setup test NFS share.
:param force_start: Whether to make NFS service start anyway.
"""
error_context.context("Setting up test NFS share", logging.info)
for d in [self.nfs_dir, self.mnt_dir]:
try:
os.makedirs(d)
except OSError:
pass
error_context.context("Checking available space to export",
logging.info)
stat = os.statvfs(self.nfs_dir)
free = str(stat.f_bsize * stat.f_bfree) + 'B'
available_size = float(utils_misc.normalize_data_size(free,
order_magnitude="M"))
required_size = float(utils_misc.normalize_data_size(self.required_size,
order_magnitude="M"))
if available_size < required_size:
raise NFSCorruptError("Space available: %fM, space needed: %fM"
% (available_size, required_size))
if force_start:
self.start_service()
else:
if not self.is_service_active():
self.start_service()
process.run("exportfs %s:%s -o rw,no_root_squash" %
(self.nfs_ip, self.nfs_dir), shell=True)
process.run("mount %s:%s %s -o rw,soft,timeo=30,retrans=1,vers=3" %
(self.nfs_ip, self.nfs_dir, self.mnt_dir), shell=True)
def get_image_size(self, image):
"""
Get image size for given image.
:param image: image file.
:return: image size.
"""
params = self.params.object_params(image)
qemu_image = qemu_storage.QemuImg(params, self.data_dir, image)
image_info = qemu_image.info()
if not image_info:
self.test.error("Get image info failed.")
image_size = re.findall("disk size: (\d\.?\d*?.*)", image_info)[0]
image_size = int(float(utils_misc.normalize_data_size(image_size, "B")))
logging.info("Image size of %s is %s" % (image, image_size))
return image_size
def get_win_mon_free_mem(self, session):
"""
Get Performance Monitored Free memory.
:param session: shell Object
:return string: freespace M-bytes
"""
cmd = 'typeperf "\Memory\Free & Zero Page List Bytes" -sc 1'
status, output = session.cmd_status_output(cmd)
if status == 0:
free = "%s" % re.findall("\d+\.\d+", output)[2]
free = float(utils_misc.normalize_data_size(free, order_magnitude="M"))
return int(free)
else:
error.TestFail("Failed to get windows guest free memory")
def get_tmpfs_write_speed():
"""
Get the tmpfs write speed of the host
return: The write speed of tmpfs, the unit is kb/s.
"""
utils.run("mkdir -p /tmp/test_speed && mount -t tmpfs none /tmp/test_speed")
output = utils.run("dd if=/dev/urandom of=/tmp/test_speed/test bs=1k count=1024")
try:
speed = re.search("\s([\w\s\.]+)/s", output.stderr, re.I).group(1)
return float(utils_misc.normalize_data_size(speed, 'K', 1024))
except Exception:
return 3072
finally:
utils.run("umount /tmp/test_speed")
os.removedirs("/tmp/test_speed")
def get_tmpfs_write_speed():
"""
Get the tmpfs write speed of the host
return: The write speed of tmpfs, the unit is kb/s.
"""
utils.run("mkdir /tmp/test_speed && mount -t tmpfs none /tmp/test_speed")
output = utils.run("dd if=/dev/urandom of=/tmp/test_speed/test bs=1k count=1024")
try:
speed = re.search("\s([\w\s\.]+)/s", output.stderr, re.I).group(1)
return float(utils_misc.normalize_data_size(speed, 'K', 1024))
except Exception:
return 3072
finally:
os.remove("/tmp/test_speed/test")
utils.run("umount /tmp/test_speed")
def get_block_size(session, block_cmd, block_pattern):
"""
Get block size inside guest.
"""
output = session.cmd_output(block_cmd)
block_size = re.findall(block_pattern, output)
if block_size:
if not re.search("[a-zA-Z]", block_size[0]):
return int(block_size[0])
else:
return float(utils_misc.normalize_data_size(block_size[0], order_magnitude="B"))
else:
raise error.TestError(
"Can not find the block size for the" " deivce. The output of command" " is: %s" % output
)
def get_block_size(session, block_cmd, block_pattern):
"""
Get block size inside guest.
"""
output = session.cmd_output(block_cmd)
block_size = re.findall(block_pattern, output)
if block_size:
if not re.search("[a-zA-Z]", block_size[0]):
return int(block_size[0])
else:
return float(utils_misc.normalize_data_size(block_size[0],
order_magnitude="B"))
else:
raise error.TestError("Can not find the block size for the"
" deivce. The output of command"
" is: %s" % output)
def test_normalize_data_size(self):
n1 = utils_misc.normalize_data_size("12M")
n2 = utils_misc.normalize_data_size("1024M", "G")
n3 = utils_misc.normalize_data_size("1024M", "T")
n4 = utils_misc.normalize_data_size("1000M", "G", 1000)
n5 = utils_misc.normalize_data_size("1T", "G", 1000)
n6 = utils_misc.normalize_data_size("1T", "M")
self.assertEqual(n1, "12.0")
self.assertEqual(n2, "1.0")
self.assertEqual(n3, "0.0009765625")
self.assertEqual(n4, "1.0")
self.assertEqual(n5, "1000.0")
self.assertEqual(n6, "1048576.0")
def test_normalize_data_size(self):
n1 = utils_misc.normalize_data_size("12M")
n2 = utils_misc.normalize_data_size("1024M", "G")
n3 = utils_misc.normalize_data_size("1024M", "T")
n4 = utils_misc.normalize_data_size("1000M", "G", 1000)
n5 = utils_misc.normalize_data_size("1T", "G", 1000)
n6 = utils_misc.normalize_data_size("1T", "M")
self.assertEqual(n1, "12.0")
self.assertEqual(n2, "1.0")
self.assertEqual(n3, "0.0009765625")
self.assertEqual(n4, "1.0")
self.assertEqual(n5, "1000.0")
self.assertEqual(n6, "1048576.0")
def get_nodes_size(size_type='MemTotal', session=None):
"""
Get the node size of each node in host/guest, descending sort with size
:param size_type: the type of the node size
:param session: ShellSession object
:return: a list of tuple include node id and node size(M)
:rtype: list
"""
numa_info = NumaInfo(session=session)
nodes_size = {}
numa_nodes = numa_info.online_nodes
for node in numa_nodes:
node_size = numa_info.online_nodes_meminfo[node][size_type]
nodes_size[node] = float(normalize_data_size('%s KB' % node_size))
nodes_size = sorted(nodes_size.items(),
key=lambda item: item[1],
reverse=True)
return nodes_size
def check_memory_in_procfs(test, params, vm):
"""
Check memory info in procfs
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param vm: VM object
"""
qemu_pid = vm.get_pid()
policy = params['policy_mem']
if policy == 'preferred':
policy = 'prefer'
for mem_dev in params['mem_devs'].split():
memdev_params = params.object_params(mem_dev)
mem_size = memdev_params['size']
mem_size = int(float(utils_misc.normalize_data_size(mem_size, "K")))
smaps = process.system_output("grep -1 %d /proc/%d/smaps"
% (mem_size, qemu_pid))
smaps = decode_to_text(smaps).strip()
mem_path = memdev_params.get("mem-path")
if mem_path and (mem_path not in smaps):
test.fail("memdev = %s: mem-path '%s' is not in smaps '%s'!"
% (mem_dev, mem_path, smaps))
mem_start = smaps.split('-')[0]
numa_maps = process.system_output("grep %s /proc/%d/numa_maps"
% (mem_start, qemu_pid))
numa_maps = decode_to_text(numa_maps).strip()
if mem_path and (mem_path not in numa_maps):
test.fail("memdev = %s: mem-path '%s' is not in numa_maps '%s'!"
% (mem_dev, mem_path, numa_maps))
policy_numa = numa_maps.split()[1].split(':')
if policy != policy_numa[0]:
test.fail("memdev = %s:"
" 'policy' in numa_maps is '%s', but not '%s'!"
% (mem_dev, policy_numa[0], policy))
elif (policy != 'default'):
host_node = memdev_params['host-nodes']
if (policy_numa[1] != host_node):
test.fail("memdev = %s:"
" 'host-nodes' in numa_maps is '%s', but not '%s'!"
% (mem_dev, policy_numa[1], host_node))
def run(test, params, env):
"""
Simple test to check if NUMA options are being parsed properly
This _does not_ test if NUMA information is being properly exposed to the
guest.
"""
dbg("starting numa_opts test...")
vm = env.get_vm(params["main_vm"])
numa = vm.monitors[0].info_numa()
dbg("info numa reply: %r", numa)
numa_nodes = params.get("numa_nodes")
if numa_nodes:
numa_nodes = int(params.get("numa_nodes"))
if len(numa) != numa_nodes:
test.fail("Wrong number of numa nodes: %d. Expected: %d" %
(len(numa), numa_nodes))
for nodenr, node in enumerate(numa):
mdev = params.get("numa_memdev_node%d" % (nodenr))
if mdev:
mdev = mdev.split('-')[1]
size = float(normalize_data_size(params.get("size_%s" % mdev)))
else:
size = params.get_numeric("mem")
if size != numa[nodenr][0]:
test.fail("Wrong size of numa node %d: %d. Expected: %d" %
(nodenr, numa[nodenr][0], size))
cpus = params.get("numa_cpus_node%d" % (nodenr))
if cpus is not None:
cpus = set([int(v) for v in cpus.split(",")])
if cpus != numa[nodenr][1]:
test.fail("Wrong CPU set on numa node %d: %s. Expected: %s" %
(nodenr, numa[nodenr][1], cpus))
def run(test, params, env):
"""
Memory balloon with thp
1. Boot up a guest with balloon support, record memory fragement
2. Make fragement in guest with tmpfs
3. check the memory fragement with proc system, should increase
4. Do memory balloon the memory size ballooned should be a legal value
5. Check the memory fragement with proc system, should decrease
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
def get_buddy_info(sleep=False):
"""Get buddy info"""
if sleep:
time.sleep(10)
buddy_info = utils_memory.get_buddy_info('0', session=session)['0']
logging.info('Checked buddy info, value is %s', buddy_info)
return buddy_info
fragement_dir = params['fragement_dir']
vm = env.get_vm(params["main_vm"])
session = vm.wait_for_login()
buddy_info_bf = get_buddy_info()
logging.info('Making fragement on guest...')
session.cmd_output_safe(params['cmd_make_fragement'], timeout=600)
for i in range(1, 10, 2):
session.cmd_output_safe('rm -f %s/*%s' % (fragement_dir, i))
funcatexit.register(env, params['type'], clean_env, session, fragement_dir)
buddy_info_af_fragement = get_buddy_info(sleep=True)
if buddy_info_bf >= buddy_info_af_fragement:
test.fail('Buddy info should increase.')
mem = int(float(utils_misc.normalize_data_size("%sM" % params["mem"])))
vm.balloon(mem - 1024)
buddy_info_af_balloon = get_buddy_info(sleep=True)
if buddy_info_af_balloon >= buddy_info_af_fragement:
test.fail('Buddy info should decrease.')
def get_memory_boundary(self, balloon_type='evict'):
"""
Get the legal memory boundary for balloon operation.
:param balloon_type: evict or enlarge
:type balloon_type: string
:return: min and max size of the memory
:rtype: tuple
"""
max_size = self.ori_mem
min_size = self.params.get("minmem", "512M")
min_size = int(float(utils_misc.normalize_data_size(min_size)))
if balloon_type == 'enlarge':
min_size = self.current_mmem
else:
vm_total = self.get_memory_status()
vm_mem_free = self.get_free_mem()
# leave 16M buffer to ensure os keep working when do
# evict balloon device.
used_size = vm_total - vm_mem_free + 16
min_size = max(used_size, min_size)
return min_size, max_size
def get_memory_boundary(self, balloon_type="evict"):
"""
Get the legal memory boundary for balloon operation.
:param balloon_type: evict or enlarge
:type balloon_type: string
:return: min and max size of the memory
:rtype: tuple
"""
max_size = self.ori_mem
min_size = self.params.get("minmem", "512M")
min_size = int(float(utils_misc.normalize_data_size(min_size)))
if balloon_type == "enlarge":
min_size = self.current_mmem
else:
vm_total = self.get_memory_status()
vm_mem_free = self.get_free_mem()
# leave 16M buffer to ensure os keep working when do
# evict balloon device.
used_size = vm_total - vm_mem_free + 16
min_size = max(used_size, min_size)
return min_size, max_size
def edit_memory(source):
"""
Modify vm's maximum and current memory(unit and value).
:param source: virsh edit's option.
:return: True if edit succeeded,False if edit failed.
"""
mem_unit = params.get("mem_unit", "K")
mem_value = params.get("mem_value", "1048576")
mem_delta = params.get("mem_delta", 1000)
edit_cmd = []
del_cmd = r":g/currentMemory/d"
edit_cmd.append(del_cmd)
update_cmd = r":%s/<memory unit='KiB'>[0-9]*<\/memory>/<memory unit='"
update_cmd += mem_unit + "'>" + mem_value + r"<\/memory>"
edit_cmd.append(update_cmd)
try:
expected_mem = int(
utils_misc.normalize_data_size(mem_value + mem_unit,
'K').split('.')[0])
except ValueError:
logging.error("Fail to translate %s to KiB", mem_value + mem_unit)
return False
logging.debug("Expected max memory is %s", expected_mem)
status = libvirt.exec_virsh_edit(source, edit_cmd)
try:
if status:
# Restart vm to check memory value
virsh.destroy(vm_name)
virsh.start(vm_name)
new_mem = vm.get_max_mem()
if new_mem - expected_mem > int(mem_delta):
logging.error("New max memory %s is not excepted", new_mem)
return False
except Exception as e:
logging.error("Error occurred when check domain memory: %s", e)
return False
return status
def edit_memory(source):
"""
Modify vm's maximum and current memory(unit and value).
:param source: virsh edit's option.
:return: True if edit successed,False if edit failed.
"""
mem_unit = params.get("mem_unit", "K")
mem_value = params.get("mem_value", "1048576")
mem_delta = params.get("mem_delta", 1000)
edit_cmd = []
del_cmd = r":g/currentMemory/d"
edit_cmd.append(del_cmd)
update_cmd = r":%s/<memory unit='KiB'>[0-9]*<\/memory>/<memory unit='"
update_cmd += mem_unit + "'>" + mem_value + r"<\/memory>"
edit_cmd.append(update_cmd)
try:
expected_mem = int(utils_misc.normalize_data_size(
mem_value + mem_unit, 'K').split('.')[0])
except ValueError:
logging.error("Fail to translate %s to KiB", mem_value + mem_unit)
return False
logging.debug("Expected max memory is %s", expected_mem)
status = libvirt.exec_virsh_edit(source, edit_cmd)
try:
if status:
# Restart vm to check memory value
virsh.destroy(vm_name)
virsh.start(vm_name)
new_mem = vm.get_max_mem()
if new_mem - expected_mem > int(mem_delta):
logging.error("New max memory %s is not excepted", new_mem)
return False
except Exception as e:
logging.error("Error occured when check domain memory: %s", e)
return False
return status
def create_luks_vol(vol_name, sec_uuid, params):
"""
Create a luks volume
:param vol_name: the name of the volume
:param sec_uuid: secret's uuid to be used for luks encryption
:param params: detailed params to create volume
"""
pool_name = params.get("pool_name")
extra_option = params.get("extra_option", "")
unprivileged_user = params.get('unprivileged_user')
uri = params.get("virsh_uri")
vol_arg = {}
for key in list(params.keys()):
if (key.startswith('vol_') and not key.startswith('vol_new')):
if key[4:] in ['capacity', 'allocation']:
vol_arg[key[4:]] = int(float(utils_misc.normalize_data_size(params[key],
"B", 1024)))
elif key[4:] in ['owner', 'group']:
vol_arg[key[4:]] = int(params[key])
else:
vol_arg[key[4:]] = params[key]
vol_arg['name'] = vol_name
volxml = libvirt_xml.VolXML()
newvol = volxml.new_vol(**vol_arg)
luks_encryption_params = {}
luks_encryption_params.update({"format": "luks"})
luks_encryption_params.update({"secret": {"type": "passphrase",
"uuid": sec_uuid}})
newvol.encryption = volxml.new_encryption(**luks_encryption_params)
vol_xml = newvol['xml']
if params.get('setup_libvirt_polkit') == 'yes':
process.run("chmod 666 %s" % vol_xml, ignore_status=True,
shell=True)
logging.debug("Create volume from XML: %s" % newvol.xmltreefile)
cmd_result = virsh.vol_create(pool_name, vol_xml, extra_option,
unprivileged_user=unprivileged_user, uri=uri,
ignore_status=True, debug=True)
def run(test, params, env):
"""
1. Create a pool
2. Create n number of volumes(vol-create-as)
3. Check the volume details from the following commands
vol-info
vol-key
vol-list
vol-name
vol-path
vol-pool
qemu-img info
4. Delete the volume and check in vol-list
5. Repeat the steps for number of volumes given
6. Delete the pool and target
TODO: Handle negative testcases
"""
def delete_volume(expected_vol):
"""
Deletes Volume
"""
pool_name = expected_vol['pool_name']
vol_name = expected_vol['name']
pv = libvirt_storage.PoolVolume(pool_name)
if not pv.delete_volume(vol_name):
raise error.TestFail("Delete volume failed." % vol_name)
else:
logging.debug("Volume: %s successfully deleted on pool: %s",
vol_name, pool_name)
def get_vol_list(pool_name, vol_name):
"""
Parse the volume list
"""
output = virsh.vol_list(pool_name, "--details")
rg = re.compile(
r'^(\S+)\s+(\S+)\s+(\S+)\s+(\d+.\d+\s\S+)\s+(\d+.\d+.*)')
vol = {}
vols = []
volume_detail = None
for line in output.stdout.splitlines():
match = re.search(rg, line.lstrip())
if match is not None:
vol['name'] = match.group(1)
vol['path'] = match.group(2)
vol['type'] = match.group(3)
vol['capacity'] = match.group(4)
vol['allocation'] = match.group(5)
vols.append(vol)
vol = {}
for volume in vols:
if volume['name'] == vol_name:
volume_detail = volume
return volume_detail
def norm_capacity(capacity):
"""
Normalize the capacity values to bytes
"""
# Normaize all values to bytes
norm_capacity = {}
des = {'B': 'B', 'bytes': 'B', 'b': 'B', 'kib': 'K',
'KiB': 'K', 'K': 'K', 'k': 'K', 'KB': 'K',
'mib': 'M', 'MiB': 'M', 'M': 'M', 'm': 'M',
'MB': 'M', 'gib': 'G', 'GiB': 'G', 'G': 'G',
'g': 'G', 'GB': 'G', 'Gb': 'G', 'tib': 'T',
'TiB': 'T', 'TB': 'T', 'T': 'T', 't': 'T'
}
val = {'B': 1,
'K': 1024,
'M': 1048576,
'G': 1073741824,
'T': 1099511627776
}
reg_list = re.compile(r'(\S+)\s(\S+)')
match_list = re.search(reg_list, capacity['list'])
if match_list is not None:
mem_value = float(match_list.group(1))
norm = val[des[match_list.group(2)]]
norm_capacity['list'] = int(mem_value * norm)
else:
raise error.TestFail("Error in parsing capacity value in"
" virsh vol-list")
match_info = re.search(reg_list, capacity['info'])
if match_info is not None:
mem_value = float(match_info.group(1))
norm = val[des[match_list.group(2)]]
norm_capacity['info'] = int(mem_value * norm)
else:
raise error.TestFail("Error in parsing capacity value "
"in virsh vol-info")
norm_capacity['qemu_img'] = capacity['qemu_img']
norm_capacity['xml'] = int(capacity['xml'])
return norm_capacity
def check_vol(expected, avail=True):
"""
Checks the expected volume details with actual volume details from
vol-dumpxml
vol-list
vol-info
vol-key
vol-path
qemu-img info
"""
error_count = 0
volume_xml = {}
pv = libvirt_storage.PoolVolume(expected['pool_name'])
pool_exists = pv.volume_exists(expected['name'])
if pool_exists:
if not avail:
error_count += 1
logging.error("Expect volume %s not exists but find it",
expected['name'])
return error_count
else:
if avail:
error_count += 1
logging.error("Expect volume %s exists but not find it",
expected['name'])
return error_count
else:
logging.info("Volume %s checked successfully for deletion",
expected['name'])
return error_count
actual_list = get_vol_list(expected['pool_name'], expected['name'])
actual_info = pv.volume_info(expected['name'])
# Get values from vol-dumpxml
volume_xml = vol_xml.VolXML.get_vol_details_by_name(expected['name'],
expected['pool_name'])
# Check against virsh vol-key
vol_key = virsh.vol_key(expected['name'], expected['pool_name'])
if vol_key.stdout.strip() != volume_xml['key']:
logging.error("Volume key is mismatch \n%s"
"Key from xml: %s\nKey from command: %s",
expected['name'], volume_xml['key'], vol_key)
error_count += 1
else:
logging.debug("virsh vol-key for volume: %s successfully"
" checked against vol-dumpxml", expected['name'])
# Check against virsh vol-name
get_vol_name = virsh.vol_name(expected['path'])
if get_vol_name.stdout.strip() != expected['name']:
logging.error("Volume name mismatch\n"
"Expected name: %s\nOutput of vol-name: %s",
expected['name'], get_vol_name)
# Check against virsh vol-path
vol_path = virsh.vol_path(expected['name'], expected['pool_name'])
if expected['path'] != vol_path.stdout.strip():
logging.error("Volume path mismatch for volume: %s\n"
"Expected path: %s\nOutput of vol-path: %s\n",
expected['name'],
expected['path'], vol_path)
error_count += 1
else:
logging.debug("virsh vol-path for volume: %s successfully checked"
" against created volume path", expected['name'])
# Check path against virsh vol-list
if expected['path'] != actual_list['path']:
logging.error("Volume path mismatch for volume:%s\n"
"Expected Path: %s\nPath from virsh vol-list: %s",
expected['name'], expected['path'],
actual_list['path'])
error_count += 1
else:
logging.debug("Path of volume: %s from virsh vol-list "
"successfully checked against created "
"volume path", expected['name'])
# Check path against virsh vol-dumpxml
if expected['path'] != volume_xml['path']:
logging.error("Volume path mismatch for volume: %s\n"
"Expected Path: %s\nPath from virsh vol-dumpxml: %s",
expected['name'], expected['path'], volume_xml['path'])
error_count += 1
else:
logging.debug("Path of volume: %s from virsh vol-dumpxml "
"successfully checked against created volume path",
expected['name'])
# Check type against virsh vol-list
if expected['type'] != actual_list['type']:
logging.error("Volume type mismatch for volume: %s\n"
"Expected Type: %s\n Type from vol-list: %s",
expected['name'], expected['type'],
actual_list['type'])
error_count += 1
else:
logging.debug("Type of volume: %s from virsh vol-list "
"successfully checked against the created "
"volume type", expected['name'])
# Check type against virsh vol-info
if expected['type'] != actual_info['Type']:
logging.error("Volume type mismatch for volume: %s\n"
"Expected Type: %s\n Type from vol-info: %s",
expected['name'], expected['type'],
actual_info['Type'])
error_count += 1
else:
logging.debug("Type of volume: %s from virsh vol-info successfully"
" checked against the created volume type",
expected['name'])
# Check name against virsh vol-info
if expected['name'] != actual_info['Name']:
logging.error("Volume name mismatch for volume: %s\n"
"Expected name: %s\n Name from vol-info: %s",
expected['name'],
expected['name'], actual_info['Name'])
error_count += 1
else:
logging.debug("Name of volume: %s from virsh vol-info successfully"
" checked against the created volume name",
expected['name'])
# Check format from against qemu-img info
img_info = utils_misc.get_image_info(expected['path'])
if expected['format']:
if expected['format'] != img_info['format']:
logging.error("Volume format mismatch for volume: %s\n"
"Expected format: %s\n"
"Format from qemu-img info: %s",
expected['name'], expected['format'],
img_info['format'])
error_count += 1
else:
logging.debug("Format of volume: %s from qemu-img info "
"checked successfully against the created "
"volume format", expected['name'])
# Check format against vol-dumpxml
if expected['format']:
if expected['format'] != volume_xml['format']:
logging.error("Volume format mismatch for volume: %s\n"
"Expected format: %s\n"
"Format from vol-dumpxml: %s",
expected['name'], expected['format'],
volume_xml['format'])
error_count += 1
else:
logging.debug("Format of volume: %s from virsh vol-dumpxml "
"checked successfully against the created"
" volume format", expected['name'])
# Check pool name against vol-pool
vol_pool = virsh.vol_pool(expected['path'])
if expected['pool_name'] != vol_pool.stdout.strip():
logging.error("Pool name mismatch for volume: %s against"
"virsh vol-pool", expected['name'])
error_count += 1
else:
logging.debug("Pool name of volume: %s checked successfully"
" against the virsh vol-pool", expected['name'])
norm_cap = {}
capacity = {}
capacity['list'] = actual_list['capacity']
capacity['info'] = actual_info['Capacity']
capacity['xml'] = volume_xml['capacity']
capacity['qemu_img'] = img_info['vsize']
norm_cap = norm_capacity(capacity)
delta_size = params.get('delta_size', "1024")
if abs(expected['capacity'] - norm_cap['list']) > delta_size:
logging.error("Capacity mismatch for volume: %s against virsh"
" vol-list\nExpected: %s\nActual: %s",
expected['name'], expected['capacity'],
norm_cap['list'])
error_count += 1
else:
logging.debug("Capacity value checked successfully against"
" virsh vol-list for volume %s", expected['name'])
if abs(expected['capacity'] - norm_cap['info']) > delta_size:
logging.error("Capacity mismatch for volume: %s against virsh"
" vol-info\nExpected: %s\nActual: %s",
expected['name'], expected['capacity'],
norm_cap['info'])
error_count += 1
else:
logging.debug("Capacity value checked successfully against"
" virsh vol-info for volume %s", expected['name'])
if abs(expected['capacity'] - norm_cap['xml']) > delta_size:
logging.error("Capacity mismatch for volume: %s against virsh"
" vol-dumpxml\nExpected: %s\nActual: %s",
expected['name'], expected['capacity'],
norm_cap['xml'])
error_count += 1
else:
logging.debug("Capacity value checked successfully against"
" virsh vol-dumpxml for volume: %s",
expected['name'])
if abs(expected['capacity'] - norm_cap['qemu_img']) > delta_size:
logging.error("Capacity mismatch for volume: %s against "
"qemu-img info\nExpected: %s\nActual: %s",
expected['name'], expected['capacity'],
norm_cap['qemu_img'])
error_count += 1
else:
logging.debug("Capacity value checked successfully against"
" qemu-img info for volume: %s",
expected['name'])
return error_count
# Initialize the variables
pool_name = params.get("pool_name")
pool_type = params.get("pool_type")
pool_target = params.get("pool_target")
if os.path.dirname(pool_target) is "":
pool_target = os.path.join(test.tmpdir, pool_target)
vol_name = params.get("volume_name")
vol_number = int(params.get("number_of_volumes", "2"))
capacity = params.get("volume_size", "1048576")
allocation = params.get("volume_allocation", "1048576")
vol_format = params.get("volume_format")
source_name = params.get("gluster_source_name", "gluster-vol1")
source_path = params.get("gluster_source_path", "/")
emulated_image = params.get("emulated_image")
emulated_image_size = params.get("emulated_image_size")
try:
str_capa = utils_misc.normalize_data_size(capacity, "B")
int_capa = int(str(str_capa).split('.')[0])
except ValueError:
raise error.TestError("Translate size %s to 'B' failed" % capacity)
try:
str_capa = utils_misc.normalize_data_size(allocation, "B")
int_allo = int(str(str_capa).split('.')[0])
except ValueError:
raise error.TestError("Translate size %s to 'B' failed" % allocation)
expected_vol = {}
vol_type = 'file'
if pool_type in ['disk', 'logical']:
vol_type = 'block'
if pool_type == 'gluster':
vol_type = 'network'
logging.debug("Debug:\npool_name:%s\npool_type:%s\npool_target:%s\n"
"vol_name:%s\nvol_number:%s\ncapacity:%s\nallocation:%s\n"
"vol_format:%s", pool_name, pool_type, pool_target,
vol_name, vol_number, capacity, allocation, vol_format)
libv_pvt = utlv.PoolVolumeTest(test, params)
# Run Testcase
total_err_count = 0
try:
# Create a new pool
libv_pvt.pre_pool(pool_name=pool_name,
pool_type=pool_type,
pool_target=pool_target,
emulated_image=emulated_image,
image_size=emulated_image_size,
source_name=source_name,
source_path=source_path)
for i in range(vol_number):
volume_name = "%s_%d" % (vol_name, i)
expected_vol['pool_name'] = pool_name
expected_vol['pool_type'] = pool_type
expected_vol['pool_target'] = pool_target
expected_vol['capacity'] = int_capa
expected_vol['allocation'] = int_allo
expected_vol['format'] = vol_format
expected_vol['name'] = volume_name
expected_vol['type'] = vol_type
# Creates volume
if pool_type != "gluster":
expected_vol['path'] = pool_target + '/' + volume_name
libv_pvt.pre_vol(volume_name, vol_format, capacity,
allocation, pool_name)
else:
ip_addr = utlv.get_host_ipv4_addr()
expected_vol['path'] = "gluster://%s/%s/%s" % (ip_addr,
source_name,
volume_name)
utils.run("qemu-img create -f %s %s %s" % (vol_format,
expected_vol['path'],
capacity))
virsh.pool_refresh(pool_name)
# Check volumes
total_err_count += check_vol(expected_vol)
# Delete volume and check for results
delete_volume(expected_vol)
total_err_count += check_vol(expected_vol, False)
if total_err_count > 0:
raise error.TestFail("Test case failed due to previous errors.\n"
"Check for error logs")
finally:
# Clean up
try:
libv_pvt.cleanup_pool(pool_name, pool_type, pool_target,
emulated_image, source_name)
except error.TestFail, detail:
logging.error(str(detail))
def get_expect_info(new_capacity, vol_path, resize_option=None):
"""
Get the expect volume capacity and allocation size, for comparation
after volume resize. As virsh vol-info return imprecise values, so we
need get volume info from qemu side. The process is:
1) Transform new capacity size to bytes.
2) Get image info by qemu-img info(byte size).
3) Calculate the expect info according to volume resie option.
:param new_capacity: New capacity for the vol, as scaled integer
:param vol_path: Absolute path of volume
:return: Expect volume capacity and allocation
"""
if new_capacity.isdigit():
# Default bytes
new_capacity = new_capacity + "b"
suffixes_list1 = ['B', 'K', 'KIB', 'M', 'MIB', 'G', 'GIB', 'T', 'TIB']
suffixes_list2 = ['KB', 'MB', 'GB', 'TB']
expect_info = {}
suffix = "B"
factor = "1024"
try:
suffix = re.findall(r"[\s\d](\D+)", new_capacity, re.I)[-1].strip()
except IndexError:
raise error.TestError("Incorrect size format %s." % new_capacity)
if suffix in suffixes_list1:
factor = "1024"
elif suffix in suffixes_list2:
factor = "1000"
else:
raise error.TestError("Unsupport size unit '%s'." % suffix)
try:
# Transform the size to bytes
new_size = utils_misc.normalize_data_size(new_capacity, "B", factor)
# Get image info
img_info = utils_misc.get_image_info(vol_path)
# Init expect_info
expect_info['Capacity'] = img_info['vsize']
expect_info['Allocation'] = img_info['dsize']
# Deal with resize options
if not resize_option:
expect_info['Capacity'] = int(float(new_size))
return expect_info
support_options = ["--allocate", "--delta", "--shrink"]
find_delt = False
find_allo = False
for option in resize_option.split():
logging.debug("Find '%s' in volume resize option", option)
if option not in support_options:
# Give an invalid option is acceptable in the test, so just
# output debug log
logging.debug("Invalid resize option: %s.", option)
return expect_info
if option == "--shrink":
# vol-resize --shrink has a bug now, so output error
logging.error("Shrink volume not support in this test.")
return expect_info
if option == "--allocate":
find_allo = True
logging.debug("Allocate the new capacity, rather than "
"leaving it sparse.")
if option == "--delta":
find_delt = True
logging.debug("Use capacity as a delta to current size, "
"rather than the new size")
if find_allo and find_delt:
expect_info['Capacity'] += int(float(new_size))
expect_info['Allocation'] += int(float(new_size))
elif find_allo:
expect_info['Capacity'] = int(float(new_size))
expect_info['Allocation'] += int(float(new_size)) - img_info['vsize']
elif find_delt:
expect_info['Capacity'] += int(float(new_size))
else:
pass
return expect_info
except (IndexError, ValueError), detail:
raise error.TestError("Fail to get expect volume info:\n%s" % detail)
def result_sum(topdir, params, guest_ver, resultsdir, test):
case_type = params.get("test")
unit_std = params.get("unit_std", "M")
no_table_list = params.get("no_table_list", "").split()
ignore_cases = params.get("ignore_cases", "").split()
repeatn = ""
if "repeat" in test.outputdir:
repeatn = re.findall("repeat\d+", test.outputdir)[0]
category_key = re.split("/", test.outputdir)[-1]
category_key = re.split(case_type, category_key)[0]
category_key = re.sub("\.repeat\d+", "", category_key)
kvm_ver = utils.system_output(params.get('ver_cmd', "rpm -q qemu-kvm"))
host_ver = os.uname()[2]
test.write_test_keyval({'kvm-userspace-ver': kvm_ver})
test.write_test_keyval({'host-kernel-ver': host_ver})
test.write_test_keyval({'guest-kernel-ver': guest_ver})
# Find the results files
results_files = {}
file_list = ['guest_result', 'guest_monitor_result.*sum',
'host_monitor_result.*sum']
if params.get("file_list"):
file_list = params.get("file_list").split()
for files in os.walk(topdir):
if files[2]:
for file in files[2]:
jump_flag = False
for ignore_case in ignore_cases:
if ignore_case in files[0]:
jump_flag = True
if jump_flag:
continue
file_dir_norpt = re.sub("\.repeat\d+", "", files[0])
if (repeatn in files[0]
and category_key in file_dir_norpt
and case_type in files[0]):
for i, pattern in enumerate(file_list):
if re.findall(pattern, file):
prefix = re.findall("%s\.[\d\w_\.]+" % case_type,
file_dir_norpt)[0]
prefix = re.sub("\.|_", "--", prefix)
if prefix not in results_files.keys():
results_files[prefix] = []
tmp = []
for j in range(len(file_list)):
tmp.append(None)
results_files[prefix] = tmp
tmp_file = utils_misc.get_path(files[0], file)
results_files[prefix][i] = tmp_file
# Start to read results from results file and monitor file
results_matrix = {}
no_table_results = {}
thread_tag = params.get("thread_tag", "thread")
order_list = []
for prefix in results_files:
marks = params.get("marks", "").split()
case_infos = prefix.split("--")
case_type = case_infos[0]
threads = ""
refresh_order_list = True
prefix_perf = prefix
if case_type == "ffsb":
category = "-".join(case_infos[:-1])
threads = case_infos[-1]
elif case_type == "qcow2perf":
refresh_order_list = False
if len(case_infos) > 2:
category = "-".join(case_infos[:-2])
thread_tag = case_infos[-2]
threads = " "
marks[0] = re.sub("TIME", case_infos[-1], marks[0])
else:
category = case_infos[-1]
marks[0] = re.sub("TIME", case_infos[-1], marks[0])
prefix_perf = "--".join(case_infos[:-1])
else:
category = "-".join(case_infos)
if refresh_order_list:
order_list = []
if (category not in results_matrix.keys()
and category not in no_table_list):
results_matrix[category] = {}
if threads:
if threads not in results_matrix[category].keys():
results_matrix[category][threads] = {}
results_matrix["thread_tag"] = thread_tag
tmp_dic = results_matrix[category][threads]
elif category not in no_table_list:
tmp_dic = results_matrix[category]
result_context_file = open(results_files[prefix][0], 'r')
result_context = result_context_file.read()
result_context_file.close()
for mark in marks:
mark_tag, mark_key = mark.split(":")
datas = re.findall(mark_key, result_context)
if isinstance(datas[0], tuple):
data = time_ana(datas[0])
else:
tmp_data = 0.0
for data in datas:
if re.findall("[bmkg]", data, re.I):
data = utils_misc.normalize_data_size(data, unit_std)
tmp_data += float(data)
data = str(tmp_data)
if data:
if mark_tag in no_table_list:
no_table_results[mark_tag] = utils_misc.aton(data)
perf_value = no_table_results[mark_tag]
else:
tmp_dic[mark_tag] = utils_misc.aton(data)
perf_value = tmp_dic[mark_tag]
else:
raise error.TestError("Can not get the right data from result."
"Please check the debug file.")
if mark_tag not in no_table_list and mark_tag not in order_list:
order_list.append(mark_tag)
test.write_perf_keyval({'%s-%s' % (prefix_perf, mark_tag):
perf_value})
# start analyze the mpstat results
if params.get('mpstat') == "yes":
guest_cpu_infos = mpstat_ana(results_files[prefix][1])
for vcpu in guest_cpu_infos:
if vcpu != "all":
tmp_dic[vcpu] = float(guest_cpu_infos[vcpu])
order_list.append(vcpu)
host_cpu_infos = mpstat_ana(results_files[prefix][2])
tmp_dic["Hostcpu"] = float(host_cpu_infos["all"])
order_list.append("Hostcpu")
# Add some special key for cases
if case_type == "ffsb":
tmp_dic["MBps_per_Hostcpu"] = (tmp_dic["Thro-MBps"] /
tmp_dic["Hostcpu"])
order_list.append("MBps_per_Hostcpu")
elif case_type == "iozone":
sum_kbps = 0
for mark in marks:
mark_tag, _ = mark.split(":")
sum_kbps += tmp_dic[mark_tag]
tmp_dic["SUMKbps_per_Hostcpu"] = sum_kbps / tmp_dic["Hostcpu"]
order_list.append("SUMKbps_per_Hostcpu")
sum_marks = params.get("sum_marks", "").split()
sum_matrix = {}
order_line = ""
if results_matrix.get("thread_tag"):
headline = "%20s|" % results_matrix["thread_tag"]
results_matrix.pop("thread_tag")
else:
headline = ""
for index, tag in enumerate(order_list):
headline += "%s|" % format_result(tag)
order_line += "DATA%d|" % index
headline = headline.rstrip("|")
order_line = order_line.rstrip("|")
result_path = utils_misc.get_path(resultsdir,
"%s-result.RHS" % case_type)
if os.path.isfile(result_path):
result_file = open(result_path, "r+")
else:
result_file = open(result_path, "w")
result_file.write("### kvm-userspace-version : %s\n" % kvm_ver)
result_file.write("### kvm-version : %s\n" % host_ver)
result_file.write("### guest-kernel-version :%s\n" % guest_ver)
test.write_test_keyval({'category': headline})
result_file.write("Category:ALL\n")
matrix_order = params.get("matrix_order", "").split()
if not matrix_order:
matrix_order = results_matrix.keys()
matrix_order.sort()
for category in matrix_order:
out_loop_line = order_line
result_file.write("%s\n" % category)
line = ""
write_out_loop = True
result_file.write("%s\n" % headline)
for item in results_matrix[category]:
if isinstance(results_matrix[category][item], dict):
tmp_dic = results_matrix[category][item]
line = "%s|" % format_result(item)
for tag in order_list:
line += "%s|" % format_result(tmp_dic[tag])
if tag in sum_marks:
sum_matrix = get_sum_result(sum_matrix, tmp_dic[tag],
tag)
result_file.write("%s\n" % line.rstrip("|"))
write_out_loop = False
else:
#line += "%s|" % format_result(results_matrix[category][item])
re_data = "DATA%s" % order_list.index(item)
out_loop_line = re.sub(re_data,
format_result(
results_matrix[category][item]),
out_loop_line)
if tag in sum_marks:
sum_matrix = get_sum_result(sum_matrix, tmp_dic[tag],
tag)
if write_out_loop:
result_file.write("%s\n" % out_loop_line)
if sum_matrix:
if case_type == "ffsb":
sum_matrix["MBps_per_Hostcpu"] = (sum_matrix["Thro-MBps"] /
sum_matrix["Hostcpu"])
sum_marks.append("MBps_per_Hostcpu")
result_file.write("Category:SUM\n")
headline = ""
line = ""
if len(sum_matrix) < 4:
for i in range(4 - len(sum_matrix)):
headline += "%20s|" % "None"
line += "%20d|" % 0
for tag in sum_marks:
headline += "%20s|" % tag
line += "%s|" % format_result(sum_matrix[tag])
result_file.write("%s\n" % headline.rstrip("|"))
result_file.write("%s\n" % line.rstrip("|"))
if no_table_results:
no_table_order = params.get("no_table_order", "").split()
if not no_table_order:
no_table_order = no_table_results.keys()
no_table_order.sort()
for item in no_table_order:
result_file.write("%s: %s\n" % (item, no_table_results[item]))
result_file.close()
def run(test, params, env):
"""
KVM nfs performance test:
1) boot guest over virtio driver.
2) mount nfs server in guest with tcp protocol.
3) test write performance in guest using dd commands.
4) test read performance in guest using dd commands.
Note: This test is used for performance benchmark test,
not for the functional test usage. The result of
this test depends on the regression.py tool,
though you can run it during function testing,
you may not get any possible error report from
this script directly.
:param test: kvm test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment
"""
def _do_clean_up(func, *args):
try:
if args:
func(*args)
else:
func()
except Exception as e:
logging.warn("Failed to execute function '%s'."
" error message:\n%s", func.__name__, e)
def _clean_up(step_cnt):
error_context.context("Clean up", logging.info)
if step_cnt >= STEP_5:
# remove test file.
cmd = "rm -f %s" % " ".join(test_file_list)
_do_clean_up(session.cmd, cmd)
if step_cnt >= STEP_4:
# umount nfs partition.
cmd = "umount %s" % mnt_point
_do_clean_up(session.cmd, cmd)
if step_cnt >= STEP_3:
# remove mount ponit directory.
cmd = "rm -rf %s" % mnt_point
_do_clean_up(session.cmd, cmd)
if step_cnt >= STEP_2:
# close result file.
_do_clean_up(result_file.close)
if step_cnt >= STEP_1:
# close session.
_do_clean_up(session.close)
def _do_write_test(blk_size, test_file):
# Clean up caches
session.cmd("echo 3 >/proc/sys/vm/drop_caches")
error_context.context("test %s size block write performance in guest"
" using dd commands" % blk_size, logging.info)
dd_cmd = "dd"
dd_cmd += " if=/dev/zero"
dd_cmd += " of=%s" % test_file
dd_cmd += " bs=%s" % blk_size
dd_cmd += " oflag=direct"
dd_cmd += " count=10000"
try:
out = session.cmd_output(dd_cmd, timeout=test_timeout)
except Exception:
_clean_up(STEP_4)
raise
return out
def _do_read_test(blk_size, test_file):
# Clean up caches
session.cmd("echo 3 >/proc/sys/vm/drop_caches")
error_context.context("test %s size block read performance in guest"
" using dd commands" % blk_size, logging.info)
dd_cmd = "dd"
dd_cmd += " if=%s" % test_file
dd_cmd += " of=/dev/null"
dd_cmd += " bs=%s" % blk_size
dd_cmd += " iflag=direct"
try:
out = session.cmd_output(dd_cmd, timeout=test_timeout)
except Exception:
_clean_up(STEP_5)
raise
# After STEP 6
return out
if not hasattr(test, "write_perf_keyval"):
test.cancel("There is no 'write_perf_keyval' method in"
" test object, skip this test")
error_context.context("boot guest over virtio driver", logging.info)
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = int(params.get("login_timeout", 360))
test_timeout = int(params["test_timeout"])
session = vm.wait_for_login(timeout=timeout)
guest_ver = session.cmd_output("uname -r").strip()
host_ver = os.uname()[2]
kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")
if kvm_userspace_ver_cmd:
try:
cmd_result = process.run(kvm_userspace_ver_cmd, shell=True)
qemu_version = cmd_result.stdout.strip()
except process.CmdError:
qemu_version = "Unknown"
else:
qemu_path = utils_misc.get_qemu_binary(params)
version_line = process.system_output("%s -help | head -n 1"
% qemu_path, shell=True)
matches = re.findall("version .*?,", version_line, re.I)
if matches:
qemu_version = " ".join(matches[0].split()[1:]).strip(",")
else:
qemu_version = "Unknown"
# After STEP 1
try:
result_name = params.get("result_name", "nfs-perf.RHS")
result_file_path = utils_misc.get_path(test.resultsdir, result_name)
result_file = open(result_file_path, 'w')
except Exception:
_clean_up(STEP_1)
raise
# After STEP 2
error_context.context("mount nfs server in guest with tcp protocol",
logging.info)
nfs_server = params.get("nfs_server")
nfs_path = params.get("nfs_path")
mnt_option = params.get("mnt_option")
mnt_point = "/tmp/nfs_perf_%s" % utils_misc.generate_random_string(4)
test_file_prefix = os.path.join(mnt_point, "test_%si_" %
utils_misc.generate_random_string(4))
blk_size_list = params.get("blk_size_list", "8k").split()
test_file_list = list(map(lambda x: test_file_prefix + x, blk_size_list))
if (not nfs_server) or (not nfs_path) or (not mnt_point):
_clean_up(STEP_2)
test.error("Missing configuration for nfs partition."
" Check your config files")
try:
session.cmd("mkdir -p %s" % mnt_point)
except Exception:
_clean_up(STEP_2)
raise
# After STEP 3
# Prepare nfs partition.
mnt_cmd = "mount"
mnt_cmd += " -t nfs"
if mnt_option:
mnt_cmd += " -o %s" % mnt_option
mnt_cmd += " %s:%s" % (nfs_server, nfs_path)
mnt_cmd_out = mnt_cmd + " /tmp/***_****_****"
mnt_cmd += " %s" % mnt_point
try:
session.cmd(mnt_cmd)
except Exception:
_clean_up(STEP_3)
raise
# After STEP 4
# Record mount command in result file.
try:
result_file.write("### kvm-userspace-ver : %s\n" % qemu_version)
result_file.write("### kvm_version : %s\n" % host_ver)
result_file.write("### guest-kernel-ver : %s\n" % guest_ver)
result_file.write("### %s\n" % mnt_cmd_out)
result_file.write("Category:ALL\n")
except (IOError, ValueError) as e:
logging.error("Failed to write to result file,"
" error message:\n%s", e)
result_list = ["%s|%016s|%016s" % ("blk_size", "Write", "Read")]
speed_pattern = r"(\d+ bytes).*?([\d\.]+ s).*?([\d\.]+ [KkMmGgTt])B/s"
try:
prefix = "nfs"
for blk_size in blk_size_list:
prefix += "--%s" % blk_size
test_file = test_file_list[blk_size_list.index(blk_size)]
result = "%08s|" % blk_size[:-1]
# Get write test result.
out = _do_write_test(blk_size, test_file)
tmp_list = re.findall(speed_pattern, out)
if not tmp_list:
_clean_up(STEP_5)
test.error("Could not get correct write result."
" dd cmd output:\n%s" % out)
_, _, speed = tmp_list[0]
speed = utils_misc.normalize_data_size(speed)
result += "%016s|" % speed
test.write_perf_keyval({"%s--%s" % (prefix, "write"): speed})
# Get read test result.
out = _do_read_test(blk_size, test_file)
tmp_list = re.findall(speed_pattern, out)
if not tmp_list:
_clean_up(STEP_6)
test.error("Could not get correct read result."
" dd cmd output:\n%s" % out)
_, _, speed = tmp_list[0]
speed = utils_misc.normalize_data_size(speed)
result += "%016s" % speed
test.write_perf_keyval({"%s--%s" % (prefix, "read"): speed})
# Append result into result list.
result_list.append(result)
finally:
try:
result_file.write("\n".join(result_list))
except (IOError, ValueError) as e:
logging.error("Failed to write to result file,"
" error message:\n%s", e)
_clean_up(STEP_6)
def run(test, params, env):
"""
KVM block resize test:
1) Start guest with data image and check the data image size.
2) Enlarge(or Decrease) the data image and check it in guest.
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
def get_block_size(session, block_cmd, block_pattern):
"""
Get block size inside guest.
"""
output = session.cmd_output(block_cmd)
block_size = re.findall(block_pattern, output)
if block_size:
if not re.search("[a-zA-Z]", block_size[0]):
return int(block_size[0])
else:
return float(utils_misc.normalize_data_size(block_size[0], order_magnitude="B"))
else:
raise error.TestError(
"Can not find the block size for the" " deivce. The output of command" " is: %s" % output
)
def compare_block_size(session, block_cmd, block_pattern):
"""
Compare the current block size with the expected size.
"""
global current_size
current_size = get_block_size(session, block_size_cmd, block_size_pattern)
if current_size <= block_size and current_size >= block_size * (1 - accept_ratio):
logging.info(
"Block Resizing Finished !!! \n" "Current size %s is same as the expected %s", current_size, block_size
)
return True
return
error.context("Check image size in guest", logging.info)
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = float(params.get("login_timeout", 240))
session = vm.wait_for_login(timeout=timeout)
data_image = params.get("images").split()[-1]
data_image_params = params.object_params(data_image)
data_image_size = data_image_params.get("image_size")
data_image_size = float(utils_misc.normalize_data_size(data_image_size, order_magnitude="B"))
data_image_filename = storage.get_image_filename(data_image_params, data_dir.get_data_dir())
data_image_dev = vm.get_block({"file": data_image_filename})
block_size_cmd = params["block_size_cmd"]
block_size_pattern = params.get("block_size_pattern")
need_reboot = params.get("need_reboot", "no") == "yes"
accept_ratio = float(params.get("accept_ratio", 0))
block_size = get_block_size(session, block_size_cmd, block_size_pattern)
if block_size > data_image_size or block_size < data_image_size * (1 - accept_ratio):
raise error.TestError(
"Please check your system and image size check"
" command. The command output is not compatible"
" with the image size."
)
if params.get("guest_prepare_cmd"):
session.cmd(params.get("guest_prepare_cmd"))
disk_update_cmd = params.get("disk_update_cmd")
if disk_update_cmd:
disk_update_cmd = disk_update_cmd.split("::")
disk_rescan_cmd = params.get("disk_rescan_cmd")
block_size = data_image_size
disk_change_ratio = params["disk_change_ratio"]
for index, ratio in enumerate(disk_change_ratio.strip().split()):
old_block_size = block_size
block_size = int(int(data_image_size) * float(ratio))
if block_size == old_block_size:
logging.warn("Block size is not changed in round %s." " Just skip it" % index)
continue
if disk_update_cmd:
if "DISK_CHANGE_SIZE" in disk_update_cmd[index]:
disk_unit = params.get("disk_unit", "M")
size = abs(block_size - old_block_size)
change_size = utils_misc.normalize_data_size("%sB" % size, disk_unit)
disk_update_cmd[index] = re.sub("DISK_CHANGE_SIZE", change_size.split(".")[0], disk_update_cmd[index])
error.context("Change the disk size to %s" % block_size, logging.info)
# So far only virtio drivers support online auto block size change in
# linux guest. So we need manully update the the disk or even reboot
# guest to get the right block size after change it from monitor.
# We need shrink the disk in guest first, than in monitor
if block_size < old_block_size and disk_update_cmd:
session.cmd(disk_update_cmd[index])
vm.monitor.block_resize(data_image_dev, block_size)
if need_reboot:
session = vm.reboot(session=session)
elif disk_rescan_cmd:
session.cmd(disk_rescan_cmd)
# We need expand disk in monitor first than extend it in guest
if block_size > old_block_size and disk_update_cmd:
session.cmd(disk_update_cmd[index])
global current_size
current_size = 0
if not utils_misc.wait_for(
lambda: compare_block_size(session, block_size_cmd, block_size_pattern), 20, 0, 1, "Block Resizing"
):
raise error.TestFail(
"Guest reported a wrong disk size:\n"
" reported: %s\n"
" expect: %s\n" % (current_size, block_size)
)
def run(test, params, env):
"""
Block performance test with fio
Steps:
1) boot up guest with one data disk on specified backend and pin qemu-kvm process to the last numa node on host
2) pin guest vcpu and vhost threads to cpus of last numa node repectively
3) format data disk and run fio in guest
4) collect fio results and host info
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment
"""
def fio_thread():
"""
run fio command in guest
"""
session.cmd_status(run_cmd, cmd_timeout)
def _pin_vm_threads(node):
"""
pin guest vcpu and vhost threads to cpus of a numa node repectively
:param node: which numa node to pin
"""
if node:
if not isinstance(node, utils_misc.NumaNode):
node = utils_misc.NumaNode(int(node))
utils_test.qemu.pin_vm_threads(vm, node)
def fio_install(tarball):
"""
check whether fio is installed in guest, if no, install it, if yes, do nothing.
:param tarball: fio tar package
"""
if session.cmd_status(check_install_fio):
tarball = os.path.join(data_dir.get_deps_dir(), tarball)
if os_type == "linux":
vm.copy_files_to(tarball, "/tmp")
session.cmd("cd /tmp/ && tar -zxvf /tmp/%s" % os.path.basename(tarball), cmd_timeout)
session.cmd("cd %s && %s" % (fio_path, compile_cmd), cmd_timeout)
elif os_type == "windows":
session.cmd("md %s" % fio_path)
vm.copy_files_to(tarball, fio_path)
# login virtual machine
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
login_timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=login_timeout)
process.system_output("numactl --hardware")
process.system_output("numactl --show")
_pin_vm_threads(params.get("numa_node"))
# get parameter from dictionary
check_install_fio = params["check_install_fio"]
tarball = params["tarball"]
fio_path = params["fio_path"]
compile_cmd = params.get("compile_cmd")
fio_cmd = params["fio_cmd"]
rw = params["rw"]
block_size = params["block_size"]
iodepth = params["iodepth"]
threads = params["threads"]
cmd_timeout = int(params.get("cmd_timeout", 1200))
order_list = params["order_list"]
driver_format = params.get("drive_format")
kvm_ver_chk_cmd = params.get("kvm_ver_chk_cmd")
guest_ver_cmd = params["guest_ver_cmd"]
pattern = params["pattern"]
pre_cmd = params["pre_cmd"]
guest_result_file = params["guest_result_file"]
format = params.get("format")
os_type = params.get("os_type", "linux")
drop_cache = params.get("drop_cache")
num_disk = params.get("num_disk")
result_path = utils_misc.get_path(test.resultsdir,
"fio_result.RHS")
result_file = open(result_path, "w")
# scratch host and windows guest version info
get_version(session, result_file, kvm_ver_chk_cmd, guest_ver_cmd, os_type,
driver_format, cmd_timeout)
# install fio tool in guest
fio_install(tarball)
# online disk
if os_type == "windows":
for num in range(1, int(num_disk) + 1):
disks = check_disk_status(session, cmd_timeout, num)
diskstatus = re.findall("Disk\s+\d+\s+(\w+).*?\s+\d+", disks[0])[0]
if diskstatus == "Offline":
online_disk_cmd = params.get("online_disk_cmd")
online_disk_run = online_disk_cmd % num
(s, o) = session.cmd_status_output(online_disk_run,
timeout=cmd_timeout)
if s:
raise exceptions.TestFail("Failed to online disk: %s" % o)
# format disk
if format == "True":
session.cmd(pre_cmd, cmd_timeout)
# get order_list
order_line = ""
for order in order_list.split():
order_line += "%s|" % format_result(order)
# get result tested by each scenario
for io_pattern in rw.split():
result_file.write("Category:%s\n" % io_pattern)
result_file.write("%s\n" % order_line.rstrip("|"))
for bs in block_size.split():
for io_depth in iodepth.split():
for numjobs in threads.split():
line = ""
line += "%s|" % format_result(bs[:-1])
line += "%s|" % format_result(io_depth)
line += "%s|" % format_result(numjobs)
if format == "True":
file_name = io_pattern + "_" + bs + "_" + io_depth
run_cmd = fio_cmd % (io_pattern, bs, io_depth, file_name, numjobs)
else:
run_cmd = fio_cmd % (io_pattern, bs, io_depth, numjobs)
logging.info("run_cmd is: %s" % run_cmd)
if os_type == "linux":
(s, o) = session.cmd_status_output(drop_cache,
timeout=cmd_timeout)
if s:
raise exceptions.TestFail("Failed to free memory: %s" % o)
cpu_file = os.path.join(data_dir.get_tmp_dir(), "cpus")
io_exits_b = int(process.system_output("cat /sys/kernel/debug/kvm/exits"))
fio_t = threading.Thread(target=fio_thread)
fio_t.start()
process.system_output("mpstat 1 60 > %s" % cpu_file, shell=True)
fio_t.join()
io_exits_a = int(process.system_output("cat /sys/kernel/debug/kvm/exits"))
vm.copy_files_from(guest_result_file, data_dir.get_tmp_dir())
fio_result_file = os.path.join(data_dir.get_tmp_dir(), "fio_result")
o = process.system_output("egrep '(read|write)' %s" % fio_result_file)
results = re.findall(pattern, o)
o = process.system_output("egrep 'lat' %s" % fio_result_file)
laten = re.findall("\s{5}lat\s\((\wsec)\).*?avg=[\s]?(\d+(?:[\.][\d]+)?).*?", o)
bw = float(utils_misc.normalize_data_size(results[0][0]))
iops = int(results[0][1])
if os_type == "linux":
o = process.system_output("egrep 'util' %s" % fio_result_file)
util = float(re.findall(".*?util=(\d+(?:[\.][\d]+))%", o)[0])
lat = float(laten[0][1]) / 1000 if laten[0][0] == "usec" else float(laten[0][1])
if re.findall("rw", io_pattern):
bw = bw + float(utils_misc.normalize_data_size(results[1][0]))
iops = iops + int(results[1][1])
lat1 = float(laten[1][1]) / 1000 if laten[1][0] == "usec" else float(laten[1][1])
lat = lat + lat1
ret = process.system_output("tail -n 1 %s" % cpu_file)
idle = float(ret.split()[-1])
iowait = float(ret.split()[5])
cpu = 100 - idle - iowait
normal = bw / cpu
io_exits = io_exits_a - io_exits_b
for result in bw, iops, lat, cpu, normal:
line += "%s|" % format_result(result)
if os_type == "windows":
line += "%s" % format_result(io_exits)
if os_type == "linux":
line += "%s|" % format_result(io_exits)
line += "%s" % format_result(util)
result_file.write("%s\n" % line)
# del temporary files in guest
clean_tmp_files(session, check_install_fio, tarball, os_type, guest_result_file, fio_path, cmd_timeout)
session.close()
speed_pattern = r"(\d+ bytes).*?([\d\.]+ s).*?([\d\.]+ [KkMmGgTt])B/s"
try:
prefix = "nfs"
for blk_size in blk_size_list:
prefix += "--%s" % blk_size
test_file = test_file_list[blk_size_list.index(blk_size)]
result = "%08s|" % blk_size[:-1]
# Get write test result.
out = _do_write_test(blk_size, test_file)
tmp_list = re.findall(speed_pattern, out)
if not tmp_list:
_clean_up(STEP_5)
raise error.TestError("Could not get correct write result."
" dd cmd output:\n%s" % out)
_, _, speed = tmp_list[0]
speed = utils_misc.normalize_data_size(speed)
result += "%016s|" % speed
test.write_perf_keyval({"%s--%s" % (prefix, "write"): speed})
# Get read test result.
out = _do_read_test(blk_size, test_file)
tmp_list = re.findall(speed_pattern, out)
if not tmp_list:
_clean_up(STEP_6)
raise error.TestError("Could not get correct read result."
" dd cmd output:\n%s" % out)
_, _, speed = tmp_list[0]
speed = utils_misc.normalize_data_size(speed)
result += "%016s" % speed
test.write_perf_keyval({"%s--%s" % (prefix, "read"): speed})
# Append result into result list.
def run(test, params, env):
"""
1. Create a pool
2. Create n number of volumes(vol-create-as)
3. Check the volume details from the following commands
vol-info
vol-key
vol-list
vol-name
vol-path
vol-pool
qemu-img info
4. Delete the volume and check in vol-list
5. Repeat the steps for number of volumes given
6. Delete the pool and target
TODO: Handle negative testcases
"""
def delete_volume(expected_vol):
"""
Deletes Volume
"""
pool_name = expected_vol['pool_name']
vol_name = expected_vol['name']
pv = libvirt_storage.PoolVolume(pool_name)
if not pv.delete_volume(vol_name):
raise error.TestFail("Delete volume failed." % vol_name)
else:
logging.debug("Volume: %s successfully deleted on pool: %s",
vol_name, pool_name)
def get_vol_list(pool_name, vol_name):
"""
Parse the volume list
"""
output = virsh.vol_list(pool_name, "--details")
rg = re.compile(
r'^(\S+)\s+(\S+)\s+(\S+)\s+(\d+.\d+\s\S+)\s+(\d+.\d+.*)')
vol = {}
vols = []
volume_detail = None
for line in output.stdout.splitlines():
match = re.search(rg, line.lstrip())
if match is not None:
vol['name'] = match.group(1)
vol['path'] = match.group(2)
vol['type'] = match.group(3)
vol['capacity'] = match.group(4)
vol['allocation'] = match.group(5)
vols.append(vol)
vol = {}
for volume in vols:
if volume['name'] == vol_name:
volume_detail = volume
return volume_detail
def norm_capacity(capacity):
"""
Normalize the capacity values to bytes
"""
# Normaize all values to bytes
norm_capacity = {}
des = {'B': 'B', 'bytes': 'B', 'b': 'B', 'kib': 'K',
'KiB': 'K', 'K': 'K', 'k': 'K', 'KB': 'K',
'mib': 'M', 'MiB': 'M', 'M': 'M', 'm': 'M',
'MB': 'M', 'gib': 'G', 'GiB': 'G', 'G': 'G',
'g': 'G', 'GB': 'G', 'Gb': 'G', 'tib': 'T',
'TiB': 'T', 'TB': 'T', 'T': 'T', 't': 'T'
}
val = {'B': 1,
'K': 1024,
'M': 1048576,
'G': 1073741824,
'T': 1099511627776
}
reg_list = re.compile(r'(\S+)\s(\S+)')
match_list = re.search(reg_list, capacity['list'])
if match_list is not None:
mem_value = float(match_list.group(1))
norm = val[des[match_list.group(2)]]
norm_capacity['list'] = int(mem_value * norm)
else:
raise error.TestFail("Error in parsing capacity value in"
" virsh vol-list")
match_info = re.search(reg_list, capacity['info'])
if match_info is not None:
mem_value = float(match_info.group(1))
norm = val[des[match_list.group(2)]]
norm_capacity['info'] = int(mem_value * norm)
else:
raise error.TestFail("Error in parsing capacity value "
"in virsh vol-info")
norm_capacity['qemu_img'] = capacity['qemu_img']
norm_capacity['xml'] = int(capacity['xml'])
return norm_capacity
def check_vol(expected, avail=True):
"""
Checks the expected volume details with actual volume details from
vol-dumpxml
vol-list
vol-info
vol-key
vol-path
qemu-img info
"""
error_count = 0
pv = libvirt_storage.PoolVolume(expected['pool_name'])
vol_exists = pv.volume_exists(expected['name'])
if vol_exists:
if not avail:
error_count += 1
logging.error("Expect volume %s not exists but find it",
expected['name'])
return error_count
else:
if avail:
error_count += 1
logging.error("Expect volume %s exists but not find it",
expected['name'])
return error_count
else:
logging.info("Volume %s checked successfully for deletion",
expected['name'])
return error_count
actual_list = get_vol_list(expected['pool_name'], expected['name'])
actual_info = pv.volume_info(expected['name'])
# Get values from vol-dumpxml
volume_xml = vol_xml.VolXML.new_from_vol_dumpxml(expected['name'],
expected['pool_name'])
# Check against virsh vol-key
vol_key = virsh.vol_key(expected['name'], expected['pool_name'])
if vol_key.stdout.strip() != volume_xml.key:
logging.error("Volume key is mismatch \n%s"
"Key from xml: %s\nKey from command: %s",
expected['name'], volume_xml.key, vol_key)
error_count += 1
else:
logging.debug("virsh vol-key for volume: %s successfully"
" checked against vol-dumpxml", expected['name'])
# Check against virsh vol-name
get_vol_name = virsh.vol_name(expected['path'])
if get_vol_name.stdout.strip() != expected['name']:
logging.error("Volume name mismatch\n"
"Expected name: %s\nOutput of vol-name: %s",
expected['name'], get_vol_name)
# Check against virsh vol-path
vol_path = virsh.vol_path(expected['name'], expected['pool_name'])
if expected['path'] != vol_path.stdout.strip():
logging.error("Volume path mismatch for volume: %s\n"
"Expected path: %s\nOutput of vol-path: %s\n",
expected['name'],
expected['path'], vol_path)
error_count += 1
else:
logging.debug("virsh vol-path for volume: %s successfully checked"
" against created volume path", expected['name'])
# Check path against virsh vol-list
if expected['path'] != actual_list['path']:
logging.error("Volume path mismatch for volume:%s\n"
"Expected Path: %s\nPath from virsh vol-list: %s",
expected['name'], expected['path'],
actual_list['path'])
error_count += 1
else:
logging.debug("Path of volume: %s from virsh vol-list "
"successfully checked against created "
"volume path", expected['name'])
# Check path against virsh vol-dumpxml
if expected['path'] != volume_xml.path:
logging.error("Volume path mismatch for volume: %s\n"
"Expected Path: %s\nPath from virsh vol-dumpxml: %s",
expected['name'], expected['path'], volume_xml.path)
error_count += 1
else:
logging.debug("Path of volume: %s from virsh vol-dumpxml "
"successfully checked against created volume path",
expected['name'])
# Check type against virsh vol-list
if expected['type'] != actual_list['type']:
logging.error("Volume type mismatch for volume: %s\n"
"Expected Type: %s\n Type from vol-list: %s",
expected['name'], expected['type'],
actual_list['type'])
error_count += 1
else:
logging.debug("Type of volume: %s from virsh vol-list "
"successfully checked against the created "
"volume type", expected['name'])
# Check type against virsh vol-info
if expected['type'] != actual_info['Type']:
logging.error("Volume type mismatch for volume: %s\n"
"Expected Type: %s\n Type from vol-info: %s",
expected['name'], expected['type'],
actual_info['Type'])
error_count += 1
else:
logging.debug("Type of volume: %s from virsh vol-info successfully"
" checked against the created volume type",
expected['name'])
# Check name against virsh vol-info
if expected['name'] != actual_info['Name']:
logging.error("Volume name mismatch for volume: %s\n"
"Expected name: %s\n Name from vol-info: %s",
expected['name'],
expected['name'], actual_info['Name'])
error_count += 1
else:
logging.debug("Name of volume: %s from virsh vol-info successfully"
" checked against the created volume name",
expected['name'])
# Check format from against qemu-img info
img_info = utils_misc.get_image_info(expected['path'])
if expected['format']:
if expected['format'] != img_info['format']:
logging.error("Volume format mismatch for volume: %s\n"
"Expected format: %s\n"
"Format from qemu-img info: %s",
expected['name'], expected['format'],
img_info['format'])
error_count += 1
else:
logging.debug("Format of volume: %s from qemu-img info "
"checked successfully against the created "
"volume format", expected['name'])
# Check format against vol-dumpxml
if expected['format']:
if expected['format'] != volume_xml.format:
logging.error("Volume format mismatch for volume: %s\n"
"Expected format: %s\n"
"Format from vol-dumpxml: %s",
expected['name'], expected['format'],
volume_xml.format)
error_count += 1
else:
logging.debug("Format of volume: %s from virsh vol-dumpxml "
"checked successfully against the created"
" volume format", expected['name'])
logging.info(expected['encrypt_format'])
# Check encrypt against vol-dumpxml
if expected['encrypt_format']:
# As the 'default' format will change to specific valut(qcow), so
# just output it here
logging.debug("Encryption format of volume '%s' is: %s",
expected['name'], volume_xml.encryption.format)
# And also output encryption secret uuid
secret_uuid = volume_xml.encryption.secret['uuid']
logging.debug("Encryption secret of volume '%s' is: %s",
expected['name'], secret_uuid)
if expected['encrypt_secret']:
if expected['encrypt_secret'] != secret_uuid:
logging.error("Encryption secret mismatch for volume: %s\n"
"Expected secret uuid: %s\n"
"Secret uuid from vol-dumpxml: %s",
expected['name'], expected['encrypt_secret'],
secret_uuid)
error_count += 1
else:
# If no set encryption secret value, automatically
# generate a secret value at the time of volume creation
logging.debug("Volume encryption secret is %s", secret_uuid)
# Check pool name against vol-pool
vol_pool = virsh.vol_pool(expected['path'])
if expected['pool_name'] != vol_pool.stdout.strip():
logging.error("Pool name mismatch for volume: %s against"
"virsh vol-pool", expected['name'])
error_count += 1
else:
logging.debug("Pool name of volume: %s checked successfully"
" against the virsh vol-pool", expected['name'])
norm_cap = {}
capacity = {}
capacity['list'] = actual_list['capacity']
capacity['info'] = actual_info['Capacity']
capacity['xml'] = volume_xml.capacity
capacity['qemu_img'] = img_info['vsize']
norm_cap = norm_capacity(capacity)
delta_size = params.get('delta_size', "1024")
if abs(expected['capacity'] - norm_cap['list']) > delta_size:
logging.error("Capacity mismatch for volume: %s against virsh"
" vol-list\nExpected: %s\nActual: %s",
expected['name'], expected['capacity'],
norm_cap['list'])
error_count += 1
else:
logging.debug("Capacity value checked successfully against"
" virsh vol-list for volume %s", expected['name'])
if abs(expected['capacity'] - norm_cap['info']) > delta_size:
logging.error("Capacity mismatch for volume: %s against virsh"
" vol-info\nExpected: %s\nActual: %s",
expected['name'], expected['capacity'],
norm_cap['info'])
error_count += 1
else:
logging.debug("Capacity value checked successfully against"
" virsh vol-info for volume %s", expected['name'])
if abs(expected['capacity'] - norm_cap['xml']) > delta_size:
logging.error("Capacity mismatch for volume: %s against virsh"
" vol-dumpxml\nExpected: %s\nActual: %s",
expected['name'], expected['capacity'],
norm_cap['xml'])
error_count += 1
else:
logging.debug("Capacity value checked successfully against"
" virsh vol-dumpxml for volume: %s",
expected['name'])
if abs(expected['capacity'] - norm_cap['qemu_img']) > delta_size:
logging.error("Capacity mismatch for volume: %s against "
"qemu-img info\nExpected: %s\nActual: %s",
expected['name'], expected['capacity'],
norm_cap['qemu_img'])
error_count += 1
else:
logging.debug("Capacity value checked successfully against"
" qemu-img info for volume: %s",
expected['name'])
return error_count
def get_all_secrets():
"""
Return all exist libvirt secrets uuid in a list
"""
secret_list = []
secrets = virsh.secret_list().stdout.strip()
for secret in secrets.splitlines()[2:]:
secret_list.append(secret.strip().split()[0])
return secret_list
# Initialize the variables
pool_name = params.get("pool_name")
pool_type = params.get("pool_type")
pool_target = params.get("pool_target")
if os.path.dirname(pool_target) is "":
pool_target = os.path.join(test.tmpdir, pool_target)
vol_name = params.get("volume_name")
vol_number = int(params.get("number_of_volumes", "2"))
capacity = params.get("volume_size", "1048576")
allocation = params.get("volume_allocation", "1048576")
vol_format = params.get("volume_format")
source_name = params.get("gluster_source_name", "gluster-vol1")
source_path = params.get("gluster_source_path", "/")
encrypt_format = params.get("vol_encrypt_format")
encrypt_secret = params.get("encrypt_secret")
emulated_image = params.get("emulated_image")
emulated_image_size = params.get("emulated_image_size")
try:
str_capa = utils_misc.normalize_data_size(capacity, "B")
int_capa = int(str(str_capa).split('.')[0])
except ValueError:
raise error.TestError("Translate size %s to 'B' failed" % capacity)
try:
str_capa = utils_misc.normalize_data_size(allocation, "B")
int_allo = int(str(str_capa).split('.')[0])
except ValueError:
raise error.TestError("Translate size %s to 'B' failed" % allocation)
# Stop multipathd to avoid start pool fail(For fs like pool, the new add
# disk may in use by device-mapper, so start pool will report disk already
# mounted error).
multipathd = service.Factory.create_service("multipathd")
multipathd_status = multipathd.status()
if multipathd_status:
multipathd.stop()
# Get exists libvirt secrets before test
ori_secrets = get_all_secrets()
expected_vol = {}
vol_type = 'file'
if pool_type in ['disk', 'logical']:
vol_type = 'block'
if pool_type == 'gluster':
vol_type = 'network'
logging.debug("Debug:\npool_name:%s\npool_type:%s\npool_target:%s\n"
"vol_name:%s\nvol_number:%s\ncapacity:%s\nallocation:%s\n"
"vol_format:%s", pool_name, pool_type, pool_target,
vol_name, vol_number, capacity, allocation, vol_format)
libv_pvt = utlv.PoolVolumeTest(test, params)
# Run Testcase
total_err_count = 0
try:
# Create a new pool
libv_pvt.pre_pool(pool_name=pool_name,
pool_type=pool_type,
pool_target=pool_target,
emulated_image=emulated_image,
image_size=emulated_image_size,
source_name=source_name,
source_path=source_path)
for i in range(vol_number):
volume_name = "%s_%d" % (vol_name, i)
expected_vol['pool_name'] = pool_name
expected_vol['pool_type'] = pool_type
expected_vol['pool_target'] = pool_target
expected_vol['capacity'] = int_capa
expected_vol['allocation'] = int_allo
expected_vol['format'] = vol_format
expected_vol['name'] = volume_name
expected_vol['type'] = vol_type
expected_vol['encrypt_format'] = encrypt_format
expected_vol['encrypt_secret'] = encrypt_secret
# Creates volume
if pool_type != "gluster":
expected_vol['path'] = pool_target + '/' + volume_name
new_volxml = vol_xml.VolXML()
new_volxml.name = volume_name
new_volxml.capacity = int_capa
new_volxml.allocation = int_allo
if vol_format:
new_volxml.format = vol_format
encrypt_dict = {}
if encrypt_format:
encrypt_dict.update({"format": encrypt_format})
if encrypt_secret:
encrypt_dict.update({"secret": {'uuid': encrypt_secret}})
if encrypt_dict:
new_volxml.encryption = new_volxml.new_encryption(**encrypt_dict)
logging.debug("Volume XML for creation:\n%s", str(new_volxml))
virsh.vol_create(pool_name, new_volxml.xml, debug=True)
else:
ip_addr = utlv.get_host_ipv4_addr()
expected_vol['path'] = "gluster://%s/%s/%s" % (ip_addr,
source_name,
volume_name)
utils.run("qemu-img create -f %s %s %s" % (vol_format,
expected_vol['path'],
capacity))
virsh.pool_refresh(pool_name)
# Check volumes
total_err_count += check_vol(expected_vol)
# Delete volume and check for results
delete_volume(expected_vol)
total_err_count += check_vol(expected_vol, False)
if total_err_count > 0:
raise error.TestFail("Get %s errors when checking volume" % total_err_count)
finally:
# Clean up
for sec in get_all_secrets():
if sec not in ori_secrets:
virsh.secret_undefine(sec)
try:
libv_pvt.cleanup_pool(pool_name, pool_type, pool_target,
emulated_image, source_name=source_name)
except error.TestFail, detail:
logging.error(str(detail))
if multipathd_status:
multipathd.start()
