def get_asset_info(asset):
asset_path = os.path.join(data_dir.get_download_dir(), '%s.ini' % asset)
asset_cfg = ConfigParser.ConfigParser()
asset_cfg.read(asset_path)
url = asset_cfg.get(asset, 'url')
sha1_url = asset_cfg.get(asset, 'sha1_url')
title = asset_cfg.get(asset, 'title')
destination = asset_cfg.get(asset, 'destination')
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
asset_exists = os.path.isfile(destination)
# Optional fields
try:
destination_uncompressed = asset_cfg.get(asset,
'destination_uncompressed')
if not os.path.isabs(destination_uncompressed):
destination_uncompressed = os.path.join(data_dir.get_data_dir(),
destination)
uncompress_cmd = asset_cfg.get(asset, 'uncompress_cmd')
except:
destination_uncompressed = None
uncompress_cmd = None
return {'url': url, 'sha1_url': sha1_url, 'destination': destination,
'destination_uncompressed': destination_uncompressed,
'uncompress_cmd': uncompress_cmd, 'shortname': asset,
'title': title,
'downloaded': asset_exists}
def print_guest_list(options):
"""
Helper function to pretty print the guest list.
This function uses a paginator, if possible (inspired on git).
@param options: OptParse object with cmdline options.
@param cartesian_parser: Cartesian parser object with test options.
"""
cfg = os.path.join(data_dir.get_root_dir(), options.type,
"cfg", "guest-os.cfg")
cartesian_parser = cartesian_config.Parser()
cartesian_parser.parse_file(cfg)
pipe = get_paginator()
index = 0
pipe.write("Searched %s for guest images\n" %
os.path.join(data_dir.get_data_dir(), 'images'))
pipe.write("Available guests:")
pipe.write("\n\n")
for params in cartesian_parser.get_dicts():
index += 1
image_name = storage.get_image_filename(params, data_dir.get_data_dir())
shortname = ".".join(params['name'].split(".")[1:])
if os.path.isfile(image_name):
out = (bcolors.blue + str(index) + bcolors.end + " " +
shortname + "\n")
else:
out = (bcolors.blue + str(index) + bcolors.end + " " +
shortname + " " + bcolors.yellow +
"(missing %s)" % os.path.basename(image_name) +
bcolors.end + "\n")
pipe.write(out)
def print_guest_list(options):
"""
Helper function to pretty print the guest list.
This function uses a paginator, if possible (inspired on git).
:param options: OptParse object with cmdline options.
:param cartesian_parser: Cartesian parser object with test options.
"""
pipe = get_paginator()
# lvsb testing has no concept of guests
if options.vt_type == 'lvsb':
pipe.write("No guest types available for lvsb testing")
return
index = 0
pipe.write("Searched %s for guest images\n" %
os.path.join(data_dir.get_data_dir(), 'images'))
pipe.write("Available guests:")
pipe.write("\n\n")
for params in get_guest_name_parser(options).get_dicts():
index += 1
base_dir = params.get("images_base_dir", data_dir.get_data_dir())
image_name = storage.get_image_filename(params, base_dir)
name = params['name']
if os.path.isfile(image_name):
out = (bcolors.blue + str(index) + bcolors.end + " " +
name + "\n")
else:
out = (bcolors.blue + str(index) + bcolors.end + " " +
name + " " + bcolors.yellow +
"(missing %s)" % os.path.basename(image_name) +
bcolors.end + "\n")
pipe.write(out)
def print_guest_list(options):
"""
Helper function to pretty print the guest list.
This function uses a paginator, if possible (inspired on git).
@param options: OptParse object with cmdline options.
@param cartesian_parser: Cartesian parser object with test options.
"""
pipe = get_paginator()
index = 0
pipe.write("Searched %s for guest images\n" %
os.path.join(data_dir.get_data_dir(), 'images'))
pipe.write("Available guests:")
pipe.write("\n\n")
for params in get_guest_name_parser(options).get_dicts():
index += 1
image_name = storage.get_image_filename(params, data_dir.get_data_dir())
shortname = params['shortname']
if os.path.isfile(image_name):
out = (bcolors.blue + str(index) + bcolors.end + " " +
shortname + "\n")
else:
out = (bcolors.blue + str(index) + bcolors.end + " " +
shortname + " " + bcolors.yellow +
"(missing %s)" % os.path.basename(image_name) +
bcolors.end + "\n")
pipe.write(out)
def bootstrap_tests(options):
"""
Bootstrap process (download the appropriate JeOS file to data dir).
This function will check whether the JeOS is in the right location of the
data dir, if not, it will download it non interactively.
@param options: OptParse object with program command line options.
"""
test_dir = os.path.dirname(sys.modules[__name__].__file__)
if options.type:
test_dir = os.path.abspath(os.path.join(os.path.dirname(test_dir), options.type))
elif options.config:
test_dir = os.path.dirname(os.path.dirname(options.config))
test_dir = os.path.abspath(test_dir)
if options.type == "qemu":
check_modules = ["kvm", "kvm-%s" % utils_misc.get_cpu_vendor(verbose=False)]
else:
check_modules = None
online_docs_url = "https://github.com/autotest/virt-test/wiki"
kwargs = {
"test_name": options.type,
"test_dir": test_dir,
"base_dir": data_dir.get_data_dir(),
"default_userspace_paths": None,
"check_modules": check_modules,
"online_docs_url": online_docs_url,
"restore_image": options.restore,
"interactive": False,
}
# Tolerance we have without printing a message for the user to wait (3 s)
tolerance = 3
failed = False
wait_message_printed = False
bg = utils.InterruptedThread(bootstrap.bootstrap, kwargs=kwargs)
t_begin = time.time()
bg.start()
while bg.isAlive():
t_elapsed = time.time() - t_begin
if t_elapsed > tolerance and not wait_message_printed:
print_stdout("Running setup. Please wait...")
wait_message_printed = True
sys.stdout.switch()
time.sleep(0.1)
if wait_message_printed:
sys.stdout.switch()
reason = None
try:
bg.join()
except Exception, e:
failed = True
reason = e
def get_file_asset(title, src_path, destination):
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
for ext in (".xz", ".gz", ".7z", ".bz2"):
if os.path.exists(src_path + ext):
destination = destination + ext
logging.debug("Found source image %s", destination)
return {
"url": None,
"sha1_url": None,
"destination": src_path + ext,
"destination_uncompressed": destination,
"uncompress_cmd": None,
"shortname": title,
"title": title,
"downloaded": True,
}
if os.path.exists(src_path):
logging.debug("Found source image %s", destination)
return {
"url": src_path,
"sha1_url": None,
"destination": destination,
"destination_uncompressed": None,
"uncompress_cmd": None,
"shortname": title,
"title": title,
"downloaded": os.path.exists(destination),
}
return None
def postprocess_image(test, params, image_name):
"""
Postprocess a single QEMU image according to the instructions in params.
:param test: An Autotest test object.
:param params: A dict containing image postprocessing parameters.
"""
clone_master = params.get("clone_master", None)
base_dir = data_dir.get_data_dir()
image = qemu_storage.QemuImg(params, base_dir, image_name)
if params.get("check_image") == "yes":
try:
if clone_master is None:
image.check_image(params, base_dir)
elif clone_master == "yes":
if image_name in params.get("master_images_clone").split():
image.check_image(params, base_dir)
# Allow test to overwrite any pre-testing automatic backup
# with a new backup. i.e. assume pre-existing image/backup
# would not be usable after this test succeeds. The best
# example for this is when 'unattended_install' is run.
if params.get("backup_image", "no") == "yes":
image.backup_image(params, base_dir, "backup", True)
elif params.get("restore_image", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
except Exception, e:
if params.get("restore_image_on_check_error", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
if params.get("remove_image_on_check_error", "no") == "yes":
cl_images = params.get("master_images_clone", "")
if image_name in cl_images.split():
image.remove()
raise e
def bootstrap_tests(options):
"""
Bootstrap process (download the appropriate JeOS file to data dir).
This function will check whether the JeOS is in the right location of the
data dir, if not, it will download it non interactively.
@param options: OptParse object with program command line options.
"""
test_dir = os.path.dirname(sys.modules[__name__].__file__)
if options.type:
test_dir = os.path.abspath(
os.path.join(os.path.dirname(test_dir), options.type))
elif options.config:
test_dir = os.path.dirname(os.path.dirname(options.config))
test_dir = os.path.abspath(test_dir)
check_modules = [
"kvm", "kvm-%s" % utils_misc.get_cpu_vendor(verbose=False)
]
online_docs_url = "https://github.com/autotest/virt-test/wiki"
kwargs = {
'test_name': options.type,
'test_dir': test_dir,
'base_dir': data_dir.get_data_dir(),
'default_userspace_paths': None,
'check_modules': check_modules,
'online_docs_url': online_docs_url,
'restore_image': options.restore,
'interactive': False
}
# Tolerance we have without printing a message for the user to wait (3 s)
tolerance = 3
failed = False
wait_message_printed = False
bg = utils.InterruptedThread(bootstrap.bootstrap, kwargs=kwargs)
t_begin = time.time()
bg.start()
while bg.isAlive():
t_elapsed = time.time() - t_begin
if t_elapsed > tolerance and not wait_message_printed:
print_stdout("Running setup. Please wait...")
wait_message_printed = True
sys.stdout.switch()
time.sleep(0.1)
if wait_message_printed:
sys.stdout.switch()
reason = None
try:
bg.join()
except Exception, e:
failed = True
reason = e
def preprocess_image(test, params, image_name):
"""
Preprocess a single QEMU image according to the instructions in params.
:param test: Autotest test object.
:param params: A dict containing image preprocessing parameters.
:note: Currently this function just creates an image if requested.
"""
base_dir = params.get("images_base_dir", data_dir.get_data_dir())
image_filename = storage.get_image_filename(params,
base_dir)
create_image = False
if params.get("force_create_image") == "yes":
create_image = True
elif (params.get("create_image") == "yes" and not
os.path.exists(image_filename)):
create_image = True
if params.get("backup_image_before_testing", "no") == "yes":
image = qemu_storage.QemuImg(params, base_dir, image_name)
image.backup_image(params, base_dir, "backup", True, True)
if create_image:
image = qemu_storage.QemuImg(params, base_dir, image_name)
image.create(params)
def preprocess_image(test, params, image_name):
"""
Preprocess a single QEMU image according to the instructions in params.
:param test: Autotest test object.
:param params: A dict containing image preprocessing parameters.
:note: Currently this function just creates an image if requested.
"""
base_dir = params.get("images_base_dir", data_dir.get_data_dir())
if not storage.preprocess_image_backend(base_dir, params, image_name):
logging.error("Backend can't be prepared correctly.")
image_filename = storage.get_image_filename(params,
base_dir)
create_image = False
if params.get("force_create_image") == "yes":
create_image = True
elif (params.get("create_image") == "yes" and not
storage.file_exists(params, image_filename)):
create_image = True
if params.get("backup_image_before_testing", "no") == "yes":
image = qemu_storage.QemuImg(params, base_dir, image_name)
image.backup_image(params, base_dir, "backup", True, True)
if create_image:
image = qemu_storage.QemuImg(params, base_dir, image_name)
image.create(params)
def get_file_asset(title, src_path, destination):
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
for ext in (".xz", ".gz", ".7z", ".bz2"):
if os.path.exists(src_path + ext):
destination = destination + ext
logging.debug('Found source image %s', destination)
return {
'url': None,
'sha1_url': None,
'destination': src_path + ext,
'destination_uncompressed': destination,
'uncompress_cmd': None,
'shortname': title,
'title': title,
'downloaded': True
}
if os.path.exists(src_path):
logging.debug('Found source image %s', destination)
return {
'url': src_path,
'sha1_url': None,
'destination': destination,
'destination_uncompressed': None,
'uncompress_cmd': None,
'shortname': title,
'title': title,
'downloaded': os.path.exists(destination)
}
return None
def create_gluster_vol(params):
vol_name = params.get("gluster_volume_name")
force = params.get('force_recreate_gluster') == "yes"
brick_path = params.get("gluster_brick")
if not os.path.isabs(brick_path): # do nothing when path is absolute
base_dir = params.get("images_base_dir", data_dir.get_data_dir())
brick_path = os.path.join(base_dir, brick_path)
error.context("Host name lookup failed")
hostname = socket.gethostname()
if not hostname or hostname == "(none)":
if_up = utils_net.get_net_if(state="UP")
for i in if_up:
ipv4_value = utils_net.get_net_if_addrs(i)["ipv4"]
logging.debug("ipv4_value is %s", ipv4_value)
if ipv4_value != []:
ip_addr = ipv4_value[0]
break
hostname = ip_addr
# Start the gluster dameon, if not started
glusterd_start()
# Check for the volume is already present, if not create one.
if not is_gluster_vol_avail(vol_name) or force:
return gluster_vol_create(vol_name, hostname, brick_path, force)
else:
return True
def create_gluster_vol(params):
vol_name = params.get("gluster_volume_name")
force = params.get('force_recreate_gluster') == "yes"
brick_path = params.get("gluster_brick")
if not os.path.isabs(brick_path): # do nothing when path is absolute
base_dir = params.get("images_base_dir", data_dir.get_data_dir())
brick_path = os.path.join(base_dir, brick_path)
error.context("Host name lookup failed")
hostname = socket.gethostname()
if not hostname or hostname == "(none)":
if_up = utils_net.get_net_if(state="UP")
for i in if_up:
ipv4_value = utils_net.get_net_if_addrs(i)["ipv4"]
logging.debug("ipv4_value is %s", ipv4_value)
if ipv4_value != []:
ip_addr = ipv4_value[0]
break
hostname = ip_addr
# Start the gluster dameon, if not started
glusterd_start()
# Check for the volume is already present, if not create one.
if not is_gluster_vol_avail(vol_name) or force:
return gluster_vol_create(vol_name, hostname, brick_path, force)
else:
return True
def postprocess_image(test, params, image_name):
"""
Postprocess a single QEMU image according to the instructions in params.
:param test: An Autotest test object.
:param params: A dict containing image postprocessing parameters.
"""
clone_master = params.get("clone_master", None)
base_dir = data_dir.get_data_dir()
image = qemu_storage.QemuImg(params, base_dir, image_name)
if params.get("check_image") == "yes":
try:
if clone_master is None:
image.check_image(params, base_dir)
elif clone_master == "yes":
if image_name in params.get("master_images_clone").split():
image.check_image(params, base_dir)
# Allow test to overwrite any pre-testing automatic backup
# with a new backup. i.e. assume pre-existing image/backup
# would not be usable after this test succeeds. The best
# example for this is when 'unattended_install' is run.
if params.get("backup_image", "no") == "yes":
image.backup_image(params, base_dir, "backup", True)
elif params.get("restore_image", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
except Exception, e:
if params.get("restore_image_on_check_error", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
if params.get("remove_image_on_check_error", "no") == "yes":
cl_images = params.get("master_images_clone", "")
if image_name in cl_images.split():
image.remove()
raise e
def preprocess_image(test, params, image_name, vm_process_status=None):
"""
Preprocess a single QEMU image according to the instructions in params.
:param test: Autotest test object.
:param params: A dict containing image preprocessing parameters.
:param vm_process_status: This is needed in postprocess_image. Add it here
only for keep it work with process_images()
:note: Currently this function just creates an image if requested.
"""
base_dir = params.get("images_base_dir", data_dir.get_data_dir())
if not storage.preprocess_image_backend(base_dir, params, image_name):
logging.error("Backend can't be prepared correctly.")
image_filename = storage.get_image_filename(params, base_dir)
create_image = False
if params.get("force_create_image") == "yes":
create_image = True
elif (params.get("create_image") == "yes"
and not storage.file_exists(params, image_filename)):
create_image = True
if params.get("backup_image_before_testing", "no") == "yes":
image = qemu_storage.QemuImg(params, base_dir, image_name)
image.backup_image(params, base_dir, "backup", True, True)
if create_image:
image = qemu_storage.QemuImg(params, base_dir, image_name)
image.create(params)
def create_config_files(options):
"""
Check if the appropriate configuration files are present.
If the files are not present, create them.
:param options: OptParser object with options.
"""
shared_dir = os.path.dirname(data_dir.get_data_dir())
test_dir = os.path.dirname(shared_dir)
if options.vt_type and options.vt_config:
test_dir = data_dir.get_backend_dir(options.vt_type)
elif options.vt_type:
test_dir = data_dir.get_backend_dir(options.vt_type)
elif options.vt_config:
parent_config_dir = os.path.dirname(options.vt_config)
parent_config_dir = os.path.dirname(parent_config_dir)
options.vt_type = parent_config_dir
test_dir = os.path.join(test_dir, parent_config_dir)
if not os.path.exists(os.path.join(test_dir, "cfg")):
print_stdout("Setup error: %s does not exist" %
os.path.join(test_dir, "cfg"))
print_stdout("Perhaps you have not specified -t?")
sys.exit(1)
# lvsb test doesn't use shared configs
if options.vt_type != 'lvsb':
bootstrap.create_config_files(test_dir, shared_dir, interactive=False)
bootstrap.create_guest_os_cfg(options.vt_type)
bootstrap.create_subtests_cfg(options.vt_type)
def postprocess_image(test, params, image_name):
"""
Postprocess a single QEMU image according to the instructions in params.
:param test: An Autotest test object.
:param params: A dict containing image postprocessing parameters.
"""
clone_master = params.get("clone_master", None)
base_dir = data_dir.get_data_dir()
image = qemu_storage.QemuImg(params, base_dir, image_name)
if params.get("check_image") == "yes":
try:
if clone_master is None:
image.check_image(params, base_dir)
elif clone_master == "yes":
if image_name in params.get("master_images_clone").split():
image.check_image(params, base_dir)
if params.get("restore_image", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
except Exception, e:
if params.get("restore_image_on_check_error", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
if params.get("remove_image_on_check_error", "no") == "yes":
cl_images = params.get("master_images_clone", "")
if image_name in cl_images.split():
image.remove()
raise e
def preprocess_image(test, params, image_name):
"""
Preprocess a single QEMU image according to the instructions in params.
@param test: Autotest test object.
@param params: A dict containing image preprocessing parameters.
@note: Currently this function just creates an image if requested.
"""
base_dir = data_dir.get_data_dir()
if params.get("storage_type") == "iscsi":
iscsidev = kvm_storage.Iscsidev(params, base_dir, image_name)
params["image_name"] = iscsidev.setup()
else:
image_filename = storage.get_image_filename(params,
base_dir)
create_image = False
if params.get("force_create_image") == "yes":
create_image = True
elif (params.get("create_image") == "yes" and not
os.path.exists(image_filename)):
create_image = True
if create_image:
image = kvm_storage.QemuImg(params, base_dir, image_name)
if not image.create(params):
raise error.TestError("Could not create image")
def postprocess_image(test, params, image_name):
"""
Postprocess a single QEMU image according to the instructions in params.
:param test: An Autotest test object.
:param params: A dict containing image postprocessing parameters.
"""
clone_master = params.get("clone_master", None)
base_dir = data_dir.get_data_dir()
image = qemu_storage.QemuImg(params, base_dir, image_name)
if params.get("check_image") == "yes":
try:
if clone_master is None:
image.check_image(params, base_dir)
elif clone_master == "yes":
if image_name in params.get("master_images_clone").split():
image.check_image(params, base_dir)
if params.get("restore_image", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
except Exception, e:
if params.get("restore_image_on_check_error", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
if params.get("remove_image_on_check_error", "no") == "yes":
cl_images = params.get("master_images_clone", "")
if image_name in cl_images.split():
image.remove()
raise e
def preprocess_image(test, params, image_name):
"""
Preprocess a single QEMU image according to the instructions in params.
@param test: Autotest test object.
@param params: A dict containing image preprocessing parameters.
@note: Currently this function just creates an image if requested.
"""
base_dir = data_dir.get_data_dir()
if params.get("storage_type") == "iscsi":
iscsidev = qemu_storage.Iscsidev(params, base_dir, image_name)
params["image_name"] = iscsidev.setup()
else:
image_filename = storage.get_image_filename(params,
base_dir)
create_image = False
if params.get("force_create_image") == "yes":
create_image = True
elif (params.get("create_image") == "yes" and not
os.path.exists(image_filename)):
create_image = True
if create_image:
image = qemu_storage.QemuImg(params, base_dir, image_name)
if not image.create(params):
raise error.TestError("Could not create image")
def postprocess_image(test, params, image_name, vm_process_status=None):
"""
Postprocess a single QEMU image according to the instructions in params.
:param test: An Autotest test object.
:param params: A dict containing image postprocessing parameters.
:param vm_process_status: (optional) vm process status like running, dead
or None for no vm exist.
"""
clone_master = params.get("clone_master", None)
base_dir = data_dir.get_data_dir()
image = qemu_storage.QemuImg(params, base_dir, image_name)
check_image_flag = params.get("check_image") == "yes"
if vm_process_status == "running" and check_image_flag:
if params.get("skip_image_check_during_running") == "yes":
logging.debug("Guest is still running, skip the image check.")
check_image_flag = False
else:
image_info_output = image.info()
image_info = {}
for image_info_item in image_info_output.splitlines():
option = image_info_item.split(":")
if len(option) == 2:
image_info[option[0].strip()] = option[1].strip()
if ("lazy refcounts" in image_info
and image_info["lazy refcounts"] == "true"):
logging.debug("Should not check image while guest is alive"
" when the image is create with lazy refcounts."
" Skip the image check.")
check_image_flag = False
if check_image_flag:
try:
if clone_master is None:
image.check_image(params, base_dir)
elif clone_master == "yes":
if image_name in params.get("master_images_clone").split():
image.check_image(params, base_dir)
# Allow test to overwrite any pre-testing automatic backup
# with a new backup. i.e. assume pre-existing image/backup
# would not be usable after this test succeeds. The best
# example for this is when 'unattended_install' is run.
if params.get("backup_image", "no") == "yes":
image.backup_image(params, base_dir, "backup", True)
elif params.get("restore_image", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
except Exception, e:
if params.get("restore_image_on_check_error", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
if params.get("remove_image_on_check_error", "no") == "yes":
cl_images = params.get("master_images_clone", "")
if image_name in cl_images.split():
image.remove()
if (params.get("skip_cluster_leak_warn") == "yes"
and "Leaked clusters" in e.message):
logging.warn(e.message)
else:
raise e
def postprocess_image(test, params, image_name, vm_process_status=None):
"""
Postprocess a single QEMU image according to the instructions in params.
:param test: An Autotest test object.
:param params: A dict containing image postprocessing parameters.
:param vm_process_status: (optional) vm process status like running, dead
or None for no vm exist.
"""
clone_master = params.get("clone_master", None)
base_dir = data_dir.get_data_dir()
image = qemu_storage.QemuImg(params, base_dir, image_name)
check_image_flag = params.get("check_image") == "yes"
if vm_process_status == "running" and check_image_flag:
if params.get("skip_image_check_during_running") == "yes":
logging.debug("Guest is still running, skip the image check.")
check_image_flag = False
else:
image_info_output = image.info()
image_info = {}
for image_info_item in image_info_output.splitlines():
option = image_info_item.split(":")
if len(option) == 2:
image_info[option[0].strip()] = option[1].strip()
if ("lazy refcounts" in image_info
and image_info["lazy refcounts"] == "true"):
logging.debug("Should not check image while guest is alive"
" when the image is create with lazy refcounts."
" Skip the image check.")
check_image_flag = False
if check_image_flag:
try:
if clone_master is None:
image.check_image(params, base_dir)
elif clone_master == "yes":
if image_name in params.get("master_images_clone").split():
image.check_image(params, base_dir)
# Allow test to overwrite any pre-testing automatic backup
# with a new backup. i.e. assume pre-existing image/backup
# would not be usable after this test succeeds. The best
# example for this is when 'unattended_install' is run.
if params.get("backup_image", "no") == "yes":
image.backup_image(params, base_dir, "backup", True)
elif params.get("restore_image", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
except Exception, e:
if params.get("restore_image_on_check_error", "no") == "yes":
image.backup_image(params, base_dir, "restore", True)
if params.get("remove_image_on_check_error", "no") == "yes":
cl_images = params.get("master_images_clone", "")
if image_name in cl_images.split():
image.remove()
if (params.get("skip_cluster_leak_warn") == "yes"
and "Leaked clusters" in e.message):
logging.warn(e.message)
else:
raise e
def get_asset_info(asset):
asset_path = os.path.join(data_dir.get_download_dir(), '%s.ini' % asset)
asset_cfg = ConfigParser.ConfigParser()
asset_cfg.read(asset_path)
url = asset_cfg.get(asset, 'url')
try:
sha1_url = asset_cfg.get(asset, 'sha1_url')
except ConfigParser.NoOptionError:
sha1_url = None
title = asset_cfg.get(asset, 'title')
destination = asset_cfg.get(asset, 'destination')
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
asset_exists = os.path.isfile(destination)
# Optional fields
try:
destination_uncompressed = asset_cfg.get(asset,
'destination_uncompressed')
if not os.path.isabs(destination_uncompressed):
destination_uncompressed = os.path.join(data_dir.get_data_dir(),
destination_uncompressed)
except:
destination_uncompressed = None
try:
uncompress_cmd = asset_cfg.get(asset, 'uncompress_cmd')
except:
uncompress_cmd = None
return {
'url': url,
'sha1_url': sha1_url,
'destination': destination,
'destination_uncompressed': destination_uncompressed,
'uncompress_cmd': uncompress_cmd,
'shortname': asset,
'title': title,
'downloaded': asset_exists
}
def check_disk_params(self, params):
"""
Check gathered info from qtree/block with params
@param params: autotest params
@return: number of errors
"""
err = 0
disks = {}
for disk in self.disks:
if isinstance(disk, QtreeDisk):
disks[disk.get_qname()] = disk.get_params().copy()
# We don't have the params name so we need to map file_names instead
qname = None
for name in params.objects('images'):
current = None
image_params = params.object_params(name)
image_name = os.path.realpath(
storage.get_image_filename(image_params,
data_dir.get_data_dir()))
for (qname, disk) in disks.iteritems():
if disk.get('image_name') == image_name:
current = disk
# autotest params might use relative path
current['image_name'] = image_params.get('image_name')
break
if not current:
logging.error("Disk %s is not in qtree but is in params.",
name)
err += 1
continue
for prop in current.iterkeys():
handled = False
if prop == "drive_format":
# HOOK: params to qemu translation
if current.get(prop).startswith(image_params.get(prop)):
handled = True
elif (image_params.get(prop)
and image_params.get(prop) == current.get(prop)):
handled = True
if not handled:
logging.error(
"Disk %s property %s=%s doesn't match params"
" %s", qname, prop, current.get(prop),
image_params.get(prop))
err += 1
disks.pop(qname)
if disks:
logging.error(
'Some disks were in qtree but not in autotest params'
': %s', disks)
err += 1
return err
def get_asset_info(asset):
asset_info = {}
asset_path = os.path.join(data_dir.get_download_dir(), '%s.ini' % asset)
asset_cfg = test_config.config_loader(asset_path)
asset_info['url'] = asset_cfg.get(asset, 'url')
asset_info['sha1_url'] = asset_cfg.get(asset, 'sha1_url')
asset_info['title'] = asset_cfg.get(asset, 'title')
destination = asset_cfg.get(asset, 'destination')
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
asset_info['destination'] = destination
asset_info['asset_exists'] = os.path.isfile(destination)
# Optional fields
d_uncompressed = asset_cfg.get(asset, 'destination_uncompressed')
if d_uncompressed is not None and not os.path.isabs(d_uncompressed):
d_uncompressed = os.path.join(data_dir.get_data_dir(), d_uncompressed)
asset_info['destination_uncompressed'] = d_uncompressed
asset_info['uncompress_cmd'] = asset_cfg.get(asset, 'uncompress_cmd')
return asset_info
def get_asset_info(asset):
asset_info = {}
asset_path = os.path.join(data_dir.get_download_dir(), "%s.ini" % asset)
asset_cfg = test_config.config_loader(asset_path)
asset_info["url"] = asset_cfg.get(asset, "url")
asset_info["sha1_url"] = asset_cfg.get(asset, "sha1_url")
asset_info["title"] = asset_cfg.get(asset, "title")
destination = asset_cfg.get(asset, "destination")
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
asset_info["destination"] = destination
asset_info["asset_exists"] = os.path.isfile(destination)
# Optional fields
d_uncompressed = asset_cfg.get(asset, "destination_uncompressed")
if d_uncompressed is not None and not os.path.isabs(d_uncompressed):
d_uncompressed = os.path.join(data_dir.get_data_dir(), d_uncompressed)
asset_info["destination_uncompressed"] = d_uncompressed
asset_info["uncompress_cmd"] = asset_cfg.get(asset, "uncompress_cmd")
return asset_info
def get_asset_info(asset):
asset_info = {}
asset_path = os.path.join(data_dir.get_download_dir(), '%s.ini' % asset)
asset_cfg = test_config.config_loader(asset_path)
asset_info['url'] = asset_cfg.get(asset, 'url')
asset_info['sha1_url'] = asset_cfg.get(asset, 'sha1_url')
asset_info['title'] = asset_cfg.get(asset, 'title')
destination = asset_cfg.get(asset, 'destination')
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
asset_info['destination'] = destination
asset_info['asset_exists'] = os.path.isfile(destination)
# Optional fields
d_uncompressed = asset_cfg.get(asset, 'destination_uncompressed')
if d_uncompressed is not None and not os.path.isabs(d_uncompressed):
d_uncompressed = os.path.join(data_dir.get_data_dir(),
d_uncompressed)
asset_info['destination_uncompressed'] = d_uncompressed
asset_info['uncompress_cmd'] = asset_cfg.get(asset, 'uncompress_cmd')
return asset_info
def check_disk_params(self, params):
"""
Check gathered info from qtree/block with params
@param params: autotest params
@return: number of errors
"""
err = 0
disks = {}
for disk in self.disks:
if isinstance(disk, QtreeDisk):
disks[disk.get_qname()] = disk.get_params().copy()
# We don't have the params name so we need to map file_names instead
qname = None
for name in params.objects('images'):
current = None
image_params = params.object_params(name)
image_name = os.path.realpath(
storage.get_image_filename(image_params,
data_dir.get_data_dir()))
for (qname, disk) in disks.iteritems():
if disk.get('image_name') == image_name:
current = disk
# autotest params might use relative path
current['image_name'] = image_params.get('image_name')
break
if not current:
logging.error("Disk %s is not in qtree but is in params.", name)
err += 1
continue
for prop in current.iterkeys():
handled = False
if prop == "drive_format":
# HOOK: params to qemu translation
if current.get(prop).startswith(image_params.get(prop)):
handled = True
elif (image_params.get(prop) and
image_params.get(prop) == current.get(prop)):
handled = True
if not handled:
logging.error("Disk %s property %s=%s doesn't match params"
" %s", qname, prop, current.get(prop),
image_params.get(prop))
err += 1
disks.pop(qname)
if disks:
logging.error('Some disks were in qtree but not in autotest params'
': %s', disks)
err += 1
return err
def get_asset_info(asset):
asset_path = os.path.join(data_dir.get_download_dir(), "%s.ini" % asset)
asset_cfg = ConfigParser.ConfigParser()
asset_cfg.read(asset_path)
url = asset_cfg.get(asset, "url")
try:
sha1_url = asset_cfg.get(asset, "sha1_url")
except ConfigParser.NoOptionError:
sha1_url = None
title = asset_cfg.get(asset, "title")
destination = asset_cfg.get(asset, "destination")
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
asset_exists = os.path.isfile(destination)
# Optional fields
try:
destination_uncompressed = asset_cfg.get(asset, "destination_uncompressed")
if not os.path.isabs(destination_uncompressed):
destination_uncompressed = os.path.join(data_dir.get_data_dir(), destination)
uncompress_cmd = asset_cfg.get(asset, "uncompress_cmd")
except:
destination_uncompressed = None
uncompress_cmd = None
return {
"url": url,
"sha1_url": sha1_url,
"destination": destination,
"destination_uncompressed": destination_uncompressed,
"uncompress_cmd": uncompress_cmd,
"shortname": asset,
"title": title,
"downloaded": asset_exists,
}
def download_asset(asset, interactive=True, restore_image=False):
"""
Download an asset defined on an asset file.
Asset files are located under /shared/download.d, are .ini files with the
following keys defined:
title: Title string to display in the download progress bar.
url = URL of the resource
sha1_url = URL with SHA1 information for the resource, in the form
sha1sum file_basename
destination = Location of your file relative to the data directory
(TEST_SUITE_ROOT/shared/data)
destination = Location of the uncompressed file relative to the data
directory (TEST_SUITE_ROOT/shared/data)
uncompress_cmd = Command that needs to be executed with the compressed
file as a parameter
@param asset: String describing an asset file.
@param interactive: Whether to ask the user before downloading the file.
@param restore_image: If the asset is a compressed image, we can uncompress
in order to restore the image.
"""
asset_info = get_asset_info(asset)
destination = os.path.join(data_dir.get_data_dir(),
asset_info['destination'])
if (interactive and not os.path.isfile(destination)):
answer = utils.ask("File %s not present. Do you want to download it?" %
asset_info['title'])
else:
answer = "y"
if answer == "y":
had_to_download = download_file(asset=asset, interactive=interactive)
requires_uncompress = asset_info['uncompress_cmd'] is not None
if requires_uncompress:
destination_uncompressed = asset_info['destination_uncompressed']
uncompressed_file_exists = os.path.exists(destination_uncompressed)
restore_image = (restore_image or had_to_download or not
uncompressed_file_exists)
if os.path.isfile(destination) and restore_image:
os.chdir(os.path.dirname(destination))
uncompress_cmd = asset_info['uncompress_cmd']
utils.run("%s %s" % (uncompress_cmd, destination))
def get_file_asset(title, src_path, destination):
if not os.path.isabs(destination):
destination = os.path.join(data_dir.get_data_dir(), destination)
for ext in (".xz", ".gz", ".7z", ".bz2"):
if os.path.exists(src_path + ext):
destination = destination + ext
logging.debug('Found source image %s', destination)
return {'url': None, 'sha1_url': None, 'destination': src_path + ext,
'destination_uncompressed': destination,
'uncompress_cmd': None, 'shortname': title, 'title': title,
'downloaded': True}
if os.path.exists(src_path):
logging.debug('Found source image %s', destination)
return {'url': src_path, 'sha1_url': None, 'destination': destination,
'destination_uncompressed': None, 'uncompress_cmd': None,
'shortname': title, 'title': title,
'downloaded': os.path.exists(destination)}
return None
def _parse_params(self):
'''
Parses the params items for entries related to guest kernel
'''
configure_opt_key = '%s_config' % self.prefix
self.config = self.params.get(configure_opt_key, '')
build_image_key = '%s_build_image' % self.prefix
self.build_image = self.params.get(build_image_key,
'arch/x86/boot/bzImage')
build_target_key = '%s_build_target' % self.prefix
self.build_target = self.params.get(build_target_key, 'bzImage')
kernel_path_key = '%s_kernel_path' % self.prefix
image_dir = os.path.join(data_dir.get_data_dir(), 'images')
default_kernel_path = os.path.join(image_dir, self.build_target)
self.kernel_path = self.params.get(kernel_path_key,
default_kernel_path)
logging.info('Parsing Linux kernel build parameters for %s',
self.prefix)
def create_config_files(options):
"""
Check if the appropriate configuration files are present.
If the files are not present, create them.
@param options: OptParser object with options.
"""
shared_dir = os.path.dirname(data_dir.get_data_dir())
test_dir = os.path.dirname(shared_dir)
shared_dir = os.path.join(shared_dir, "cfg")
if (options.type and options.config):
test_dir = os.path.join(test_dir, options.type)
elif options.type:
test_dir = os.path.join(test_dir, options.type)
elif options.config:
parent_config_dir = os.path.dirname(options.config)
parent_config_dir = os.path.basename(parent_config_dir)
test_dir = os.path.join(test_dir, parent_config_dir)
bootstrap.create_config_files(test_dir, shared_dir, interactive=False)
def create_config_files(options):
"""
Check if the appropriate configuration files are present.
If the files are not present, create them.
@param options: OptParser object with options.
"""
shared_dir = os.path.dirname(data_dir.get_data_dir())
test_dir = os.path.dirname(shared_dir)
shared_dir = os.path.join(shared_dir, "cfg")
if (options.type and options.config):
test_dir = os.path.join(test_dir, options.type)
elif options.type:
test_dir = os.path.join(test_dir, options.type)
elif options.config:
parent_config_dir = os.path.dirname(options.config)
parent_config_dir = os.path.basename(parent_config_dir)
test_dir = os.path.join(test_dir, parent_config_dir)
utils_misc.create_config_files(test_dir, shared_dir, interactive=False)
def _parse_params(self):
'''
Parses the params items for entries related to guest kernel
'''
configure_opt_key = '%s_config' % self.prefix
self.config = self.params.get(configure_opt_key, '')
build_image_key = '%s_build_image' % self.prefix
self.build_image = self.params.get(build_image_key,
'arch/x86/boot/bzImage')
build_target_key = '%s_build_target' % self.prefix
self.build_target = self.params.get(build_target_key, 'bzImage')
kernel_path_key = '%s_kernel_path' % self.prefix
image_dir = os.path.join(data_dir.get_data_dir(), 'images')
default_kernel_path = os.path.join(image_dir,
self.build_target)
self.kernel_path = self.params.get(kernel_path_key,
default_kernel_path)
logging.info('Parsing Linux kernel build parameters for %s',
self.prefix)
def create_config_files(options):
"""
Check if the appropriate configuration files are present.
If the files are not present, create them.
@param options: OptParser object with options.
"""
shared_dir = os.path.dirname(data_dir.get_data_dir())
test_dir = os.path.dirname(shared_dir)
if (options.type and options.config):
test_dir = os.path.join(test_dir, options.type)
elif options.type:
test_dir = os.path.join(test_dir, options.type)
elif options.config:
parent_config_dir = os.path.dirname(options.config)
parent_config_dir = os.path.dirname(parent_config_dir)
options.type = parent_config_dir
test_dir = os.path.join(test_dir, parent_config_dir)
bootstrap.create_config_files(test_dir, shared_dir, interactive=False)
bootstrap.create_subtests_cfg(options.type)
bootstrap.create_guest_os_cfg(options.type)
def preprocess(test, params, env):
"""
Preprocess all VMs and images according to the instructions in params.
Also, collect some host information, such as the KVM version.
:param test: An Autotest test object.
:param params: A dict containing all VM and image parameters.
:param env: The environment (a dict-like object).
"""
error.context("preprocessing")
# First, let's verify if this test does require root or not. If it
# does and the test suite is running as a regular user, we shall just
# throw a TestNAError exception, which will skip the test.
if params.get('requires_root', 'no') == 'yes':
utils_misc.verify_running_as_root()
port = params.get('shell_port')
prompt = params.get('shell_prompt')
address = params.get('ovirt_node_address')
username = params.get('ovirt_node_user')
password = params.get('ovirt_node_password')
setup_pb = False
for nic in params.get('nics', "").split():
nic_params = params.object_params(nic)
if nic_params.get('netdst') == 'private':
setup_pb = True
params_pb = nic_params
params['netdst_%s' % nic] = nic_params.get("priv_brname", 'atbr0')
if setup_pb:
brcfg = test_setup.PrivateBridgeConfig(params_pb)
brcfg.setup()
base_dir = data_dir.get_data_dir()
if params.get("storage_type") == "iscsi":
iscsidev = qemu_storage.Iscsidev(params, base_dir, "iscsi")
params["image_name"] = iscsidev.setup()
params["image_raw_device"] = "yes"
if params.get("storage_type") == "lvm":
lvmdev = qemu_storage.LVMdev(params, base_dir, "lvm")
params["image_name"] = lvmdev.setup()
params["image_raw_device"] = "yes"
env.register_lvmdev("lvm_%s" % params["main_vm"], lvmdev)
if params.get("storage_type") == "nfs":
image_nfs = nfs.Nfs(params)
image_nfs.setup()
image_name_only = os.path.basename(params["image_name"])
params['image_name'] = os.path.join(image_nfs.mount_dir,
image_name_only)
for image_name in params.objects("images"):
name_tag = "image_name_%s" % image_name
if params.get(name_tag):
image_name_only = os.path.basename(params[name_tag])
params[name_tag] = os.path.join(image_nfs.mount_dir,
image_name_only)
# Start tcpdump if it isn't already running
# The fact it has to be started here is so that the test params
# have to be honored.
env.start_tcpdump(params)
# Destroy and remove VMs that are no longer needed in the environment
requested_vms = params.objects("vms")
for key in env.keys():
vm = env[key]
if not isinstance(vm, virt_vm.BaseVM):
continue
if vm.name not in requested_vms:
vm.destroy()
del env[key]
if (params.get("auto_cpu_model") == "yes"
and params.get("vm_type") == "qemu"):
if not env.get("cpu_model"):
env["cpu_model"] = utils_misc.get_qemu_best_cpu_model(params)
params["cpu_model"] = env.get("cpu_model")
kvm_ver_cmd = params.get("kvm_ver_cmd", "")
if kvm_ver_cmd:
try:
cmd_result = utils.run(kvm_ver_cmd)
kvm_version = cmd_result.stdout.strip()
except error.CmdError:
kvm_version = "Unknown"
else:
# Get the KVM kernel module version and write it as a keyval
if os.path.exists("/dev/kvm"):
try:
kvm_version = open("/sys/module/kvm/version").read().strip()
except Exception:
kvm_version = os.uname()[2]
else:
logging.warning("KVM module not loaded")
kvm_version = "Unknown"
logging.debug("KVM version: %s" % kvm_version)
test.write_test_keyval({"kvm_version": kvm_version})
# Get the KVM userspace version and write it as a keyval
kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")
if kvm_userspace_ver_cmd:
try:
cmd_result = utils.run(kvm_userspace_ver_cmd)
kvm_userspace_version = cmd_result.stdout.strip()
except error.CmdError:
kvm_userspace_version = "Unknown"
else:
qemu_path = utils_misc.get_qemu_binary(params)
version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
matches = re.findall("[Vv]ersion .*?,", version_line)
if matches:
kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
else:
kvm_userspace_version = "Unknown"
logging.debug("KVM userspace version: %s" % kvm_userspace_version)
test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
if params.get("setup_hugepages") == "yes":
h = test_setup.HugePageConfig(params)
suggest_mem = h.setup()
if suggest_mem is not None:
params['mem'] = suggest_mem
if params.get("vm_type") == "libvirt":
utils_libvirtd.libvirtd_restart()
if params.get("setup_thp") == "yes":
thp = test_setup.TransparentHugePageConfig(test, params)
thp.setup()
if params.get("setup_ksm") == "yes":
ksm = test_setup.KSMConfig(params, env)
ksm.setup(env)
# Execute any pre_commands
if params.get("pre_command"):
process_command(test, params, env, params.get("pre_command"),
int(params.get("pre_command_timeout", "600")),
params.get("pre_command_noncritical") == "yes")
# if you want set "pci=nomsi" before test, set "disable_pci_msi = yes"
# and pci_msi_sensitive = "yes"
if params.get("pci_msi_sensitive", "no") == "yes":
disable_pci_msi = params.get("disable_pci_msi", "no")
image_filename = storage.get_image_filename(params,
data_dir.get_data_dir())
grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
r".*vmlinuz-\d+.*")
msi_keyword = params.get("msi_keyword", " pci=nomsi")
disk_obj = utils_disk.GuestFSModiDisk(image_filename)
kernel_config_ori = disk_obj.read_file(grub_file)
kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
if not kernel_config:
raise error.TestError("Cannot find the kernel config, reg is %s" %
kernel_cfg_pos_reg)
kernel_config_line = kernel_config[0]
kernel_need_modify = False
if disable_pci_msi == "yes":
if not re.findall(msi_keyword, kernel_config_line):
kernel_config_set = kernel_config_line + msi_keyword
kernel_need_modify = True
else:
if re.findall(msi_keyword, kernel_config_line):
kernel_config_set = re.sub(msi_keyword, "", kernel_config_line)
kernel_need_modify = True
if kernel_need_modify:
for vm in env.get_all_vms():
if vm:
vm.destroy()
env.unregister_vm(vm.name)
disk_obj.replace_image_file_content(grub_file, kernel_config_line,
kernel_config_set)
logging.debug("Guest cmdline 'pci=nomsi' setting is: [ %s ]" %
disable_pci_msi)
kernel_extra_params = params.get("kernel_extra_params")
if kernel_extra_params:
image_filename = storage.get_image_filename(params,
data_dir.get_data_dir())
grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
r".*vmlinuz-\d+.*")
disk_obj = utils_disk.GuestFSModiDisk(image_filename)
kernel_config_ori = disk_obj.read_file(grub_file)
kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
if not kernel_config:
raise error.TestError("Cannot find the kernel config, reg is %s" %
kernel_cfg_pos_reg)
kernel_config_line = kernel_config[0]
kernel_need_modify = False
if not re.findall(kernel_extra_params, kernel_config_line):
kernel_config_set = kernel_config_line + kernel_extra_params
kernel_need_modify = True
if kernel_need_modify:
for vm in env.get_all_vms():
if vm:
vm.destroy()
env.unregister_vm(vm.name)
disk_obj.replace_image_file_content(grub_file, kernel_config_line,
kernel_config_set)
logging.debug("Guest cmdline extra_params setting is: [ %s ]" %
kernel_extra_params)
# Clone master image from vms.
base_dir = data_dir.get_data_dir()
if params.get("master_images_clone"):
for vm_name in params.get("vms").split():
vm = env.get_vm(vm_name)
if vm:
vm.destroy()
env.unregister_vm(vm_name)
vm_params = params.object_params(vm_name)
for image in vm_params.get("master_images_clone").split():
image_obj = qemu_storage.QemuImg(params, base_dir, image)
image_obj.clone_image(params, vm_name, image, base_dir)
# Preprocess all VMs and images
if params.get("not_preprocess", "no") == "no":
process(test, params, env, preprocess_image, preprocess_vm)
# Start the screendump thread
if params.get("take_regular_screendumps") == "yes":
global _screendump_thread, _screendump_thread_termination_event
_screendump_thread_termination_event = threading.Event()
_screendump_thread = threading.Thread(target=_take_screendumps,
name='ScreenDump',
args=(test, params, env))
_screendump_thread.start()
# Start the register query thread
if params.get("store_vm_register") == "yes":
global _vm_register_thread, _vm_register_thread_termination_event
_vm_register_thread_termination_event = threading.Event()
_vm_register_thread = threading.Thread(target=_store_vm_register,
name='VmRegister',
args=(test, params, env))
_vm_register_thread.start()
return params
def create(self, name=None, params=None, root_dir=None, timeout=5.0,
migration_mode=None, mac_source=None):
"""
Start the VM by running a qemu command.
All parameters are optional. If name, params or root_dir are not
supplied, the respective values stored as class attributes are used.
@param name: The name of the object
@param params: A dict containing VM params
@param root_dir: Base directory for relative filenames
@param migration_mode: If supplied, start VM for incoming migration
using this protocol (either 'tcp', 'unix' or 'exec')
@param migration_exec_cmd: Command to embed in '-incoming "exec: ..."'
(e.g. 'gzip -c -d filename') if migration_mode is 'exec'
@param mac_source: A VM object from which to copy MAC addresses. If not
specified, new addresses will be generated.
@raise VMCreateError: If qemu terminates unexpectedly
@raise VMKVMInitError: If KVM initialization fails
@raise VMHugePageError: If hugepage initialization fails
@raise VMImageMissingError: If a CD image is missing
@raise VMHashMismatchError: If a CD image hash has doesn't match the
expected hash
@raise VMBadPATypeError: If an unsupported PCI assignment type is
requested
@raise VMPAError: If no PCI assignable devices could be assigned
"""
error.context("creating '%s'" % self.name)
self.destroy(free_mac_addresses=False)
if name is not None:
self.name = name
if params is not None:
self.params = params
if root_dir is not None:
self.root_dir = root_dir
name = self.name
params = self.params
root_dir = self.root_dir
# Verify the md5sum of the ISO images
for cdrom in params.objects("cdroms"):
if params.get("medium") == "import":
break
cdrom_params = params.object_params(cdrom)
iso = cdrom_params.get("cdrom")
if ((self.driver_type == 'xen') and
(params.get('hvm_or_pv') == 'pv') and
(os.path.basename(iso) == 'ks.iso')):
continue
if iso:
iso = utils_misc.get_path(data_dir.get_data_dir(), iso)
if not os.path.exists(iso):
raise virt_vm.VMImageMissingError(iso)
compare = False
if cdrom_params.get("md5sum_1m"):
logging.debug("Comparing expected MD5 sum with MD5 sum of "
"first MB of ISO file...")
actual_hash = utils.hash_file(iso, 1048576, method="md5")
expected_hash = cdrom_params.get("md5sum_1m")
compare = True
elif cdrom_params.get("md5sum"):
logging.debug("Comparing expected MD5 sum with MD5 sum of "
"ISO file...")
actual_hash = utils.hash_file(iso, method="md5")
expected_hash = cdrom_params.get("md5sum")
compare = True
elif cdrom_params.get("sha1sum"):
logging.debug("Comparing expected SHA1 sum with SHA1 sum "
"of ISO file...")
actual_hash = utils.hash_file(iso, method="sha1")
expected_hash = cdrom_params.get("sha1sum")
compare = True
if compare:
if actual_hash == expected_hash:
logging.debug("Hashes match")
else:
raise virt_vm.VMHashMismatchError(actual_hash,
expected_hash)
# Make sure the following code is not executed by more than one thread
# at the same time
lockfile = open("/tmp/libvirt-autotest-vm-create.lock", "w+")
fcntl.lockf(lockfile, fcntl.LOCK_EX)
try:
# Handle port redirections
redir_names = params.objects("redirs")
host_ports = utils_misc.find_free_ports(5000, 6000, len(redir_names))
self.redirs = {}
for i in range(len(redir_names)):
redir_params = params.object_params(redir_names[i])
guest_port = int(redir_params.get("guest_port"))
self.redirs[guest_port] = host_ports[i]
# Find available PCI devices
self.pci_devices = []
for device in params.objects("pci_devices"):
self.pci_devices.append(device)
# Find available VNC port, if needed
if params.get("display") == "vnc":
if params.get("vnc_autoport") == "yes":
self.vnc_port = None
self.vnc_autoport = True
else:
self.vnc_port = utils_misc.find_free_port(5900, 6100)
self.vnc_autoport = False
# Find available spice port, if needed
if params.get("spice"):
self.spice_port = utils_misc.find_free_port(8000, 8100)
# Find random UUID if specified 'uuid = random' in config file
if params.get("uuid") == "random":
f = open("/proc/sys/kernel/random/uuid")
self.uuid = f.read().strip()
f.close()
# Generate or copy MAC addresses for all NICs
for nic in self.virtnet:
nic_params = dict(nic)
if mac_source:
# Will raise exception if source doesn't
# have cooresponding nic
logging.debug("Copying mac for nic %s from VM %s"
% (nic.nic_name, mac_source.nam))
nic_params['mac'] = mac_source.get_mac_address(nic.nic_name)
# make_create_command() calls vm.add_nic (i.e. on a copy)
nic = self.add_nic(**nic_params)
logging.debug('VM.create activating nic %s' % nic)
self.activate_nic(nic.nic_name)
# Make qemu command
install_command = self.make_create_command()
logging.info("Running libvirt command (reformatted):")
for item in install_command.replace(" -", " \n -").splitlines():
logging.info("%s", item)
utils.run(install_command, verbose=False)
# Wait for the domain to be created
utils_misc.wait_for(func=self.is_alive, timeout=60,
text=("waiting for domain %s to start" %
self.name))
self.uuid = virsh.domuuid(self.name, uri=self.connect_uri)
# Establish a session with the serial console
if self.only_pty == True:
self.serial_console = aexpect.ShellSession(
"virsh console %s" % self.name,
auto_close=False,
output_func=utils_misc.log_line,
output_params=("serial-%s.log" % name,))
else:
self.serial_console = aexpect.ShellSession(
"tail -f %s" % self.get_serial_console_filename(),
auto_close=False,
output_func=utils_misc.log_line,
output_params=("serial-%s.log" % name,))
finally:
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
def run_tests(parser, options):
"""
Runs the sequence of KVM tests based on the list of dctionaries
generated by the configuration system, handling dependencies.
@param parser: Config parser object.
@return: True, if all tests ran passed, False if any of them failed.
"""
debugdir = os.path.join(data_dir.get_root_dir(), 'logs',
'run-%s' % time.strftime('%Y-%m-%d-%H.%M.%S'))
if not os.path.isdir(debugdir):
os.makedirs(debugdir)
debuglog = os.path.join(debugdir, "debug.log")
configure_file_logging(debuglog)
print_stdout(bcolors.HEADER +
"DATA DIR: %s" % data_dir.get_backing_data_dir() +
bcolors.ENDC)
print_header("DEBUG LOG: %s" % debuglog)
last_index = -1
logging.info("Starting test job at %s" % time.strftime('%Y-%m-%d %H:%M:%S'))
logging.info("")
logging.debug("Options received from the command line:")
utils_misc.display_attributes(options)
logging.debug("")
logging.debug("Cleaning up previous job tmp files")
d = parser.get_dicts().next()
env_filename = os.path.join(data_dir.get_root_dir(),
options.type, d.get("env", "env"))
env = utils_env.Env(env_filename, Test.env_version)
env.destroy()
try:
address_pool_files = glob.glob("/tmp/address_pool*")
for address_pool_file in address_pool_files:
os.remove(address_pool_file)
aexpect_tmp = "/tmp/aexpect_spawn/"
if os.path.isdir(aexpect_tmp):
shutil.rmtree("/tmp/aexpect_spawn/")
except (IOError, OSError):
pass
logging.debug("")
if options.restore_image_between_tests:
logging.debug("Creating first backup of guest image")
qemu_img = storage.QemuImg(d, data_dir.get_data_dir(), "image")
qemu_img.backup_image(d, data_dir.get_data_dir(), 'backup', True)
logging.debug("")
if options.type == 'qemu':
logging.info("We're running the qemu test with:")
logging.info("qemu binary: %s" % d.get('qemu_binary'))
logging.info("qemu img binary: %s" % d.get('qemu_img_binary'))
logging.info("qemu io binary: %s" % d.get('qemu_io_binary'))
logging.info("")
logging.info("Defined test set:")
for i, d in enumerate(parser.get_dicts()):
if options.config is None and options.type in TEST_TYPES_STRIP_NAMES:
shortname = ".".join(d['name'].split(".")[12:])
else:
shortname = ".".join(d['shortname'].split("."))
logging.info("Test %4d: %s" % (i + 1, shortname))
last_index += 1
if last_index == -1:
print_stdout("No tests generated by config file %s" % parser.filename)
print_stdout("Please check the file for errors (bad variable names, "
"wrong indentation)")
sys.exit(-1)
logging.info("")
n_tests = last_index + 1
print_header("TESTS: %s" % n_tests)
status_dct = {}
failed = False
# Add the parameter decide if setup host env in the test case
# For some special tests we only setup host in the first and last case
# When we need to setup host env we need the host_setup_flag as following:
# 0(00): do nothing
# 1(01): setup env
# 2(10): cleanup env
# 3(11): setup and cleanup env
index = 0
setup_flag = 1
cleanup_flag = 2
for dct in parser.get_dicts():
if options.config is None and options.type in TEST_TYPES_STRIP_NAMES:
shortname = ".".join(d['name'].split(".")[12:])
else:
shortname = ".".join(d['shortname'].split("."))
if index == 0:
if dct.get("host_setup_flag", None) is not None:
flag = int(dct["host_setup_flag"])
dct["host_setup_flag"] = flag | setup_flag
else:
dct["host_setup_flag"] = setup_flag
if index == last_index:
if dct.get("host_setup_flag", None) is not None:
flag = int(dct["host_setup_flag"])
dct["host_setup_flag"] = flag | cleanup_flag
else:
dct["host_setup_flag"] = cleanup_flag
index += 1
# Add kvm module status
dct["kvm_default"] = utils_misc.get_module_params(
dct.get("sysfs_dir", "sys"), "kvm")
if dct.get("skip") == "yes":
continue
dependencies_satisfied = True
for dep in dct.get("dep"):
for test_name in status_dct.keys():
if not dep in test_name:
continue
if not status_dct[test_name]:
dependencies_satisfied = False
break
current_status = False
if dependencies_satisfied:
t = Test(dct, options)
t.set_debugdir(debugdir)
pretty_index = "(%d/%d)" % (index, n_tests)
print_stdout("%s %s:" % (pretty_index, t.tag), end=False)
try:
try:
t_begin = time.time()
t.start_file_logging()
current_status = t.run_once()
logging.info("PASS %s" % t.tag)
logging.info("")
t.stop_file_logging()
finally:
t_end = time.time()
t_elapsed = t_end - t_begin
except error.TestNAError, reason:
logging.info("SKIP %s -> %s: %s", t.tag,
reason.__class__.__name__, reason)
logging.info("")
t.stop_file_logging()
print_skip()
status_dct[dct.get("name")] = False
continue
except error.TestWarn, reason:
logging.info("WARN %s -> %s: %s", t.tag,
reason.__class__.__name__,
reason)
logging.info("")
t.stop_file_logging()
print_warn(t_elapsed)
status_dct[dct.get("name")] = True
continue
except Exception, reason:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.error("")
tb_info = traceback.format_exception(exc_type, exc_value,
exc_traceback.tb_next)
tb_info = "".join(tb_info)
for e_line in tb_info.splitlines():
logging.error(e_line)
logging.error("")
logging.error("FAIL %s -> %s: %s", t.tag,
reason.__class__.__name__,
reason)
logging.info("")
t.stop_file_logging()
current_status = False
def preprocess(test, params, env):
"""
Preprocess all VMs and images according to the instructions in params.
Also, collect some host information, such as the KVM version.
:param test: An Autotest test object.
:param params: A dict containing all VM and image parameters.
:param env: The environment (a dict-like object).
"""
error.context("preprocessing")
# First, let's verify if this test does require root or not. If it
# does and the test suite is running as a regular user, we shall just
# throw a TestNAError exception, which will skip the test.
if params.get('requires_root', 'no') == 'yes':
utils_misc.verify_running_as_root()
port = params.get('shell_port')
prompt = params.get('shell_prompt')
address = params.get('ovirt_node_address')
username = params.get('ovirt_node_user')
password = params.get('ovirt_node_password')
setup_pb = False
for nic in params.get('nics', "").split():
nic_params = params.object_params(nic)
if nic_params.get('netdst') == 'private':
setup_pb = True
params_pb = nic_params
params['netdst_%s' % nic] = nic_params.get("priv_brname", 'atbr0')
if setup_pb:
brcfg = test_setup.PrivateBridgeConfig(params_pb)
brcfg.setup()
base_dir = data_dir.get_data_dir()
if params.get("storage_type") == "iscsi":
iscsidev = qemu_storage.Iscsidev(params, base_dir, "iscsi")
params["image_name"] = iscsidev.setup()
params["image_raw_device"] = "yes"
if params.get("storage_type") == "lvm":
lvmdev = qemu_storage.LVMdev(params, base_dir, "lvm")
params["image_name"] = lvmdev.setup()
params["image_raw_device"] = "yes"
env.register_lvmdev("lvm_%s" % params["main_vm"], lvmdev)
if params.get("storage_type") == "nfs":
image_nfs = nfs.Nfs(params)
image_nfs.setup()
image_name_only = os.path.basename(params["image_name"])
params['image_name'] = os.path.join(image_nfs.mount_dir,
image_name_only)
for image_name in params.objects("images"):
name_tag = "image_name_%s" % image_name
if params.get(name_tag):
image_name_only = os.path.basename(params[name_tag])
params[name_tag] = os.path.join(image_nfs.mount_dir,
image_name_only)
# Start tcpdump if it isn't already running
# The fact it has to be started here is so that the test params
# have to be honored.
env.start_tcpdump(params)
# Destroy and remove VMs that are no longer needed in the environment
requested_vms = params.objects("vms")
for key in env.keys():
vm = env[key]
if not isinstance(vm, virt_vm.BaseVM):
continue
if not vm.name in requested_vms:
vm.destroy()
del env[key]
if (params.get("auto_cpu_model") == "yes" and
params.get("vm_type") == "qemu"):
if not env.get("cpu_model"):
env["cpu_model"] = utils_misc.get_qemu_best_cpu_model(params)
params["cpu_model"] = env.get("cpu_model")
kvm_ver_cmd = params.get("kvm_ver_cmd", "")
if kvm_ver_cmd:
try:
cmd_result = utils.run(kvm_ver_cmd)
kvm_version = cmd_result.stdout.strip()
except error.CmdError:
kvm_version = "Unknown"
else:
# Get the KVM kernel module version and write it as a keyval
if os.path.exists("/dev/kvm"):
try:
kvm_version = open("/sys/module/kvm/version").read().strip()
except Exception:
kvm_version = os.uname()[2]
else:
logging.warning("KVM module not loaded")
kvm_version = "Unknown"
logging.debug("KVM version: %s" % kvm_version)
test.write_test_keyval({"kvm_version": kvm_version})
# Get the KVM userspace version and write it as a keyval
kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")
if kvm_userspace_ver_cmd:
try:
cmd_result = utils.run(kvm_userspace_ver_cmd)
kvm_userspace_version = cmd_result.stdout.strip()
except error.CmdError:
kvm_userspace_version = "Unknown"
else:
qemu_path = utils_misc.get_qemu_binary(params)
version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
matches = re.findall("[Vv]ersion .*?,", version_line)
if matches:
kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
else:
kvm_userspace_version = "Unknown"
logging.debug("KVM userspace version: %s" % kvm_userspace_version)
test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
if params.get("setup_hugepages") == "yes":
h = test_setup.HugePageConfig(params)
suggest_mem = h.setup()
if suggest_mem is not None:
params['mem'] = suggest_mem
if params.get("vm_type") == "libvirt":
utils_libvirtd.libvirtd_restart()
if params.get("setup_thp") == "yes":
thp = test_setup.TransparentHugePageConfig(test, params)
thp.setup()
if params.get("setup_ksm") == "yes":
ksm = test_setup.KSMConfig(params, env)
ksm.setup(env)
# Execute any pre_commands
if params.get("pre_command"):
process_command(test, params, env, params.get("pre_command"),
int(params.get("pre_command_timeout", "600")),
params.get("pre_command_noncritical") == "yes")
# if you want set "pci=nomsi" before test, set "disable_pci_msi = yes"
# and pci_msi_sensitive = "yes"
if params.get("pci_msi_sensitive", "no") == "yes":
disable_pci_msi = params.get("disable_pci_msi", "no")
image_filename = storage.get_image_filename(params,
data_dir.get_data_dir())
grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
r".*vmlinuz-\d+.*")
msi_keyword = params.get("msi_keyword", " pci=nomsi")
disk_obj = utils_disk.GuestFSModiDisk(image_filename)
kernel_config_ori = disk_obj.read_file(grub_file)
kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
if not kernel_config:
raise error.TestError("Cannot find the kernel config, reg is %s" %
kernel_cfg_pos_reg)
kernel_config_line = kernel_config[0]
kernel_need_modify = False
if disable_pci_msi == "yes":
if not re.findall(msi_keyword, kernel_config_line):
kernel_config_set = kernel_config_line + msi_keyword
kernel_need_modify = True
else:
if re.findall(msi_keyword, kernel_config_line):
kernel_config_set = re.sub(msi_keyword, "", kernel_config_line)
kernel_need_modify = True
if kernel_need_modify:
for vm in env.get_all_vms():
if vm:
vm.destroy()
env.unregister_vm(vm.name)
disk_obj.replace_image_file_content(grub_file, kernel_config_line,
kernel_config_set)
logging.debug("Guest cmdline 'pci=nomsi' setting is: [ %s ]" %
disable_pci_msi)
kernel_extra_params = params.get("kernel_extra_params")
if kernel_extra_params:
image_filename = storage.get_image_filename(params,
data_dir.get_data_dir())
grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
r".*vmlinuz-\d+.*")
disk_obj = utils_disk.GuestFSModiDisk(image_filename)
kernel_config_ori = disk_obj.read_file(grub_file)
kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
if not kernel_config:
raise error.TestError("Cannot find the kernel config, reg is %s" %
kernel_cfg_pos_reg)
kernel_config_line = kernel_config[0]
kernel_need_modify = False
if not re.findall(kernel_extra_params, kernel_config_line):
kernel_config_set = kernel_config_line + kernel_extra_params
kernel_need_modify = True
if kernel_need_modify:
for vm in env.get_all_vms():
if vm:
vm.destroy()
env.unregister_vm(vm.name)
disk_obj.replace_image_file_content(grub_file, kernel_config_line,
kernel_config_set)
logging.debug("Guest cmdline extra_params setting is: [ %s ]" %
kernel_extra_params)
# Clone master image from vms.
base_dir = data_dir.get_data_dir()
if params.get("master_images_clone"):
for vm_name in params.get("vms").split():
vm = env.get_vm(vm_name)
if vm:
vm.destroy(free_mac_addresses=False)
env.unregister_vm(vm_name)
vm_params = params.object_params(vm_name)
for image in vm_params.get("master_images_clone").split():
image_obj = qemu_storage.QemuImg(params, base_dir, image)
image_obj.clone_image(params, vm_name, image, base_dir)
# Preprocess all VMs and images
if params.get("not_preprocess", "no") == "no":
process(test, params, env, preprocess_image, preprocess_vm)
# Start the screendump thread
if params.get("take_regular_screendumps") == "yes":
global _screendump_thread, _screendump_thread_termination_event
_screendump_thread_termination_event = threading.Event()
_screendump_thread = threading.Thread(target=_take_screendumps,
name='ScreenDump',
args=(test, params, env))
_screendump_thread.start()
return params
def create(self, name=None, params=None, root_dir=None, timeout=5.0, migration_mode=None, mac_source=None):
"""
Start the VM by running a qemu command.
All parameters are optional. If name, params or root_dir are not
supplied, the respective values stored as class attributes are used.
@param name: The name of the object
@param params: A dict containing VM params
@param root_dir: Base directory for relative filenames
@param migration_mode: If supplied, start VM for incoming migration
using this protocol (either 'tcp', 'unix' or 'exec')
@param migration_exec_cmd: Command to embed in '-incoming "exec: ..."'
(e.g. 'gzip -c -d filename') if migration_mode is 'exec'
@param mac_source: A VM object from which to copy MAC addresses. If not
specified, new addresses will be generated.
@raise VMCreateError: If qemu terminates unexpectedly
@raise VMKVMInitError: If KVM initialization fails
@raise VMHugePageError: If hugepage initialization fails
@raise VMImageMissingError: If a CD image is missing
@raise VMHashMismatchError: If a CD image hash has doesn't match the
expected hash
@raise VMBadPATypeError: If an unsupported PCI assignment type is
requested
@raise VMPAError: If no PCI assignable devices could be assigned
"""
error.context("creating '%s'" % self.name)
self.destroy(free_mac_addresses=False)
if name is not None:
self.name = name
if params is not None:
self.params = params
if root_dir is not None:
self.root_dir = root_dir
name = self.name
params = self.params
root_dir = self.root_dir
# Verify the md5sum of the ISO images
for cdrom in params.objects("cdroms"):
if params.get("medium") == "import":
break
cdrom_params = params.object_params(cdrom)
iso = cdrom_params.get("cdrom")
if (
(self.driver_type == "xen")
and (params.get("hvm_or_pv") == "pv")
and (os.path.basename(iso) == "ks.iso")
):
continue
if iso:
iso = utils_misc.get_path(data_dir.get_data_dir(), iso)
if not os.path.exists(iso):
raise virt_vm.VMImageMissingError(iso)
compare = False
if cdrom_params.get("md5sum_1m"):
logging.debug("Comparing expected MD5 sum with MD5 sum of " "first MB of ISO file...")
actual_hash = utils.hash_file(iso, 1048576, method="md5")
expected_hash = cdrom_params.get("md5sum_1m")
compare = True
elif cdrom_params.get("md5sum"):
logging.debug("Comparing expected MD5 sum with MD5 sum of " "ISO file...")
actual_hash = utils.hash_file(iso, method="md5")
expected_hash = cdrom_params.get("md5sum")
compare = True
elif cdrom_params.get("sha1sum"):
logging.debug("Comparing expected SHA1 sum with SHA1 sum " "of ISO file...")
actual_hash = utils.hash_file(iso, method="sha1")
expected_hash = cdrom_params.get("sha1sum")
compare = True
if compare:
if actual_hash == expected_hash:
logging.debug("Hashes match")
else:
raise virt_vm.VMHashMismatchError(actual_hash, expected_hash)
# Make sure the following code is not executed by more than one thread
# at the same time
lockfile = open("/tmp/libvirt-autotest-vm-create.lock", "w+")
fcntl.lockf(lockfile, fcntl.LOCK_EX)
try:
# Handle port redirections
redir_names = params.objects("redirs")
host_ports = utils_misc.find_free_ports(5000, 6000, len(redir_names))
self.redirs = {}
for i in range(len(redir_names)):
redir_params = params.object_params(redir_names[i])
guest_port = int(redir_params.get("guest_port"))
self.redirs[guest_port] = host_ports[i]
# Find available PCI devices
self.pci_devices = []
for device in params.objects("pci_devices"):
self.pci_devices.append(device)
# Find available VNC port, if needed
if params.get("display") == "vnc":
if params.get("vnc_autoport") == "yes":
self.vnc_port = None
self.vnc_autoport = True
else:
self.vnc_port = utils_misc.find_free_port(5900, 6100)
self.vnc_autoport = False
# Find available spice port, if needed
if params.get("spice"):
self.spice_port = utils_misc.find_free_port(8000, 8100)
# Find random UUID if specified 'uuid = random' in config file
if params.get("uuid") == "random":
f = open("/proc/sys/kernel/random/uuid")
self.uuid = f.read().strip()
f.close()
# Generate or copy MAC addresses for all NICs
for nic in self.virtnet:
nic_params = dict(nic)
if mac_source:
# Will raise exception if source doesn't
# have cooresponding nic
logging.debug("Copying mac for nic %s from VM %s" % (nic.nic_name, mac_source.nam))
nic_params["mac"] = mac_source.get_mac_address(nic.nic_name)
# make_create_command() calls vm.add_nic (i.e. on a copy)
nic = self.add_nic(**nic_params)
logging.debug("VM.create activating nic %s" % nic)
self.activate_nic(nic.nic_name)
# Make qemu command
install_command = self.make_create_command()
logging.info("Running libvirt command (reformatted):")
for item in install_command.replace(" -", " \n -").splitlines():
logging.info("%s", item)
utils.run(install_command, verbose=False)
# Wait for the domain to be created
utils_misc.wait_for(func=self.is_alive, timeout=60, text=("waiting for domain %s to start" % self.name))
self.uuid = virsh.domuuid(self.name, uri=self.connect_uri)
# Establish a session with the serial console
if self.only_pty == True:
self.serial_console = aexpect.ShellSession(
"virsh console %s" % self.name,
auto_close=False,
output_func=utils_misc.log_line,
output_params=("serial-%s.log" % name,),
)
else:
self.serial_console = aexpect.ShellSession(
"tail -f %s" % self.get_serial_console_filename(),
auto_close=False,
output_func=utils_misc.log_line,
output_params=("serial-%s.log" % name,),
)
finally:
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
def check_disk_params(self, params):
"""
Check gathered info from qtree/block with params
:param params: autotest params
:return: number of errors
"""
def check_drive_format(node, params):
""" checks the drive format according to qtree info """
expected = params.get('drive_format')
if expected == 'scsi':
if arch.ARCH == 'ppc64':
expected = 'spapr-vscsi'
else:
expected = 'lsi53c895a'
elif expected.startswith('scsi'):
expected = params.get('scsi_hba', 'virtio-scsi-pci')
elif expected.startswith('usb'):
expected = 'usb-storage'
try:
if expected == 'virtio':
actual = node.qtree['type']
else:
actual = node.parent.parent.qtree.get('type')
except AttributeError:
logging.error(
"Failed to check drive format, can't get parent"
"of:\n%s", node)
if actual == 'virtio-scsi-device': # new name for virtio-scsi
actual = 'virtio-scsi-pci'
if expected not in actual:
return ("drive format in qemu is %s, in autotest %s" %
(actual, expected))
err = 0
disks = {}
for disk in self.disks:
if isinstance(disk, QtreeDisk):
disks[disk.get_qname()] = (disk.get_params().copy(), disk)
# We don't have the params name so we need to map file_names instead
qname = None
for name in params.objects('cdroms'):
image_name = utils_misc.get_path(
data_dir.get_data_dir(),
params.object_params(name).get('cdrom', ''))
image_name = os.path.realpath(image_name)
for (qname, disk) in disks.iteritems():
if disk[0].get('image_name') == image_name:
break
else:
continue # Not /proc/scsi cdrom device
disks.pop(qname)
for name in params.objects('images'):
current = None
image_params = params.object_params(name)
base_dir = image_params.get("images_base_dir",
data_dir.get_data_dir())
image_name = os.path.realpath(
storage.get_image_filename(image_params, base_dir))
for (qname, disk) in disks.iteritems():
if disk[0].get('image_name') == image_name:
current = disk[0]
current_node = disk[1]
# autotest params might use relative path
current['image_name'] = image_params.get('image_name')
break
if not current:
logging.error("Disk %s is not in qtree but is in params.",
name)
err += 1
continue
for prop in current.iterkeys():
handled = False
if prop == "drive_format":
out = check_drive_format(current_node, image_params)
if out:
logging.error("Disk %s %s", qname, out)
err += 1
handled = True
elif (image_params.get(prop)
and image_params.get(prop) == current.get(prop)):
handled = True
if not handled:
logging.error(
"Disk %s property %s=%s doesn't match params"
" %s", qname, prop, current.get(prop),
image_params.get(prop))
err += 1
disks.pop(qname)
if disks:
logging.error(
'Some disks were in qtree but not in autotest params'
': %s', disks)
err += 1
return err
last_index = -1
logging.info("Starting test job at %s", test_start_time)
logging.info("")
logging.info(version.get_pretty_version_info())
logging.info("")
cleanup_env(parser, options)
d = parser.get_dicts().next()
if not options.vt_config:
if not options.vt_keep_image_between_tests:
logging.debug("Creating first backup of guest image")
qemu_img = storage.QemuImg(d, data_dir.get_data_dir(), "image")
qemu_img.backup_image(d, data_dir.get_data_dir(), 'backup', True)
logging.debug("")
for line in get_cartesian_parser_details(parser).splitlines():
logging.info(line)
logging.info("Defined test set:")
for count, dic in enumerate(parser.get_dicts()):
shortname = d.get("_name_map_file")["subtests.cfg"]
logging.info("Test %4d: %s", count + 1, shortname)
last_index += 1
if last_index == -1:
print_stdout("No tests generated by config file %s" % parser.filename)
def bootstrap_tests(options):
"""
Bootstrap process (download the appropriate JeOS file to data dir).
This function will check whether the JeOS is in the right location of the
data dir, if not, it will download it non interactively.
:param options: OptParse object with program command line options.
"""
test_dir = os.path.dirname(sys.modules[__name__].__file__)
if options.type:
test_dir = os.path.abspath(os.path.join(os.path.dirname(test_dir),
options.type))
elif options.config:
parent_config_dir = os.path.dirname(os.path.dirname(options.config))
parent_config_dir = os.path.dirname(parent_config_dir)
options.type = parent_config_dir
test_dir = os.path.abspath(parent_config_dir)
if options.type == 'qemu':
check_modules = arch.get_kvm_module_list()
else:
check_modules = None
online_docs_url = "https://github.com/autotest/virt-test/wiki"
kwargs = {'test_name': options.type,
'test_dir': test_dir,
'base_dir': data_dir.get_data_dir(),
'default_userspace_paths': None,
'check_modules': check_modules,
'online_docs_url': online_docs_url,
'download_image': not options.no_downloads,
'restore_image': not options.keep_image,
'interactive': False}
# Tolerance we have without printing a message for the user to wait (3 s)
tolerance = 3
failed = False
wait_message_printed = False
bg = utils.InterruptedThread(bootstrap.bootstrap, kwargs=kwargs)
t_begin = time.time()
bg.start()
while bg.isAlive():
t_elapsed = time.time() - t_begin
if t_elapsed > tolerance and not wait_message_printed:
print_stdout("Running setup. Please wait...")
wait_message_printed = True
# if bootstrap takes too long, we temporarily make stdout verbose
# again, so the user can see what's taking so long
sys.stdout.restore()
time.sleep(0.1)
# in case stdout was restored above, redirect it again
sys.stdout.redirect()
reason = None
try:
bg.join()
except Exception, e:
failed = True
reason = e
except Exception, e:
logging.error("Unexpected error:" % e)
libvirtd_inst.restart()
# Execute any pre_commands
if params.get("pre_command"):
process_command(test, params, env, params.get("pre_command"),
int(params.get("pre_command_timeout", "600")),
params.get("pre_command_noncritical") == "yes")
# if you want set "pci=nomsi" before test, set "disable_pci_msi = yes"
# and pci_msi_sensitive = "yes"
if params.get("pci_msi_sensitive", "no") == "yes":
disable_pci_msi = params.get("disable_pci_msi", "no")
image_filename = storage.get_image_filename(params,
data_dir.get_data_dir())
grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
r".*vmlinuz-\d+.*")
msi_keyword = params.get("msi_keyword", " pci=nomsi")
disk_obj = utils_disk.GuestFSModiDisk(image_filename)
kernel_config_ori = disk_obj.read_file(grub_file)
kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
if not kernel_config:
raise error.TestError("Cannot find the kernel config, reg is %s" %
kernel_cfg_pos_reg)
kernel_config_line = kernel_config[0]
kernel_need_modify = False
if disable_pci_msi == "yes":
verify_mandatory_programs(options.vt_type, guest_os)
except Exception, details:
logging.info(details)
logging.info('Install the missing programs and/or headers and '
're-run boostrap')
sys.exit(1)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(options.vt_type)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
datadir = data_dir.get_data_dir()
shared_dir = os.path.dirname(datadir)
sub_dir_list = ["images", "isos", "steps_data", "gpg"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(datadir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating",
sub_dir_path)
test_dir = data_dir.get_backend_dir(options.vt_type)
if options.vt_type == 'libvirt':
step = create_config_files(test_dir, shared_dir, interactive, step,
force_update=options.vt_update_config)
def bootstrap(test_name, test_dir, base_dir, default_userspace_paths,
check_modules, online_docs_url, restore_image=False,
download_image=True, interactive=True, selinux=False,
verbose=False, update_providers=False,
guest_os=defaults.DEFAULT_GUEST_OS):
"""
Common virt test assistant module.
:param test_name: Test name, such as "qemu".
:param test_dir: Path with the test directory.
:param base_dir: Base directory used to hold images and isos.
:param default_userspace_paths: Important programs for a successful test
execution.
:param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
:param online_docs_url: URL to an online documentation system, such as a
wiki page.
:param restore_image: Whether to restore the image from the pristine.
:param interactive: Whether to ask for confirmation.
:param verbose: Verbose output.
:param selinux: Whether setup SELinux contexts for shared/data.
:param update_providers: Whether to update test providers if they are already
downloaded.
:param guest_os: Specify the guest image used for bootstrapping. By default
the JeOS image is used.
:raise error.CmdError: If JeOS image failed to uncompress
:raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Updating all test providers", step)
asset.download_all_test_providers(update_providers)
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
sub_dir_list = ["images", "isos", "steps_data", "gpg"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating",
sub_dir_path)
datadir = data_dir.get_data_dir()
if test_name == 'libvirt':
create_config_files(test_dir, shared_dir, interactive, step)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
# Don't bother checking if changes can't be made
if os.getuid() == 0:
verify_selinux(datadir,
os.path.join(datadir, 'images'),
os.path.join(datadir, 'isos'),
data_dir.get_tmp_dir(),
interactive, selinux)
# lvsb test doesn't use any shared configs
elif test_name == 'lvsb':
create_subtests_cfg(test_name)
if os.getuid() == 0:
# Don't bother checking if changes can't be made
verify_selinux(datadir,
os.path.join(datadir, 'images'),
os.path.join(datadir, 'isos'),
data_dir.get_tmp_dir(),
interactive, selinux)
else: # Some other test
create_config_files(test_dir, shared_dir, interactive, step)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
if download_image or restore_image:
logging.info("")
step += 2
logging.info("%s - Verifying (and possibly downloading) guest image",
step)
for os_info in get_guest_os_info_list(test_name, guest_os):
os_asset = os_info['asset']
asset.download_asset(os_asset, interactive=interactive,
restore_image=restore_image)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning("Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info("%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
def bootstrap_tests(options):
"""
Bootstrap process (download the appropriate JeOS file to data dir).
This function will check whether the JeOS is in the right location of the
data dir, if not, it will download it non interactively.
:param options: OptParse object with program command line options.
"""
if options.type:
test_dir = data_dir.get_backend_dir(options.type)
elif options.config:
parent_config_dir = os.path.dirname(os.path.dirname(options.config))
parent_config_dir = os.path.dirname(parent_config_dir)
options.type = parent_config_dir
test_dir = os.path.abspath(parent_config_dir)
if options.type == 'qemu':
check_modules = arch.get_kvm_module_list()
else:
check_modules = None
online_docs_url = "https://github.com/autotest/virt-test/wiki"
if not options.config:
restore_image = not options.keep_image
else:
restore_image = False
kwargs = {'test_name': options.type,
'test_dir': test_dir,
'base_dir': data_dir.get_data_dir(),
'default_userspace_paths': None,
'check_modules': check_modules,
'online_docs_url': online_docs_url,
'download_image': not options.no_downloads,
'selinux': options.selinux_setup,
'restore_image': restore_image,
'interactive': False,
'update_providers': options.update_providers}
# Tolerance we have without printing a message for the user to wait (3 s)
tolerance = 3
failed = False
wait_message_printed = False
bg = utils.InterruptedThread(bootstrap.bootstrap, kwargs=kwargs)
t_begin = time.time()
bg.start()
while bg.isAlive():
t_elapsed = time.time() - t_begin
if t_elapsed > tolerance and not wait_message_printed:
print_stdout("Running setup. Please wait...")
wait_message_printed = True
# if bootstrap takes too long, we temporarily make stdout verbose
# again, so the user can see what's taking so long
sys.stdout.restore()
time.sleep(0.1)
# in case stdout was restored above, redirect it again
sys.stdout.redirect()
reason = None
try:
bg.join()
except Exception, e:
failed = True
reason = e
def preprocess(test, params, env):
"""
Preprocess all VMs and images according to the instructions in params.
Also, collect some host information, such as the KVM version.
@param test: An Autotest test object.
@param params: A dict containing all VM and image parameters.
@param env: The environment (a dict-like object).
"""
error.context("preprocessing")
# First, let's verify if this test does require root or not. If it
# does and the test suite is running as a regular user, we shall just
# throw a TestNAError exception, which will skip the test.
if params.get('requires_root', 'no') == 'yes':
utils_test.verify_running_as_root()
port = params.get('shell_port')
prompt = params.get('shell_prompt')
address = params.get('ovirt_node_address')
username = params.get('ovirt_node_user')
password = params.get('ovirt_node_password')
# Start tcpdump if it isn't already running
if "address_cache" not in env:
env["address_cache"] = {}
if "tcpdump" in env and not env["tcpdump"].is_alive():
env["tcpdump"].close()
del env["tcpdump"]
if "tcpdump" not in env and params.get("run_tcpdump", "yes") == "yes":
cmd = "%s -npvi any 'dst port 68'" % utils_misc.find_command("tcpdump")
if params.get("remote_preprocess") == "yes":
login_cmd = ("ssh -o UserKnownHostsFile=/dev/null -o \
PreferredAuthentications=password -p %s %s@%s" %
(port, username, address))
env["tcpdump"] = aexpect.ShellSession(
login_cmd,
output_func=_update_address_cache,
output_params=(env["address_cache"],))
remote._remote_login(env["tcpdump"], username, password, prompt)
env["tcpdump"].sendline(cmd)
else:
env["tcpdump"] = aexpect.Tail(
command=cmd,
output_func=_update_address_cache,
output_params=(env["address_cache"],))
if utils_misc.wait_for(lambda: not env["tcpdump"].is_alive(),
0.1, 0.1, 1.0):
logging.warn("Could not start tcpdump")
logging.warn("Status: %s" % env["tcpdump"].get_status())
logging.warn("Output:" + utils_misc.format_str_for_message(
env["tcpdump"].get_output()))
# Destroy and remove VMs that are no longer needed in the environment
requested_vms = params.objects("vms")
for key in env.keys():
vm = env[key]
if not isinstance(vm, virt_vm.BaseVM):
continue
if not vm.name in requested_vms:
vm.destroy()
del env[key]
# Get Host cpu type
if params.get("auto_cpu_model") == "yes":
if not env.get("cpu_model"):
env["cpu_model"] = utils_misc.get_cpu_model()
params["cpu_model"] = env.get("cpu_model")
kvm_ver_cmd = params.get("kvm_ver_cmd", "")
if kvm_ver_cmd:
try:
cmd_result = utils.run(kvm_ver_cmd)
kvm_version = cmd_result.stdout.strip()
except error.CmdError:
kvm_version = "Unknown"
else:
# Get the KVM kernel module version and write it as a keyval
if os.path.exists("/dev/kvm"):
try:
kvm_version = open("/sys/module/kvm/version").read().strip()
except Exception:
kvm_version = os.uname()[2]
else:
logging.warning("KVM module not loaded")
kvm_version = "Unknown"
logging.debug("KVM version: %s" % kvm_version)
test.write_test_keyval({"kvm_version": kvm_version})
# Get the KVM userspace version and write it as a keyval
kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")
if kvm_userspace_ver_cmd:
try:
cmd_result = utils.run(kvm_userspace_ver_cmd)
kvm_userspace_version = cmd_result.stdout.strip()
except error.CmdError:
kvm_userspace_version = "Unknown"
else:
qemu_path = utils_misc.get_path(test.bindir,
params.get("qemu_binary", "qemu"))
version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
matches = re.findall("[Vv]ersion .*?,", version_line)
if matches:
kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
else:
kvm_userspace_version = "Unknown"
logging.debug("KVM userspace version: %s" % kvm_userspace_version)
test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
if params.get("setup_hugepages") == "yes":
h = test_setup.HugePageConfig(params)
h.setup()
if params.get("vm_type") == "libvirt":
libvirt_vm.libvirtd_restart()
if params.get("setup_thp") == "yes":
thp = test_setup.TransparentHugePageConfig(test, params)
thp.setup()
# Execute any pre_commands
if params.get("pre_command"):
process_command(test, params, env, params.get("pre_command"),
int(params.get("pre_command_timeout", "600")),
params.get("pre_command_noncritical") == "yes")
#Clone master image from vms.
base_dir = data_dir.get_data_dir()
if params.get("master_images_clone"):
for vm_name in params.get("vms").split():
vm = env.get_vm(vm_name)
if vm:
vm.destroy(free_mac_addresses=False)
env.unregister_vm(vm_name)
vm_params = params.object_params(vm_name)
for image in vm_params.get("master_images_clone").split():
image_obj = qemu_storage.QemuImg(params, base_dir, image)
image_obj.clone_image(params, vm_name, image, base_dir)
# Preprocess all VMs and images
if params.get("not_preprocess","no") == "no":
process(test, params, env, preprocess_image, preprocess_vm)
# Start the screendump thread
if params.get("take_regular_screendumps") == "yes":
global _screendump_thread, _screendump_thread_termination_event
_screendump_thread_termination_event = threading.Event()
_screendump_thread = threading.Thread(target=_take_screendumps,
args=(test, params, env))
_screendump_thread.start()
def __init__(self, vm):
""" Initialize worker for use (including port init on guest) """
self.vm = vm
self.session = self.vm.wait_for_login()
self.__cmd_execute_worker = None
# Detect the OS version
guest_script_py = "virtio_console_guest.py"
out = self.session.cmd_output("echo on")
if "on" in out:
self.os_linux = True
guest_script_path = "/tmp/%s" % guest_script_py
cmd_guest_size = ("du -b %s | cut -f1"
% guest_script_path)
cmd_already_compiled_chck = "ls %so" % guest_script_path
cmd_compile = ("python -OO %s -c "
"&& echo -n 'PASS: Compile virtio_guest finished' "
"|| echo -n 'FAIL: Compile virtio_guest failed'"
% guest_script_path)
self.__cmd_execute_worker = ("python %so"
"&& echo -n 'PASS: virtio_guest finished' "
"|| echo -n 'FAIL: virtio_guest failed'"
% guest_script_path)
else:
self.os_linux = False
guest_script_path = "C:\\%s" % guest_script_py
cmd_guest_size = ("for %%I in (%s) do @echo %%~zI"
% guest_script_path)
cmd_already_compiled_chck = "dir %so" % guest_script_path
cmd_compile = ("%s -c "
"&& echo PASS: Compile virtio_guest finished "
"|| echo FAIL: Compile virtio_guest failed"
% guest_script_path)
self.__cmd_execute_worker = ("%so "
"&& echo PASS: virtio_guest finished "
"|| echo FAIL: virtio_guest failed"
% guest_script_path)
# Copy, compile and run the worker
timeout = 10
base_path = os.path.dirname(data_dir.get_data_dir())
guest_script_src = os.path.join(base_path, 'scripts',
'virtio_console_guest.py')
script_size = utils.system_output("du -b %s | cut -f1"
% guest_script_src).strip()
script_size_guest = self.session.cmd_output(cmd_guest_size).strip()
if (script_size != script_size_guest or
self.session.cmd_status(cmd_already_compiled_chck)):
if self.os_linux:
# Disable [email protected] on systemd-like hosts
self.session.cmd_status('systemctl mask '
'*****@*****.**')
self.session.cmd_status('systemctl stop '
'*****@*****.**')
# Copy virtio_console_guest.py into guests
self.vm.copy_files_to(guest_script_src, guest_script_path)
# set echo off (self.cmd() musn't contain C:)
self.session.sendline("echo off")
# Compile worker
logging.debug("Compile %s on guest %s", guest_script_py,
self.vm.name)
try:
self.cmd(cmd_compile, timeout)
except VirtioPortException:
if not self.os_linux:
logging.error("Script execution failed, do you have python"
" and pywin32 installed? Currently this "
"needs to be done manually!")
raise
self.session.sendline()
# set echo off (self.cmd() musn't contain C:)
self.session.sendline("echo off")
logging.debug("Starting %so on guest %s", guest_script_py,
self.vm.name)
self._execute_worker(timeout)
self._init_guest(timeout)
def bootstrap(test_name,
test_dir,
base_dir,
default_userspace_paths,
check_modules,
online_docs_url,
restore_image=False,
download_image=True,
interactive=True,
verbose=False):
"""
Common virt test assistant module.
@param test_name: Test name, such as "qemu".
@param test_dir: Path with the test directory.
@param base_dir: Base directory used to hold images and isos.
@param default_userspace_paths: Important programs for a successful test
execution.
@param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
@param online_docs_url: URL to an online documentation system, such as a
wiki page.
@param restore_image: Whether to restore the image from the pristine.
@param interactive: Whether to ask for confirmation.
@raise error.CmdError: If JeOS image failed to uncompress
@raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
sub_dir_list = ["images", "isos", "steps_data"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating", sub_dir_path)
create_config_files(test_dir, shared_dir, interactive, step)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
if download_image or restore_image:
logging.info("")
step += 2
logging.info("%s - Verifying (and possibly downloading) guest image",
step)
asset.download_asset('jeos-17-64',
interactive=interactive,
restore_image=restore_image)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning(
"Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info(
"%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
def make_create_command(self, name=None, params=None, root_dir=None):
"""
Generate a libvirt command line. All parameters are optional. If a
parameter is not supplied, the corresponding value stored in the
class attributes is used.
@param name: The name of the object
@param params: A dict containing VM params
@param root_dir: Base directory for relative filenames
@note: The params dict should contain:
mem -- memory size in MBs
cdrom -- ISO filename to use with the qemu -cdrom parameter
extra_params -- a string to append to the qemu command
shell_port -- port of the remote shell daemon on the guest
(SSH, Telnet or the home-made Remote Shell Server)
shell_client -- client program to use for connecting to the
remote shell daemon on the guest (ssh, telnet or nc)
x11_display -- if specified, the DISPLAY environment variable
will be be set to this value for the qemu process (useful for
SDL rendering)
images -- a list of image object names, separated by spaces
nics -- a list of NIC object names, separated by spaces
For each image in images:
drive_format -- string to pass as 'if' parameter for this
image (e.g. ide, scsi)
image_snapshot -- if yes, pass 'snapshot=on' to qemu for
this image
image_boot -- if yes, pass 'boot=on' to qemu for this image
In addition, all parameters required by get_image_filename.
For each NIC in nics:
nic_model -- string to pass as 'model' parameter for this
NIC (e.g. e1000)
"""
# helper function for command line option wrappers
def has_option(help_text, option):
return bool(re.search(r"--%s" % option, help_text, re.MULTILINE))
# Wrappers for all supported libvirt command line parameters.
# This is meant to allow support for multiple libvirt versions.
# Each of these functions receives the output of 'libvirt --help' as a
# parameter, and should add the requested command line option
# accordingly.
def add_name(help_text, name):
return " --name '%s'" % name
def add_machine_type(help_text, machine_type):
if has_option(help_text, "machine"):
return " --machine %s" % machine_type
else:
return ""
def add_hvm_or_pv(help_text, hvm_or_pv):
if hvm_or_pv == "hvm":
return " --hvm --accelerate"
elif hvm_or_pv == "pv":
return " --paravirt"
else:
logging.warning("Unknown virt type hvm_or_pv, using default.")
return ""
def add_mem(help_text, mem):
return " --ram=%s" % mem
def add_check_cpu(help_text):
if has_option(help_text, "check-cpu"):
return " --check-cpu"
else:
return ""
def add_smp(help_text, smp):
return " --vcpu=%s" % smp
def add_location(help_text, location):
if has_option(help_text, "location"):
return " --location %s" % location
else:
return ""
def add_cdrom(help_text, filename, index=None):
if has_option(help_text, "cdrom"):
return " --cdrom %s" % filename
else:
return ""
def add_pxe(help_text):
if has_option(help_text, "pxe"):
return " --pxe"
else:
return ""
def add_import(help_text):
if has_option(help_text, "import"):
return " --import"
else:
return ""
def add_drive(
help_text,
filename,
pool=None,
vol=None,
device=None,
bus=None,
perms=None,
size=None,
sparse=False,
cache=None,
fmt=None,
):
cmd = " --disk"
if filename:
cmd += " path=%s" % filename
elif pool:
if vol:
cmd += " vol=%s/%s" % (pool, vol)
else:
cmd += " pool=%s" % pool
if device:
cmd += ",device=%s" % device
if bus:
cmd += ",bus=%s" % bus
if perms:
cmd += ",%s" % perms
if size:
cmd += ",size=%s" % size.rstrip("Gg")
if sparse:
cmd += ",sparse=false"
if fmt:
cmd += ",format=%s" % fmt
if cache:
cmd += ",cache=%s" % cache
return cmd
def add_floppy(help_text, filename):
return " --disk path=%s,device=floppy,ro" % filename
def add_vnc(help_text, vnc_port=None):
if vnc_port:
return " --vnc --vncport=%d" % (vnc_port)
else:
return " --vnc"
def add_vnclisten(help_text, vnclisten):
if has_option(help_text, "vnclisten"):
return " --vnclisten=%s" % (vnclisten)
else:
return ""
def add_sdl(help_text):
if has_option(help_text, "sdl"):
return " --sdl"
else:
return ""
def add_nographic(help_text):
return " --nographics"
def add_video(help_text, video_device):
if has_option(help_text, "video"):
return " --video=%s" % (video_device)
else:
return ""
def add_uuid(help_text, uuid):
if has_option(help_text, "uuid"):
return " --uuid %s" % uuid
else:
return ""
def add_os_type(help_text, os_type):
if has_option(help_text, "os-type"):
return " --os-type %s" % os_type
else:
return ""
def add_os_variant(help_text, os_variant):
if has_option(help_text, "os-variant"):
return " --os-variant %s" % os_variant
else:
return ""
def add_pcidevice(help_text, pci_device):
if has_option(help_text, "host-device"):
return " --host-device %s" % pci_device
else:
return ""
def add_soundhw(help_text, sound_device):
if has_option(help_text, "soundhw"):
return " --soundhw %s" % sound_device
else:
return ""
def add_serial(help_text, filename):
if has_option(help_text, "serial"):
return " --serial file,path=%s --serial pty" % filename
else:
self.only_pty = True
return ""
def add_kernel_cmdline(help_text, cmdline):
return " -append %s" % cmdline
def add_connect_uri(help_text, uri):
if uri and has_option(help_text, "connect"):
return " --connect=%s" % uri
else:
return ""
def add_nic(help_text, nic_params):
"""
Return additional command line params based on dict-like nic_params
"""
mac = nic_params.get("mac")
nettype = nic_params.get("nettype")
netdst = nic_params.get("netdst")
nic_model = nic_params.get("nic_model")
if nettype:
result = " --network=%s" % nettype
else:
result = ""
if has_option(help_text, "bridge"):
# older libvirt (--network=NATdev --bridge=bridgename --mac=mac)
if nettype != "user":
result += ":%s" % netdst
if mac: # possible to specify --mac w/o --network
result += " --mac=%s" % mac
else:
# newer libvirt (--network=mynet,model=virtio,mac=00:11)
if nettype != "user":
result += "=%s" % netdst
if nettype and nic_model: # only supported along with nettype
result += ",model=%s" % nic_model
if nettype and mac:
result += ",mac=%s" % mac
elif mac: # possible to specify --mac w/o --network
result += " --mac=%s" % mac
logging.debug("vm.make_create_command.add_nic returning: %s" % result)
return result
# End of command line option wrappers
if name is None:
name = self.name
if params is None:
params = self.params
if root_dir is None:
root_dir = self.root_dir
# Clone this VM using the new params
vm = self.clone(name, params, root_dir, copy_state=True)
virt_install_binary = utils_misc.get_path(root_dir, params.get("virt_install_binary", "virt-install"))
help_text = utils.system_output("%s --help" % virt_install_binary)
# Find all supported machine types, so we can rule out an unsupported
# machine type option passed in the configuration.
hvm_or_pv = params.get("hvm_or_pv", "hvm")
# default to 'uname -m' output
arch_name = params.get("vm_arch_name", utils.get_current_kernel_arch())
capabs = libvirt_xml.LibvirtXML()
support_machine_type = capabs.os_arch_machine_map[hvm_or_pv][arch_name]
logging.debug("Machine types supported for %s\%s: %s" % (hvm_or_pv, arch_name, support_machine_type))
# Start constructing the qemu command
virt_install_cmd = ""
# Set the X11 display parameter if requested
if params.get("x11_display"):
virt_install_cmd += "DISPLAY=%s " % params.get("x11_display")
# Add the qemu binary
virt_install_cmd += virt_install_binary
# set connect uri
virt_install_cmd += add_connect_uri(help_text, self.connect_uri)
# hvm or pv specificed by libvirt switch (pv used by Xen only)
if hvm_or_pv:
virt_install_cmd += add_hvm_or_pv(help_text, hvm_or_pv)
# Add the VM's name
virt_install_cmd += add_name(help_text, name)
machine_type = params.get("machine_type")
if machine_type:
if machine_type in support_machine_type:
virt_install_cmd += add_machine_type(help_text, machine_type)
else:
raise error.TestNAError("Unsupported machine type %s." % (machine_type))
mem = params.get("mem")
if mem:
virt_install_cmd += add_mem(help_text, mem)
# TODO: should we do the check before we call ? negative case ?
check_cpu = params.get("use_check_cpu")
if check_cpu:
virt_install_cmd += add_check_cpu(help_text)
smp = params.get("smp")
if smp:
virt_install_cmd += add_smp(help_text, smp)
# TODO: directory location for vmlinuz/kernel for cdrom install ?
location = None
if params.get("medium") == "url":
location = params.get("url")
elif params.get("medium") == "kernel_initrd":
# directory location of kernel/initrd pair (directory layout must
# be in format libvirt will recognize)
location = params.get("image_dir")
elif params.get("medium") == "nfs":
location = "nfs:%s:%s" % (params.get("nfs_server"), params.get("nfs_dir"))
elif params.get("medium") == "cdrom":
if params.get("use_libvirt_cdrom_switch") == "yes":
virt_install_cmd += add_cdrom(help_text, params.get("cdrom_cd1"))
elif params.get("unattended_delivery_method") == "integrated":
virt_install_cmd += add_cdrom(
help_text, os.path.join(data_dir.get_data_dir(), params.get("cdrom_unattended"))
)
else:
location = params.get("image_dir")
kernel_dir = os.path.dirname(params.get("kernel"))
kernel_parent_dir = os.path.dirname(kernel_dir)
pxeboot_link = os.path.join(kernel_parent_dir, "pxeboot")
if os.path.islink(pxeboot_link):
os.unlink(pxeboot_link)
if os.path.isdir(pxeboot_link):
logging.info("Removed old %s leftover directory", pxeboot_link)
shutil.rmtree(pxeboot_link)
os.symlink(kernel_dir, pxeboot_link)
elif params.get("medium") == "import":
virt_install_cmd += add_import(help_text)
if location:
virt_install_cmd += add_location(help_text, location)
if params.get("display") == "vnc":
if params.get("vnc_autoport") == "yes":
vm.vnc_autoport = True
else:
vm.vnc_autoport = False
if not vm.vnc_autoport and params.get("vnc_port"):
vm.vnc_port = int(params.get("vnc_port"))
virt_install_cmd += add_vnc(help_text, vm.vnc_port)
if params.get("vnclisten"):
vm.vnclisten = params.get("vnclisten")
virt_install_cmd += add_vnclisten(help_text, vm.vnclisten)
elif params.get("display") == "sdl":
virt_install_cmd += add_sdl(help_text)
elif params.get("display") == "nographic":
virt_install_cmd += add_nographic(help_text)
video_device = params.get("video_device")
if video_device:
virt_install_cmd += add_video(help_text, video_device)
sound_device = params.get("sound_device")
if sound_device:
virt_install_cmd += add_soundhw(help_text, sound_device)
# if none is given a random UUID will be generated by libvirt
if params.get("uuid"):
virt_install_cmd += add_uuid(help_text, params.get("uuid"))
# selectable OS type
if params.get("use_os_type") == "yes":
virt_install_cmd += add_os_type(help_text, params.get("os_type"))
# selectable OS variant
if params.get("use_os_variant") == "yes":
virt_install_cmd += add_os_variant(help_text, params.get("os_variant"))
# Add serial console
virt_install_cmd += add_serial(help_text, self.get_serial_console_filename())
# If the PCI assignment step went OK, add each one of the PCI assigned
# devices to the command line.
if self.pci_devices:
for pci_id in self.pci_devices:
virt_install_cmd += add_pcidevice(help_text, pci_id)
for image_name in params.objects("images"):
image_params = params.object_params(image_name)
filename = storage.get_image_filename(image_params, data_dir.get_data_dir())
if image_params.get("use_storage_pool") == "yes":
filename = None
virt_install_cmd += add_drive(
help_text,
filename,
image_params.get("image_pool"),
image_params.get("image_vol"),
image_params.get("image_device"),
image_params.get("image_bus"),
image_params.get("image_perms"),
image_params.get("image_size"),
image_params.get("drive_sparse"),
image_params.get("drive_cache"),
image_params.get("image_format"),
)
if image_params.get("boot_drive") == "no":
continue
if filename:
virt_install_cmd += add_drive(
help_text,
filename,
None,
None,
None,
image_params.get("drive_format"),
None,
image_params.get("image_size"),
image_params.get("drive_sparse"),
image_params.get("drive_cache"),
image_params.get("image_format"),
)
if params.get("unattended_delivery_method") != "integrated" and not (
self.driver_type == "xen" and params.get("hvm_or_pv") == "pv"
):
for cdrom in params.objects("cdroms"):
cdrom_params = params.object_params(cdrom)
iso = cdrom_params.get("cdrom")
if params.get("use_libvirt_cdrom_switch") == "yes":
# we don't want to skip the winutils iso
if not cdrom == "winutils":
logging.debug("Using --cdrom instead of --disk for install")
logging.debug("Skipping CDROM:%s:%s", cdrom, iso)
continue
if params.get("medium") == "cdrom_no_kernel_initrd":
if iso == params.get("cdrom_cd1"):
logging.debug("Using cdrom or url for install")
logging.debug("Skipping CDROM: %s", iso)
continue
if iso:
virt_install_cmd += add_drive(
help_text,
utils_misc.get_path(root_dir, iso),
image_params.get("iso_image_pool"),
image_params.get("iso_image_vol"),
"cdrom",
None,
None,
None,
None,
None,
None,
)
# We may want to add {floppy_otps} parameter for -fda
# {fat:floppy:}/path/. However vvfat is not usually recommended.
# Only support to add the main floppy if you want to add the second
# one please modify this part.
floppy = params.get("floppy_name")
if floppy:
floppy = utils_misc.get_path(data_dir.get_data_dir(), floppy)
virt_install_cmd += add_drive(help_text, floppy, None, None, "floppy", None, None, None, None, None, None)
# setup networking parameters
for nic in vm.virtnet:
# make_create_command can be called w/o vm.create()
nic = vm.add_nic(**dict(nic))
logging.debug("make_create_command() setting up command for" " nic: %s" % str(nic))
virt_install_cmd += add_nic(help_text, nic)
if params.get("use_no_reboot") == "yes":
virt_install_cmd += " --noreboot"
if params.get("use_autostart") == "yes":
virt_install_cmd += " --autostart"
if params.get("virt_install_debug") == "yes":
virt_install_cmd += " --debug"
# bz still open, not fully functional yet
if params.get("use_virt_install_wait") == "yes":
virt_install_cmd += " --wait %s" % params.get("virt_install_wait_time")
kernel_params = params.get("kernel_params")
if kernel_params:
virt_install_cmd += " --extra-args '%s'" % kernel_params
virt_install_cmd += " --noautoconsole"
return virt_install_cmd
def bootstrap(test_name,
test_dir,
base_dir,
default_userspace_paths,
check_modules,
online_docs_url,
restore_image=False,
interactive=True,
selinux=False,
verbose=False,
update_providers=False,
guest_os=defaults.DEFAULT_GUEST_OS,
force_update=False):
"""
Common virt test assistant module.
:param test_name: Test name, such as "qemu".
:param test_dir: Path with the test directory.
:param base_dir: Base directory used to hold images and isos.
:param default_userspace_paths: Important programs for a successful test
execution.
:param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
:param online_docs_url: URL to an online documentation system, such as a
wiki page.
:param restore_image: Whether to restore the image from the pristine.
:param interactive: Whether to ask for confirmation.
:param verbose: Verbose output.
:param selinux: Whether setup SELinux contexts for shared/data.
:param update_providers: Whether to update test providers if they are already
downloaded.
:param guest_os: Specify the guest image used for bootstrapping. By default
the JeOS image is used.
:raise error.CmdError: If JeOS image failed to uncompress
:raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Updating all test providers", step)
asset.download_all_test_providers(update_providers)
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name, guest_os)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
sub_dir_list = ["images", "isos", "steps_data", "gpg"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating", sub_dir_path)
datadir = data_dir.get_data_dir()
if test_name == 'libvirt':
create_config_files(test_dir, shared_dir, interactive, step,
force_update)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
# Don't bother checking if changes can't be made
if os.getuid() == 0:
verify_selinux(datadir, os.path.join(datadir, 'images'),
os.path.join(datadir, 'isos'),
data_dir.get_tmp_dir(), interactive, selinux)
# lvsb test doesn't use any shared configs
elif test_name == 'lvsb':
create_subtests_cfg(test_name)
if os.getuid() == 0:
# Don't bother checking if changes can't be made
verify_selinux(datadir, os.path.join(datadir, 'images'),
os.path.join(datadir, 'isos'),
data_dir.get_tmp_dir(), interactive, selinux)
else: # Some other test
create_config_files(test_dir, shared_dir, interactive, step,
force_update)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
if restore_image:
logging.info("")
step += 1
logging.info("%s - Verifying (and possibly downloading) guest image",
step)
for os_info in get_guest_os_info_list(test_name, guest_os):
os_asset = os_info['asset']
asset.download_asset(os_asset,
interactive=interactive,
restore_image=restore_image)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning(
"Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info(
"%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
except Exception, details:
err += "\nKSM cleanup: %s" % str(details).replace('\\n', '\n ')
logging.error(details)
# Execute any post_commands
if params.get("post_command"):
try:
process_command(test, params, env, params.get("post_command"),
int(params.get("post_command_timeout", "600")),
params.get("post_command_noncritical") == "yes")
except Exception, details:
err += "\nPostprocess command: %s" % str(details).replace(
'\n', '\n ')
logging.error(details)
base_dir = data_dir.get_data_dir()
if params.get("storage_type") == "iscsi":
try:
iscsidev = qemu_storage.Iscsidev(params, base_dir, "iscsi")
iscsidev.cleanup()
except Exception, details:
err += "\niscsi cleanup: %s" % str(details).replace('\\n', '\n ')
logging.error(details)
if params.get("storage_type") == "lvm":
try:
lvmdev = env.get_lvmdev("lvm_%s" % params["main_vm"])
lvmdev.cleanup()
except Exception, details:
err += "\nLVM cleanup: %s" % str(details).replace('\\n', '\n ')
logging.error(details)
def preprocess(test, params, env):
"""
Preprocess all VMs and images according to the instructions in params.
Also, collect some host information, such as the KVM version.
:param test: An Autotest test object.
:param params: A dict containing all VM and image parameters.
:param env: The environment (a dict-like object).
"""
error.context("preprocessing")
# First, let's verify if this test does require root or not. If it
# does and the test suite is running as a regular user, we shall just
# throw a TestNAError exception, which will skip the test.
if params.get('requires_root', 'no') == 'yes':
utils_misc.verify_running_as_root()
port = params.get('shell_port')
prompt = params.get('shell_prompt')
address = params.get('ovirt_node_address')
username = params.get('ovirt_node_user')
password = params.get('ovirt_node_password')
setup_pb = False
for nic in params.get('nics', "").split():
nic_params = params.object_params(nic)
if nic_params.get('netdst') == 'private':
setup_pb = True
params_pb = nic_params
params['netdst_%s' % nic] = nic_params.get("priv_brname", 'atbr0')
if setup_pb:
brcfg = test_setup.PrivateBridgeConfig(params_pb)
brcfg.setup()
base_dir = data_dir.get_data_dir()
if params.get("storage_type") == "iscsi":
iscsidev = qemu_storage.Iscsidev(params, base_dir, "iscsi")
params["image_name"] = iscsidev.setup()
params["image_raw_device"] = "yes"
if params.get("storage_type") == "lvm":
lvmdev = qemu_storage.LVMdev(params, base_dir, "lvm")
params["image_name"] = lvmdev.setup()
params["image_raw_device"] = "yes"
env.register_lvmdev("lvm_%s" % params["main_vm"], lvmdev)
if params.get("storage_type") == "nfs":
image_nfs = nfs.Nfs(params)
image_nfs.setup()
image_name_only = os.path.basename(params["image_name"])
params['image_name'] = os.path.join(image_nfs.mount_dir,
image_name_only)
for image_name in params.objects("images"):
name_tag = "image_name_%s" % image_name
if params.get(name_tag):
image_name_only = os.path.basename(params[name_tag])
params[name_tag] = os.path.join(image_nfs.mount_dir,
image_name_only)
# Start tcpdump if it isn't already running
# The fact it has to be started here is so that the test params
# have to be honored.
env.start_tcpdump(params)
# Destroy and remove VMs that are no longer needed in the environment
requested_vms = params.objects("vms")
for key in env.keys():
vm = env[key]
if not isinstance(vm, virt_vm.BaseVM):
continue
if vm.name not in requested_vms:
vm.destroy()
del env[key]
if (params.get("auto_cpu_model") == "yes"
and params.get("vm_type") == "qemu"):
if not env.get("cpu_model"):
env["cpu_model"] = utils_misc.get_qemu_best_cpu_model(params)
params["cpu_model"] = env.get("cpu_model")
kvm_ver_cmd = params.get("kvm_ver_cmd", "")
if kvm_ver_cmd:
try:
cmd_result = utils.run(kvm_ver_cmd)
kvm_version = cmd_result.stdout.strip()
except error.CmdError:
kvm_version = "Unknown"
else:
# Get the KVM kernel module version and write it as a keyval
if os.path.exists("/dev/kvm"):
try:
kvm_version = open("/sys/module/kvm/version").read().strip()
except Exception:
kvm_version = os.uname()[2]
else:
logging.warning("KVM module not loaded")
kvm_version = "Unknown"
logging.debug("KVM version: %s" % kvm_version)
test.write_test_keyval({"kvm_version": kvm_version})
# Get the KVM userspace version and write it as a keyval
kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")
if kvm_userspace_ver_cmd:
try:
cmd_result = utils.run(kvm_userspace_ver_cmd)
kvm_userspace_version = cmd_result.stdout.strip()
except error.CmdError:
kvm_userspace_version = "Unknown"
else:
qemu_path = utils_misc.get_qemu_binary(params)
version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
matches = re.findall("[Vv]ersion .*?,", version_line)
if matches:
kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
else:
kvm_userspace_version = "Unknown"
logging.debug("KVM userspace version: %s" % kvm_userspace_version)
test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
libvirtd_inst = utils_libvirtd.Libvirtd()
if params.get("setup_hugepages") == "yes":
h = test_setup.HugePageConfig(params)
suggest_mem = h.setup()
if suggest_mem is not None:
params['mem'] = suggest_mem
if params.get("vm_type") == "libvirt":
libvirtd_inst.restart()
if params.get("setup_thp") == "yes":
thp = test_setup.TransparentHugePageConfig(test, params)
thp.setup()
if params.get("setup_ksm") == "yes":
ksm = test_setup.KSMConfig(params, env)
ksm.setup(env)
if params.get("vm_type") == "libvirt":
if params.get("setup_libvirt_polkit") == "yes":
pol = test_setup.LibvirtPolkitConfig(params)
try:
pol.setup()
except test_setup.PolkitWriteLibvirtdConfigError, e:
logging.error("e")
except test_setup.PolkitRulesSetupError, e:
logging.error("e")
except Exception, e:
logging.error("Unexpected error:" % e)
