def __init__(self, params, root_dir, tag): """ Init the default value for image object. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @param tag: Image tag defined in parameter images. """ self.image_filename = get_image_filename(params, root_dir) self.image_format = params.get("image_format", "qcow2") self.size = params.get("image_size", "10G") self.check_output = params.get("check_output") == "yes" self.image_blkdebug_filename = get_image_blkdebug_filename(params, root_dir) image_chain = params.get("image_chain") self.base_tag = None self.snapshot_tag = None if image_chain: image_chain = re.split("\s+", image_chain) if tag in image_chain: index = image_chain.index(tag) if index < len(image_chain): self.snapshot_tag = image_chain[index + 1] if index > 0: self.base_tag = image_chain[index - 1] if self.base_tag: base_params = params.object_params(self.base_tag) self.base_image_filename = virt_utils.get_path(base_params, root_dir) self.base_format = base_params.get("image_format") if self.snapshot_tag: ss_params = params.object_params(self.snapshot_tag) self.snapshot_image_filename = virt_utils.get_path(ss_params, root_dir) self.snapshot_format = ss_params.get("image_format")
def __init__(self, test, params, image_name, blkdebug_cfg="", prompt=r"qemu-io>\s*$", log_filename=None, io_options=""): self.type = "" if log_filename: log_filename += "-" + virt_utils.generate_random_string(4) self.output_func = virt_utils.log_line self.output_params = (log_filename, ) else: self.output_func = None self.output_params = () self.output_prefix = "" self.prompt = prompt self.blkdebug_cfg = blkdebug_cfg self.qemu_io_cmd = virt_utils.get_path( test.bindir, params.get("qemu_io_binary", "qemu-io")) self.io_options = io_options self.run_command = False self.image_name = image_name self.blkdebug_cfg = blkdebug_cfg
def create_image(params, root_dir): """ Create an image using qemu_image. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) image_cluster_size (optional) -- the cluster size for the image image_size -- the requested size of the image (a string qemu-img can understand, such as '10G') """ qemu_img_cmd = virt_utils.get_path(root_dir, params.get("qemu_img_binary", "qemu-img")) qemu_img_cmd += " create" format = params.get("image_format", "qcow2") qemu_img_cmd += " -f %s" % format image_cluster_size = params.get("image_cluster_size", None) if image_cluster_size is not None: qemu_img_cmd += " -o cluster_size=%s" % image_cluster_size image_filename = get_image_filename(params, root_dir) qemu_img_cmd += " %s" % image_filename size = params.get("image_size", "10G") qemu_img_cmd += " %s" % size utils.system(qemu_img_cmd) return image_filename
def create_image(params, root_dir): """ Create an image using qemu_image. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) image_cluster_size (optional) -- the cluster size for the image image_size -- the requested size of the image (a string qemu-img can understand, such as '10G') """ qemu_img_cmd = virt_utils.get_path( root_dir, params.get("qemu_img_binary", "qemu-img")) qemu_img_cmd += " create" format = params.get("image_format", "qcow2") qemu_img_cmd += " -f %s" % format image_cluster_size = params.get("image_cluster_size", None) if image_cluster_size is not None: qemu_img_cmd += " -o cluster_size=%s" % image_cluster_size image_filename = get_image_filename(params, root_dir) qemu_img_cmd += " %s" % image_filename size = params.get("image_size", "10G") qemu_img_cmd += " %s" % size utils.system(qemu_img_cmd) return image_filename
def _take_screendumps(test, params, env): global _screendump_thread_termination_event temp_dir = test.debugdir if params.get("screendump_temp_dir"): temp_dir = virt_utils.get_path(test.bindir, params.get("screendump_temp_dir")) try: os.makedirs(temp_dir) except OSError: pass temp_filename = os.path.join(temp_dir, "scrdump-%s.ppm" % virt_utils.generate_random_string(6)) delay = float(params.get("screendump_delay", 5)) quality = int(params.get("screendump_quality", 30)) cache = {} while True: for vm in env.get_all_vms(): if not vm.is_alive(): continue try: vm.monitor.screendump(filename=temp_filename, debug=False) except kvm_monitor.MonitorError, e: logging.warn(e) continue except AttributeError, e: continue if not os.path.exists(temp_filename): logging.warn("VM '%s' failed to produce a screendump", vm.name) continue if not ppm_utils.image_verify_ppm_file(temp_filename): logging.warn("VM '%s' produced an invalid screendump", vm.name) os.unlink(temp_filename) continue screendump_dir = os.path.join(test.debugdir, "screendumps_%s" % vm.name) try: os.makedirs(screendump_dir) except OSError: pass screendump_filename = os.path.join(screendump_dir, "%s_%s.jpg" % (vm.name, time.strftime("%Y-%m-%d_%H-%M-%S"))) hash = utils.hash_file(temp_filename) if hash in cache: try: os.link(cache[hash], screendump_filename) except OSError: pass else: try: image = PIL.Image.open(temp_filename) image.save(screendump_filename, format="JPEG", quality=quality) cache[hash] = screendump_filename except NameError: pass os.unlink(temp_filename)
def __init__(self, params, root_dir, tag): """ Init the default value for image object. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @param tag: Image tag defined in parameter images """ virt_storage.QemuImg.__init__(self, params, root_dir, tag) self.image_cmd = virt_utils.get_path(root_dir, params.get("qemu_img_binary","qemu-img"))
def create_image(params, root_dir): """ Create an image using qemu_image. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) image_cluster_size (optional) -- the cluster size for the image image_size -- the requested size of the image (a string qemu-img can understand, such as '10G') create_with_dd -- use dd to create the image (raw format only) """ format = params.get("image_format", "qcow2") image_filename = get_image_filename(params, root_dir) size = params.get("image_size", "10G") if params.get("create_with_dd") == "yes" and format == "raw": # maps K,M,G,T => (count, bs) human = { 'K': (1, 1), 'M': (1, 1024), 'G': (1024, 1024), 'T': (1024, 1048576), } if human.has_key(size[-1]): block_size = human[size[-1]][1] size = int(size[:-1]) * human[size[-1]][0] qemu_img_cmd = ("dd if=/dev/zero of=%s count=%s bs=%sK" % (image_filename, size, block_size)) else: qemu_img_cmd = virt_utils.get_path( root_dir, params.get("qemu_img_binary", "qemu-img")) qemu_img_cmd += " create" qemu_img_cmd += " -f %s" % format image_cluster_size = params.get("image_cluster_size", None) if image_cluster_size is not None: qemu_img_cmd += " -o cluster_size=%s" % image_cluster_size qemu_img_cmd += " %s" % image_filename qemu_img_cmd += " %s" % size utils.system(qemu_img_cmd) return image_filename
def create_image(params, root_dir): """ Create an image using qemu_image. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) image_cluster_size (optional) -- the cluster size for the image image_size -- the requested size of the image (a string qemu-img can understand, such as '10G') create_with_dd -- use dd to create the image (raw format only) """ format = params.get("image_format", "qcow2") image_filename = get_image_filename(params, root_dir) size = params.get("image_size", "10G") if params.get("create_with_dd") == "yes" and format == "raw": # maps K,M,G,T => (count, bs) human = {'K': (1, 1), 'M': (1, 1024), 'G': (1024, 1024), 'T': (1024, 1048576), } if human.has_key(size[-1]): block_size = human[size[-1]][1] size = int(size[:-1]) * human[size[-1]][0] qemu_img_cmd = ("dd if=/dev/zero of=%s count=%s bs=%sK" % (image_filename, size, block_size)) else: qemu_img_cmd = virt_utils.get_path(root_dir, params.get("qemu_img_binary", "qemu-img")) qemu_img_cmd += " create" qemu_img_cmd += " -f %s" % format image_cluster_size = params.get("image_cluster_size", None) if image_cluster_size is not None: qemu_img_cmd += " -o cluster_size=%s" % image_cluster_size qemu_img_cmd += " %s" % image_filename qemu_img_cmd += " %s" % size utils.system(qemu_img_cmd) return image_filename
def get_image_filename(params, root_dir): """ Generate an image path from params and root_dir. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) @raise VMDeviceError: When no matching disk found (in indirect method). """ image_name = params.get("image_name", "image") indirect_image_select = params.get("indirect_image_select") if indirect_image_select: re_name = image_name indirect_image_select = int(indirect_image_select) matching_images = utils.system_output("ls -1d %s" % re_name) matching_images = sorted(matching_images.split('\n')) if matching_images[-1] == '': matching_images = matching_images[:-1] try: image_name = matching_images[indirect_image_select] except IndexError: raise virt_vm.VMDeviceError( "No matching disk found for " "name = '%s', matching = '%s' and " "selector = '%s'" % (re_name, matching_images, indirect_image_select)) for protected in params.get('indirect_image_blacklist', '').split(' '): if re.match(protected, image_name): raise virt_vm.VMDeviceError( "Matching disk is in blacklist. " "name = '%s', matching = '%s' and " "selector = '%s'" % (re_name, matching_images, indirect_image_select)) image_format = params.get("image_format", "qcow2") if params.get("image_raw_device") == "yes": return image_name if image_format: image_filename = "%s.%s" % (image_name, image_format) else: image_filename = image_name image_filename = virt_utils.get_path(root_dir, image_filename) return image_filename
def get_image_filename(params, root_dir): """ Generate an image path from params and root_dir. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) @raise VMDeviceError: When no matching disk found (in indirect method). """ image_name = params.get("image_name", "image") indirect_image_select = params.get("indirect_image_select") if indirect_image_select: re_name = image_name indirect_image_select = int(indirect_image_select) matching_images = utils.system_output("ls -1d %s" % re_name) matching_images = sorted(matching_images.split("\n")) if matching_images[-1] == "": matching_images = matching_images[:-1] try: image_name = matching_images[indirect_image_select] except IndexError: raise virt_vm.VMDeviceError( "No matching disk found for " "name = '%s', matching = '%s' and " "selector = '%s'" % (re_name, matching_images, indirect_image_select) ) for protected in params.get("indirect_image_blacklist", "").split(" "): if re.match(protected, image_name): raise virt_vm.VMDeviceError( "Matching disk is in blacklist. " "name = '%s', matching = '%s' and " "selector = '%s'" % (re_name, matching_images, indirect_image_select) ) image_format = params.get("image_format", "qcow2") if params.get("image_raw_device") == "yes": return image_name if image_format: image_filename = "%s.%s" % (image_name, image_format) else: image_filename = image_name image_filename = virt_utils.get_path(root_dir, image_filename) return image_filename
def get_image_blkdebug_filename(params, root_dir): """ Generate an blkdebug file path from params and root_dir. blkdebug files allow error injection in the block subsystem. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: blkdebug -- the name of the debug file. """ blkdebug_name = params.get("drive_blkdebug", None) if blkdebug_name is not None: blkdebug_filename = virt_utils.get_path(root_dir, blkdebug_name) else: blkdebug_filename = None return blkdebug_filename
def get_image_filename(params, root_dir): """ Generate an image path from params and root_dir. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) """ image_name = params.get("image_name", "image") image_format = params.get("image_format", "qcow2") if params.get("image_raw_device") == "yes": return image_name image_filename = "%s.%s" % (image_name, image_format) image_filename = virt_utils.get_path(root_dir, image_filename) return image_filename
def preprocess(test, params, env): """ Preprocess all VMs and images according to the instructions in params. Also, collect some host information, such as the KVM version. @param test: An Autotest test object. @param params: A dict containing all VM and image parameters. @param env: The environment (a dict-like object). """ error.context("preprocessing") # Start tcpdump if it isn't already running if "address_cache" not in env: env["address_cache"] = {} if "tcpdump" in env and not env["tcpdump"].is_alive(): env["tcpdump"].close() del env["tcpdump"] if "tcpdump" not in env and params.get("run_tcpdump", "yes") == "yes": cmd = "%s -npvi any 'dst port 68'" % virt_utils.find_command("tcpdump") logging.debug("Starting tcpdump '%s'", cmd) env["tcpdump"] = aexpect.Tail(command=cmd, output_func=_update_address_cache, output_params=(env["address_cache"], )) if virt_utils.wait_for(lambda: not env["tcpdump"].is_alive(), 0.1, 0.1, 1.0): logging.warn("Could not start tcpdump") logging.warn("Status: %s" % env["tcpdump"].get_status()) logging.warn( "Output:" + virt_utils.format_str_for_message(env["tcpdump"].get_output())) # Destroy and remove VMs that are no longer needed in the environment requested_vms = params.objects("vms") for key in env.keys(): vm = env[key] if not virt_utils.is_vm(vm): continue if not vm.name in requested_vms: logging.debug("VM '%s' found in environment but not required for " "test, destroying it" % vm.name) vm.destroy() del env[key] # Get the KVM kernel module version and write it as a keyval if os.path.exists("/dev/kvm"): try: kvm_version = open("/sys/module/kvm/version").read().strip() except Exception: kvm_version = os.uname()[2] else: kvm_version = "Unknown" logging.debug("KVM module not loaded") logging.debug("KVM version: %s" % kvm_version) test.write_test_keyval({"kvm_version": kvm_version}) # Get the KVM userspace version and write it as a keyval qemu_path = virt_utils.get_path(test.bindir, params.get("qemu_binary", "qemu")) version_line = commands.getoutput("%s -help | head -n 1" % qemu_path) matches = re.findall("[Vv]ersion .*?,", version_line) if matches: kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",") else: kvm_userspace_version = "Unknown" logging.debug("KVM userspace version: %s" % kvm_userspace_version) test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version}) if params.get("setup_hugepages") == "yes": h = virt_test_setup.HugePageConfig(params) h.setup() if params.get("vm_type") == "libvirt": libvirt_vm.libvirtd_restart() # Execute any pre_commands if params.get("pre_command"): process_command(test, params, env, params.get("pre_command"), int(params.get("pre_command_timeout", "600")), params.get("pre_command_noncritical") == "yes") #Clone master image from vms. if params.get("master_images_clone"): for vm_name in params.get("vms").split(): vm = env.get_vm(vm_name) if vm: vm.destroy(free_mac_addresses=False) env.unregister_vm(vm_name) vm_params = params.object_params(vm_name) for image in vm_params.get("master_images_clone").split(): virt_vm.clone_image(params, vm_name, image, test.bindir) # Preprocess all VMs and images if params.get("not_preprocess", "no") == "no": process(test, params, env, preprocess_image, preprocess_vm) # Start the screendump thread if params.get("take_regular_screendumps") == "yes": logging.debug("Starting screendump thread") global _screendump_thread, _screendump_thread_termination_event _screendump_thread_termination_event = threading.Event() _screendump_thread = threading.Thread(target=_take_screendumps, args=(test, params, env)) _screendump_thread.start()
def backup_image(params, root_dir, action, good=True): """ Backup or restore a disk image, depending on the action chosen. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @param action: Whether we want to backup or restore the image. @param good: If we are backing up a good image(we want to restore it) or a bad image (we are saving a bad image for posterior analysis). @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) """ def backup_raw_device(src, dst): utils.system("dd if=%s of=%s bs=4k conv=sync" % (src, dst)) def backup_image_file(src, dst): logging.debug("Copying %s -> %s", src, dst) shutil.copy(src, dst) def get_backup_name(filename, backup_dir, good): if not os.path.isdir(backup_dir): os.makedirs(backup_dir) basename = os.path.basename(filename) if good: backup_filename = "%s.backup" % basename else: backup_filename = ( "%s.bad.%s" % (basename, virt_utils.generate_random_string(4))) return os.path.join(backup_dir, backup_filename) image_filename = get_image_filename(params, root_dir) backup_dir = params.get("backup_dir") if params.get('image_raw_device') == 'yes': iname = "raw_device" iformat = params.get("image_format", "qcow2") ifilename = "%s.%s" % (iname, iformat) ifilename = virt_utils.get_path(root_dir, ifilename) image_filename_backup = get_backup_name(ifilename, backup_dir, good) backup_func = backup_raw_device else: image_filename_backup = get_backup_name(image_filename, backup_dir, good) backup_func = backup_image_file if action == 'backup': image_dir = os.path.dirname(image_filename) image_dir_disk_free = utils.freespace(image_dir) image_filename_size = os.path.getsize(image_filename) image_filename_backup_size = 0 if os.path.isfile(image_filename_backup): image_filename_backup_size = os.path.getsize(image_filename_backup) disk_free = image_dir_disk_free + image_filename_backup_size minimum_disk_free = 1.2 * image_filename_size if disk_free < minimum_disk_free: image_dir_disk_free_gb = float(image_dir_disk_free) / 10**9 minimum_disk_free_gb = float(minimum_disk_free) / 10**9 logging.error( "Dir %s has %.1f GB free, less than the minimum " "required to store a backup, defined to be 120%% " "of the backup size, %.1f GB. Skipping backup...", image_dir, image_dir_disk_free_gb, minimum_disk_free_gb) return if good: # In case of qemu-img check return 1, we will make 2 backups, one # for investigation and other, to use as a 'pristine' image for # further tests state = 'good' else: state = 'bad' logging.info("Backing up %s image file %s", state, image_filename) src, dst = image_filename, image_filename_backup elif action == 'restore': if not os.path.isfile(image_filename_backup): logging.error('Image backup %s not found, skipping restore...', image_filename_backup) return logging.info("Restoring image file %s from backup", image_filename) src, dst = image_filename_backup, image_filename backup_func(src, dst)
def check_image(params, root_dir): """ Check an image using the appropriate tools for each virt backend. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) @raise VMImageCheckError: In case qemu-img check fails on the image. """ vm_type = params.get("vm_type") if vm_type == 'kvm': image_filename = get_image_filename(params, root_dir) logging.debug("Checking image file %s", image_filename) qemu_img_cmd = virt_utils.get_path( root_dir, params.get("qemu_img_binary", "qemu-img")) image_is_qcow2 = params.get("image_format") == 'qcow2' if os.path.exists(image_filename) and image_is_qcow2: # Verifying if qemu-img supports 'check' q_result = utils.run(qemu_img_cmd, ignore_status=True) q_output = q_result.stdout check_img = True if not "check" in q_output: logging.error("qemu-img does not support 'check', " "skipping check") check_img = False if not "info" in q_output: logging.error("qemu-img does not support 'info', " "skipping check") check_img = False if check_img: try: utils.system("%s info %s" % (qemu_img_cmd, image_filename)) except error.CmdError: logging.error("Error getting info from image %s", image_filename) cmd_result = utils.run("%s check %s" % (qemu_img_cmd, image_filename), ignore_status=True) # Error check, large chances of a non-fatal problem. # There are chances that bad data was skipped though if cmd_result.exit_status == 1: for e_line in cmd_result.stdout.splitlines(): logging.error("[stdout] %s", e_line) for e_line in cmd_result.stderr.splitlines(): logging.error("[stderr] %s", e_line) if params.get("backup_image_on_check_error", "no") == "yes": backup_image(params, root_dir, 'backup', False) raise error.TestWarn( "qemu-img check error. Some bad data " "in the image may have gone unnoticed") # Exit status 2 is data corruption for sure, so fail the test elif cmd_result.exit_status == 2: for e_line in cmd_result.stdout.splitlines(): logging.error("[stdout] %s", e_line) for e_line in cmd_result.stderr.splitlines(): logging.error("[stderr] %s", e_line) if params.get("backup_image_on_check_error", "no") == "yes": backup_image(params, root_dir, 'backup', False) raise VMImageCheckError(image_filename) # Leaked clusters, they are known to be harmless to data # integrity elif cmd_result.exit_status == 3: raise error.TestWarn("Leaked clusters were noticed during " "image check. No data integrity " "problem was found though.") # Just handle normal operation if params.get("backup_image", "no") == "yes": backup_image(params, root_dir, 'backup', True) else: if not os.path.exists(image_filename): logging.debug("Image file %s not found, skipping check", image_filename) elif not image_is_qcow2: logging.debug("Image file %s not qcow2, skipping check", image_filename)
def _take_screendumps(test, params, env): global _screendump_thread_termination_event temp_dir = test.debugdir if params.get("screendump_temp_dir"): temp_dir = virt_utils.get_path(test.bindir, params.get("screendump_temp_dir")) try: os.makedirs(temp_dir) except OSError: pass temp_filename = os.path.join( temp_dir, "scrdump-%s.ppm" % virt_utils.generate_random_string(6)) delay = float(params.get("screendump_delay", 5)) quality = int(params.get("screendump_quality", 30)) cache = {} while True: for vm in env.get_all_vms(): if not vm.is_alive(): continue try: vm.monitor.screendump(filename=temp_filename, debug=False) except kvm_monitor.MonitorError, e: logging.warning(e) continue except AttributeError, e: continue if not os.path.exists(temp_filename): logging.warning("VM '%s' failed to produce a screendump", vm.name) continue if not ppm_utils.image_verify_ppm_file(temp_filename): logging.warning("VM '%s' produced an invalid screendump", vm.name) os.unlink(temp_filename) continue screendump_dir = os.path.join(test.debugdir, "screendumps_%s" % vm.name) try: os.makedirs(screendump_dir) except OSError: pass screendump_filename = os.path.join( screendump_dir, "%s_%s.jpg" % (vm.name, time.strftime("%Y-%m-%d_%H-%M-%S"))) hash = utils.hash_file(temp_filename) if hash in cache: try: os.link(cache[hash], screendump_filename) except OSError: pass else: try: image = PIL.Image.open(temp_filename) image.save(screendump_filename, format="JPEG", quality=quality) cache[hash] = screendump_filename except NameError: pass os.unlink(temp_filename)
def _take_screendumps(test, params, env): global _screendump_thread_termination_event temp_dir = test.debugdir if params.get("screendump_temp_dir"): temp_dir = virt_utils.get_path(test.bindir, params.get("screendump_temp_dir")) try: os.makedirs(temp_dir) except OSError: pass temp_filename = os.path.join(temp_dir, "scrdump-%s.ppm" % virt_utils.generate_random_string(6)) delay = float(params.get("screendump_delay", 5)) quality = int(params.get("screendump_quality", 30)) cache = {} counter = {} while True: for vm in env.get_all_vms(): if vm not in counter.keys(): counter[vm] = 0 if not vm.is_alive(): continue try: vm.screendump(filename=temp_filename, debug=False) except kvm_monitor.MonitorError, e: logging.warn(e) continue except AttributeError, e: logging.warn(e) continue if not os.path.exists(temp_filename): logging.warn("VM '%s' failed to produce a screendump", vm.name) continue if not ppm_utils.image_verify_ppm_file(temp_filename): logging.warn("VM '%s' produced an invalid screendump", vm.name) os.unlink(temp_filename) continue screendump_dir = os.path.join(test.debugdir, "screendumps_%s" % vm.name) try: os.makedirs(screendump_dir) except OSError: pass counter[vm] += 1 screendump_filename = os.path.join(screendump_dir, "%04d.jpg" % counter[vm]) hash = utils.hash_file(temp_filename) if hash in cache: try: os.link(cache[hash], screendump_filename) except OSError: pass else: try: try: image = PIL.Image.open(temp_filename) image.save(screendump_filename, format="JPEG", quality=quality) cache[hash] = screendump_filename except IOError, error_detail: logging.warning("VM '%s' failed to produce a " "screendump: %s", vm.name, error_detail) # Decrement the counter as we in fact failed to # produce a converted screendump counter[vm] -= 1 except NameError: pass os.unlink(temp_filename)
def preprocess(test, params, env): """ Preprocess all VMs and images according to the instructions in params. Also, collect some host information, such as the KVM version. @param test: An Autotest test object. @param params: A dict containing all VM and image parameters. @param env: The environment (a dict-like object). """ error.context("preprocessing") # Start tcpdump if it isn't already running if "address_cache" not in env: env["address_cache"] = {} if "tcpdump" in env and not env["tcpdump"].is_alive(): env["tcpdump"].close() del env["tcpdump"] if "tcpdump" not in env and params.get("run_tcpdump", "yes") == "yes": cmd = "%s -npvi any 'dst port 68'" % virt_utils.find_command("tcpdump") logging.debug("Starting tcpdump '%s'", cmd) env["tcpdump"] = aexpect.Tail( command=cmd, output_func=_update_address_cache, output_params=(env["address_cache"],)) if virt_utils.wait_for(lambda: not env["tcpdump"].is_alive(), 0.1, 0.1, 1.0): logging.warn("Could not start tcpdump") logging.warn("Status: %s" % env["tcpdump"].get_status()) logging.warn("Output:" + virt_utils.format_str_for_message( env["tcpdump"].get_output())) # Destroy and remove VMs that are no longer needed in the environment requested_vms = params.objects("vms") for key in env.keys(): vm = env[key] if not virt_utils.is_vm(vm): continue if not vm.name in requested_vms: logging.debug("VM '%s' found in environment but not required for " "test, destroying it" % vm.name) vm.destroy() del env[key] # Get Host cpu type if params.get("auto_cpu_model") == "yes": if not env.get("cpu_model"): env["cpu_model"] = virt_utils.get_cpu_model() params["cpu_model"] = env.get("cpu_model") # Get the KVM kernel module version and write it as a keyval if os.path.exists("/dev/kvm"): try: kvm_version = open("/sys/module/kvm/version").read().strip() except Exception: kvm_version = os.uname()[2] else: kvm_version = "Unknown" logging.debug("KVM module not loaded") logging.debug("KVM version: %s" % kvm_version) test.write_test_keyval({"kvm_version": kvm_version}) # Get the KVM userspace version and write it as a keyval qemu_path = virt_utils.get_path(test.bindir, params.get("qemu_binary", "qemu")) version_line = commands.getoutput("%s -help | head -n 1" % qemu_path) matches = re.findall("[Vv]ersion .*?,", version_line) if matches: kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",") else: kvm_userspace_version = "Unknown" logging.debug("KVM userspace version: %s" % kvm_userspace_version) test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version}) if params.get("setup_hugepages") == "yes": h = virt_test_setup.HugePageConfig(params) h.setup() if params.get("vm_type") == "libvirt": libvirt_vm.libvirtd_restart() if params.get("setup_thp") == "yes": thp = virt_test_setup.TransparentHugePageConfig(test, params) thp.setup() # Execute any pre_commands if params.get("pre_command"): process_command(test, params, env, params.get("pre_command"), int(params.get("pre_command_timeout", "600")), params.get("pre_command_noncritical") == "yes") #Clone master image from vms. if params.get("master_images_clone"): for vm_name in params.get("vms").split(): vm = env.get_vm(vm_name) if vm: vm.destroy(free_mac_addresses=False) env.unregister_vm(vm_name) vm_params = params.object_params(vm_name) for image in vm_params.get("master_images_clone").split(): virt_utils.clone_image(params, vm_name, image, test.bindir) # Preprocess all VMs and images if params.get("not_preprocess","no") == "no": process(test, params, env, preprocess_image, preprocess_vm) # Start the screendump thread if params.get("take_regular_screendumps") == "yes": logging.debug("Starting screendump thread") global _screendump_thread, _screendump_thread_termination_event _screendump_thread_termination_event = threading.Event() _screendump_thread = threading.Thread(target=_take_screendumps, args=(test, params, env)) _screendump_thread.start()
kvm_version = "Unknown" logging.debug("KVM version: %s" % kvm_version) test.write_test_keyval({"kvm_version": kvm_version}) # Get the KVM userspace version and write it as a keyval kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "") if kvm_userspace_ver_cmd: try: cmd_result = utils.run(kvm_userspace_ver_cmd) kvm_userspace_version = cmd_result.stdout.strip() except error.CmdError, e: kvm_userspace_version = "Unknown" else: qemu_path = virt_utils.get_path(test.bindir, params.get("qemu_binary", "qemu")) version_line = commands.getoutput("%s -help | head -n 1" % qemu_path) matches = re.findall("[Vv]ersion .*?,", version_line) if matches: kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",") else: kvm_userspace_version = "Unknown" logging.debug("KVM userspace version: %s" % kvm_userspace_version) test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version}) if params.get("setup_hugepages") == "yes": h = virt_test_setup.HugePageConfig(params) h.setup() if params.get("vm_type") == "libvirt": libvirt_vm.libvirtd_restart()
def create(self, name=None, params=None, root_dir=None, timeout=5.0, migration_mode=None, mac_source=None): """ Start the VM by running a qemu command. All parameters are optional. If name, params or root_dir are not supplied, the respective values stored as class attributes are used. @param name: The name of the object @param params: A dict containing VM params @param root_dir: Base directory for relative filenames @param migration_mode: If supplied, start VM for incoming migration using this protocol (either 'tcp', 'unix' or 'exec') @param migration_exec_cmd: Command to embed in '-incoming "exec: ..."' (e.g. 'gzip -c -d filename') if migration_mode is 'exec' @param mac_source: A VM object from which to copy MAC addresses. If not specified, new addresses will be generated. @raise VMCreateError: If qemu terminates unexpectedly @raise VMKVMInitError: If KVM initialization fails @raise VMHugePageError: If hugepage initialization fails @raise VMImageMissingError: If a CD image is missing @raise VMHashMismatchError: If a CD image hash has doesn't match the expected hash @raise VMBadPATypeError: If an unsupported PCI assignment type is requested @raise VMPAError: If no PCI assignable devices could be assigned """ error.context("creating '%s'" % self.name) self.destroy(free_mac_addresses=False) if name is not None: self.name = name if params is not None: self.params = params if root_dir is not None: self.root_dir = root_dir name = self.name params = self.params root_dir = self.root_dir # Verify the md5sum of the ISO images for cdrom in params.objects("cdroms"): cdrom_params = params.object_params(cdrom) iso = cdrom_params.get("cdrom") if iso: iso = virt_utils.get_path(root_dir, iso) if not os.path.exists(iso): raise virt_vm.VMImageMissingError(iso) compare = False if cdrom_params.get("md5sum_1m"): logging.debug("Comparing expected MD5 sum with MD5 sum of " "first MB of ISO file...") actual_hash = utils.hash_file(iso, 1048576, method="md5") expected_hash = cdrom_params.get("md5sum_1m") compare = True elif cdrom_params.get("md5sum"): logging.debug("Comparing expected MD5 sum with MD5 sum of " "ISO file...") actual_hash = utils.hash_file(iso, method="md5") expected_hash = cdrom_params.get("md5sum") compare = True elif cdrom_params.get("sha1sum"): logging.debug("Comparing expected SHA1 sum with SHA1 sum " "of ISO file...") actual_hash = utils.hash_file(iso, method="sha1") expected_hash = cdrom_params.get("sha1sum") compare = True if compare: if actual_hash == expected_hash: logging.debug("Hashes match") else: raise virt_vm.VMHashMismatchError(actual_hash, expected_hash) # Make sure the following code is not executed by more than one thread # at the same time lockfile = open("/tmp/libvirt-autotest-vm-create.lock", "w+") fcntl.lockf(lockfile, fcntl.LOCK_EX) try: # Handle port redirections redir_names = params.objects("redirs") host_ports = virt_utils.find_free_ports(5000, 6000, len(redir_names)) self.redirs = {} for i in range(len(redir_names)): redir_params = params.object_params(redir_names[i]) guest_port = int(redir_params.get("guest_port")) self.redirs[guest_port] = host_ports[i] # Generate netdev/device IDs for all NICs self.netdev_id = [] self.device_id = [] for nic in params.objects("nics"): self.netdev_id.append(virt_utils.generate_random_id()) self.device_id.append(virt_utils.generate_random_id()) # Find available PCI devices self.pci_devices = [] for device in params.objects("pci_devices"): self.pci_devices.append(device) # Find available VNC port, if needed if params.get("display") == "vnc": self.vnc_port = virt_utils.find_free_port(5900, 6100) # Find available spice port, if needed if params.get("spice"): self.spice_port = virt_utils.find_free_port(8000, 8100) # Find random UUID if specified 'uuid = random' in config file if params.get("uuid") == "random": f = open("/proc/sys/kernel/random/uuid") self.uuid = f.read().strip() f.close() # Generate or copy MAC addresses for all NICs num_nics = len(params.objects("nics")) for vlan in range(num_nics): nic_name = params.objects("nics")[vlan] nic_params = params.object_params(nic_name) mac = (nic_params.get("nic_mac") or mac_source and mac_source.get_mac_address(vlan)) if mac: virt_utils.set_mac_address(self.instance, vlan, mac) else: virt_utils.generate_mac_address(self.instance, vlan) # Make qemu command install_command = self.__make_libvirt_command() logging.info("Running libvirt command:\n%s", install_command) utils.run(install_command, verbose=False) # Wait for the domain to be created virt_utils.wait_for(func=self.is_alive, timeout=60, text=("waiting for domain %s to start" % self.name)) # Establish a session with the serial console -- requires a version # of netcat that supports -U self.serial_console = aexpect.ShellSession( "nc -U %s" % self.get_serial_console_filename(), auto_close=False, output_func=virt_utils.log_line, output_params=("serial-%s.log" % name,)) finally: fcntl.lockf(lockfile, fcntl.LOCK_UN) lockfile.close()
def _take_screendumps(test, params, env): global _screendump_thread_termination_event temp_dir = test.debugdir if params.get("screendump_temp_dir"): temp_dir = virt_utils.get_path(test.bindir, params.get("screendump_temp_dir")) try: os.makedirs(temp_dir) except OSError: pass temp_filename = os.path.join( temp_dir, "scrdump-%s.ppm" % virt_utils.generate_random_string(6)) delay = float(params.get("screendump_delay", 5)) quality = int(params.get("screendump_quality", 30)) cache = {} counter = {} while True: for vm in env.get_all_vms(): if vm not in counter.keys(): counter[vm] = 0 if not vm.is_alive(): continue try: vm.screendump(filename=temp_filename, debug=False) except kvm_monitor.MonitorError, e: logging.warn(e) continue except AttributeError, e: continue if not os.path.exists(temp_filename): logging.warn("VM '%s' failed to produce a screendump", vm.name) continue if not ppm_utils.image_verify_ppm_file(temp_filename): logging.warn("VM '%s' produced an invalid screendump", vm.name) os.unlink(temp_filename) continue screendump_dir = os.path.join(test.debugdir, "screendumps_%s" % vm.name) try: os.makedirs(screendump_dir) except OSError: pass counter[vm] += 1 screendump_filename = os.path.join(screendump_dir, "%04d.jpg" % counter[vm]) hash = utils.hash_file(temp_filename) if hash in cache: try: os.link(cache[hash], screendump_filename) except OSError: pass else: try: try: image = PIL.Image.open(temp_filename) image.save(screendump_filename, format="JPEG", quality=quality) cache[hash] = screendump_filename except IOError, error_detail: logging.warning( "VM '%s' failed to produce a " "screendump: %s", vm.name, error_detail) # Decrement the counter as we in fact failed to # produce a converted screendump counter[vm] -= 1 except NameError: pass os.unlink(temp_filename)
def check_image(params, root_dir): """ Check an image using the appropriate tools for each virt backend. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) @raise VMImageCheckError: In case qemu-img check fails on the image. """ vm_type = params.get("vm_type") if vm_type == 'kvm': image_filename = get_image_filename(params, root_dir) logging.debug("Checking image file %s", image_filename) qemu_img_cmd = virt_utils.get_path(root_dir, params.get("qemu_img_binary", "qemu-img")) image_is_qcow2 = params.get("image_format") == 'qcow2' if os.path.exists(image_filename) and image_is_qcow2: # Verifying if qemu-img supports 'check' q_result = utils.run(qemu_img_cmd, ignore_status=True) q_output = q_result.stdout check_img = True if not "check" in q_output: logging.error("qemu-img does not support 'check', " "skipping check") check_img = False if not "info" in q_output: logging.error("qemu-img does not support 'info', " "skipping check") check_img = False if check_img: try: utils.system("%s info %s" % (qemu_img_cmd, image_filename)) except error.CmdError: logging.error("Error getting info from image %s", image_filename) cmd_result = utils.run("%s check %s" % (qemu_img_cmd, image_filename), ignore_status=True) # Error check, large chances of a non-fatal problem. # There are chances that bad data was skipped though if cmd_result.exit_status == 1: for e_line in cmd_result.stdout.splitlines(): logging.error("[stdout] %s", e_line) for e_line in cmd_result.stderr.splitlines(): logging.error("[stderr] %s", e_line) if params.get("backup_image_on_check_error", "no") == "yes": backup_image(params, root_dir, 'backup', False) raise error.TestWarn("qemu-img check error. Some bad data " "in the image may have gone unnoticed") # Exit status 2 is data corruption for sure, so fail the test elif cmd_result.exit_status == 2: for e_line in cmd_result.stdout.splitlines(): logging.error("[stdout] %s", e_line) for e_line in cmd_result.stderr.splitlines(): logging.error("[stderr] %s", e_line) if params.get("backup_image_on_check_error", "no") == "yes": backup_image(params, root_dir, 'backup', False) raise VMImageCheckError(image_filename) # Leaked clusters, they are known to be harmless to data # integrity elif cmd_result.exit_status == 3: raise error.TestWarn("Leaked clusters were noticed during " "image check. No data integrity " "problem was found though.") # Just handle normal operation if params.get("backup_image", "no") == "yes": backup_image(params, root_dir, 'backup', True) else: if not os.path.exists(image_filename): logging.debug("Image file %s not found, skipping check", image_filename) elif not image_is_qcow2: logging.debug("Image file %s not qcow2, skipping check", image_filename)
def __make_libvirt_command(self, name=None, params=None, root_dir=None): """ Generate a libvirt command line. All parameters are optional. If a parameter is not supplied, the corresponding value stored in the class attributes is used. @param name: The name of the object @param params: A dict containing VM params @param root_dir: Base directory for relative filenames @note: The params dict should contain: mem -- memory size in MBs cdrom -- ISO filename to use with the qemu -cdrom parameter extra_params -- a string to append to the qemu command shell_port -- port of the remote shell daemon on the guest (SSH, Telnet or the home-made Remote Shell Server) shell_client -- client program to use for connecting to the remote shell daemon on the guest (ssh, telnet or nc) x11_display -- if specified, the DISPLAY environment variable will be be set to this value for the qemu process (useful for SDL rendering) images -- a list of image object names, separated by spaces nics -- a list of NIC object names, separated by spaces For each image in images: drive_format -- string to pass as 'if' parameter for this image (e.g. ide, scsi) image_snapshot -- if yes, pass 'snapshot=on' to qemu for this image image_boot -- if yes, pass 'boot=on' to qemu for this image In addition, all parameters required by get_image_filename. For each NIC in nics: nic_model -- string to pass as 'model' parameter for this NIC (e.g. e1000) """ # helper function for command line option wrappers def has_option(help, option): return bool(re.search(r"--%s" % option, help, re.MULTILINE)) # Wrappers for all supported libvirt command line parameters. # This is meant to allow support for multiple libvirt versions. # Each of these functions receives the output of 'libvirt --help' as a # parameter, and should add the requested command line option # accordingly. def add_name(help, name): return " --name '%s'" % name def add_hvm_or_pv(help, hvm_or_pv): return " %s" % hvm_or_pv def add_mem(help, mem): return " --ram=%s" % mem def add_check_cpu(help): if has_option(help, "check-cpu"): return " --check-cpu" else: return "" def add_smp(help, smp): return " --vcpu=%s" % smp def add_location(help, location): if has_option(help, "location"): return " --location %s" % location else: return "" def add_cdrom(help, filename, index=None): if has_option(help, "cdrom"): return " --cdrom %s" % filename else: return "" def add_pxe(help): if has_option(help, "pxe"): return " --pxe" else: return "" def add_drive(help, filename, pool=None, vol=None, device=None, bus=None, perms=None, size=None, sparse=False, cache=None, format=None): cmd = " --disk" if filename: cmd += " path=%s" % filename elif pool: if vol: cmd += " vol=%s/%s" % (pool, vol) else: cmd += " pool=%s" % pool if device: cmd += ",device=%s" % device if bus: cmd += ",bus=%s" % bus if perms: cmd += ",%s" % perms if size: cmd += ",size=%s" % size.rstrip("Gg") if sparse: cmd += ",sparse=false" if format: cmd += ",format=%s" % format return cmd def add_floppy(help, filename): return " --disk path=%s,device=floppy,ro" % filename def add_vnc(help, vnc_port): return " --vnc --vncport=%d" % (vnc_port) def add_sdl(help): if has_option(help, "sdl"): return " --sdl" else: return "" def add_nographic(help): return " --nographics" def add_video(help, video_device): if has_option(help, "video"): return " --video=%s" % (video_device) else: return "" def add_uuid(help, uuid): if has_option(help, "uuid"): return " --uuid %s" % uuid else: return "" def add_os_type(help, os_type): if has_option(help, "os-type"): return " --os-type %s" % os_type else: return "" def add_os_variant(help, os_variant): if has_option(help, "os-variant"): return " --os-variant %s" % os_variant else: return "" def add_pcidevice(help, pci_device): if has_option(help, "host-device"): return " --host-device %s" % pci_device else: return "" def add_soundhw(help, sound_device): if has_option(help, "soundhw"): return " --soundhw %s" % sound_device else: return "" def add_serial(help, filename): if has_option(help, "serial"): return " --serial file,path=%s --serial pty" % filename else: return "" def add_kernel_cmdline(help, cmdline): return " -append %s" % cmdline def add_connect_uri(help, uri): if has_option(help, "connect"): return " --connect=%s" % uri else: return "" # End of command line option wrappers if name is None: name = self.name if params is None: params = self.params if root_dir is None: root_dir = self.root_dir # Clone this VM using the new params vm = self.clone(name, params, root_dir, copy_state=True) virt_install_binary = virt_utils.get_path( root_dir, params.get("virt_install_binary", "virt-install")) help = utils.system_output("%s --help" % virt_install_binary) # Start constructing the qemu command virt_install_cmd = "" # Set the X11 display parameter if requested if params.get("x11_display"): virt_install_cmd += "DISPLAY=%s " % params.get("x11_display") # Add the qemu binary virt_install_cmd += virt_install_binary # set connect uri virt_install_cmd += add_connect_uri(help, self.connect_uri) # hvm or pv specificed by libvirt switch (pv used by Xen only) hvm_or_pv = params.get("hvm_or_pv") if hvm_or_pv: virt_install_cmd += add_hvm_or_pv(help, hvm_or_pv) # Add the VM's name virt_install_cmd += add_name(help, name) mem = params.get("mem") if mem: virt_install_cmd += add_mem(help, mem) # TODO: should we do the check before we call ? negative case ? check_cpu = params.get("use_check_cpu") if check_cpu: virt_install_cmd += add_check_cpu(help) smp = params.get("smp") if smp: virt_install_cmd += add_smp(help, smp) # TODO: directory location for vmlinuz/kernel for cdrom install ? location = None if params.get("medium") == 'url': if params.get("url") == 'auto': location = params.get('auto_content_url') else: location = params.get('url') elif params.get("medium") == 'kernel_initrd': # directory location of kernel/initrd pair (directory layout must # be in format libvirt will recognize) location = params.get("image_dir") elif params.get("medium") == 'nfs': location = "nfs:%s:%s" % (params.get("nfs_server"), params.get("nfs_dir")) elif params.get("medium") == 'cdrom': if params.get("use_libvirt_cdrom_switch") == 'yes': virt_install_cmd += add_cdrom(help, params.get("cdrom_cd1")) else: location = params.get("image_dir") if location: virt_install_cmd += add_location(help, location) if params.get("display") == "vnc": if params.get("vnc_port"): vm.vnc_port = int(params.get("vnc_port")) virt_install_cmd += add_vnc(help, vm.vnc_port) elif params.get("display") == "sdl": virt_install_cmd += add_sdl(help) elif params.get("display") == "nographic": virt_install_cmd += add_nographic(help) video_device = params.get("video_device") if video_device: virt_install_cmd += add_video(help, video_device) sound_device = params.get("sound_device") if sound_device: virt_install_cmd += add_soundhw(help, sound_device) # if none is given a random UUID will be generated by libvirt if params.get("uuid"): virt_install_cmd += add_uuid(help, params.get("uuid")) # selectable OS type if params.get("use_os_type") == "yes": virt_install_cmd += add_os_type(help, params.get("os_type")) # selectable OS variant if params.get("use_os_variant") == "yes": virt_install_cmd += add_os_variant(help, params.get("os_variant")) # If the PCI assignment step went OK, add each one of the PCI assigned # devices to the command line. if self.pci_devices: for pci_id in self.pci_devices: virt_install_cmd += add_pcidevice(help, pci_id) for image_name in params.objects("images"): image_params = params.object_params(image_name) filename = virt_vm.get_image_filename(image_params, root_dir) if image_params.get("use_storage_pool") == "yes": filename = None if image_params.get("boot_drive") == "no": continue virt_install_cmd += add_drive(help, filename, image_params.get("image_pool"), image_params.get("image_vol"), image_params.get("image_device"), image_params.get("image_bus"), image_params.get("image_perms"), image_params.get("image_size"), image_params.get("drive_sparse"), image_params.get("drive_cache"), image_params.get("image_format")) if self.driver_type == self.LIBVIRT_QEMU: for cdrom in params.objects("cdroms"): cdrom_params = params.object_params(cdrom) iso = cdrom_params.get("cdrom") if params.get("use_libvirt_cdrom_switch") == 'yes': # we don't want to skip the winutils iso if not cdrom == 'winutils': logging.debug("Using --cdrom instead of --disk for install") logging.debug("Skipping CDROM:%s:%s", cdrom, iso) continue if params.get("medium") == 'cdrom_no_kernel_initrd': if iso == params.get("cdrom_cd1"): logging.debug("Using cdrom or url for install") logging.debug("Skipping CDROM: %s", iso) continue if iso: virt_install_cmd += add_drive(help, virt_utils.get_path(root_dir, iso), image_params.get("iso_image_pool"), image_params.get("iso_image_vol"), 'cdrom', None, None, None, None, None, None) # We may want to add {floppy_otps} parameter for -fda # {fat:floppy:}/path/. However vvfat is not usually recommended. floppy = params.get("floppy") if floppy: floppy = virt_utils.get_path(root_dir, floppy) virt_install_cmd += add_drive(help, floppy, None, None, 'floppy', None, None, None, None, None, None) # FIXME: for now in the pilot always add mac address to virt-install vlan = 0 mac = vm.get_mac_address(vlan) if mac: virt_install_cmd += " --mac %s" % mac self.nic_mac = mac if self.driver_type == self.LIBVIRT_XEN: virt_install_cmd += (" --network=%s" % params.get("virsh_network")) elif self.driver_type == self.LIBVIRT_QEMU: virt_install_cmd += (" --network=%s,model=%s" % (params.get("virsh_network"), params.get("nic_model"))) if params.get("use_no_reboot") == "yes": virt_install_cmd += " --noreboot" if params.get("use_autostart") == "yes": virt_install_cmd += " --autostart" if params.get("virt_install_debug") == "yes": virt_install_cmd += " --debug" # bz still open, not fully functional yet if params.get("use_virt_install_wait") == "yes": virt_install_cmd += (" --wait %s" % params.get("virt_install_wait_time")) extra_params = params.get("extra_params") if extra_params: virt_install_cmd += " --extra-args '%s'" % extra_params virt_install_cmd += " --noautoconsole" return virt_install_cmd
def __make_libvirt_command(self, name=None, params=None, root_dir=None): """ Generate a libvirt command line. All parameters are optional. If a parameter is not supplied, the corresponding value stored in the class attributes is used. @param name: The name of the object @param params: A dict containing VM params @param root_dir: Base directory for relative filenames @note: The params dict should contain: mem -- memory size in MBs cdrom -- ISO filename to use with the qemu -cdrom parameter extra_params -- a string to append to the qemu command shell_port -- port of the remote shell daemon on the guest (SSH, Telnet or the home-made Remote Shell Server) shell_client -- client program to use for connecting to the remote shell daemon on the guest (ssh, telnet or nc) x11_display -- if specified, the DISPLAY environment variable will be be set to this value for the qemu process (useful for SDL rendering) images -- a list of image object names, separated by spaces nics -- a list of NIC object names, separated by spaces For each image in images: drive_format -- string to pass as 'if' parameter for this image (e.g. ide, scsi) image_snapshot -- if yes, pass 'snapshot=on' to qemu for this image image_boot -- if yes, pass 'boot=on' to qemu for this image In addition, all parameters required by get_image_filename. For each NIC in nics: nic_model -- string to pass as 'model' parameter for this NIC (e.g. e1000) """ # helper function for command line option wrappers def has_option(help, option): return bool(re.search(r"--%s" % option, help, re.MULTILINE)) # Wrappers for all supported libvirt command line parameters. # This is meant to allow support for multiple libvirt versions. # Each of these functions receives the output of 'libvirt --help' as a # parameter, and should add the requested command line option # accordingly. def add_name(help, name): return " --name '%s'" % name def add_hvm_or_pv(help, hvm_or_pv): if hvm_or_pv == "hvm": return " --hvm --accelerate" elif hvm_or_pv == "pv": return " --paravirt" else: logging.warning("Unknown virt type hvm_or_pv, using default.") return "" def add_mem(help, mem): return " --ram=%s" % mem def add_check_cpu(help): if has_option(help, "check-cpu"): return " --check-cpu" else: return "" def add_smp(help, smp): return " --vcpu=%s" % smp def add_location(help, location): if has_option(help, "location"): return " --location %s" % location else: return "" def add_cdrom(help, filename, index=None): if has_option(help, "cdrom"): return " --cdrom %s" % filename else: return "" def add_pxe(help): if has_option(help, "pxe"): return " --pxe" else: return "" def add_drive(help, filename, pool=None, vol=None, device=None, bus=None, perms=None, size=None, sparse=False, cache=None, format=None): cmd = " --disk" if filename: cmd += " path=%s" % filename elif pool: if vol: cmd += " vol=%s/%s" % (pool, vol) else: cmd += " pool=%s" % pool if device: cmd += ",device=%s" % device if bus: cmd += ",bus=%s" % bus if perms: cmd += ",%s" % perms if size: cmd += ",size=%s" % size.rstrip("Gg") if sparse: cmd += ",sparse=false" if format: cmd += ",format=%s" % format return cmd def add_floppy(help, filename): return " --disk path=%s,device=floppy,ro" % filename def add_vnc(help, vnc_port): return " --vnc --vncport=%d" % (vnc_port) def add_vnclisten(help, vnclisten): return " --vnclisten=%s " % (vnclisten) def add_sdl(help): if has_option(help, "sdl"): return " --sdl" else: return "" def add_nographic(help): return " --nographics" def add_video(help, video_device): if has_option(help, "video"): return " --video=%s" % (video_device) else: return "" def add_uuid(help, uuid): if has_option(help, "uuid"): return " --uuid %s" % uuid else: return "" def add_os_type(help, os_type): if has_option(help, "os-type"): return " --os-type %s" % os_type else: return "" def add_os_variant(help, os_variant): if has_option(help, "os-variant"): return " --os-variant %s" % os_variant else: return "" def add_pcidevice(help, pci_device): if has_option(help, "host-device"): return " --host-device %s" % pci_device else: return "" def add_soundhw(help, sound_device): if has_option(help, "soundhw"): return " --soundhw %s" % sound_device else: return "" def add_serial(help, filename): if has_option(help, "serial"): return " --serial file,path=%s --serial pty" % filename else: return "" def add_kernel_cmdline(help, cmdline): return " -append %s" % cmdline def add_connect_uri(help, uri): if has_option(help, "connect"): return " --connect=%s" % uri else: return "" # End of command line option wrappers if name is None: name = self.name if params is None: params = self.params if root_dir is None: root_dir = self.root_dir # Clone this VM using the new params vm = self.clone(name, params, root_dir, copy_state=True) virt_install_binary = virt_utils.get_path( root_dir, params.get("virt_install_binary", "virt-install")) help = utils.system_output("%s --help" % virt_install_binary) # Start constructing the qemu command virt_install_cmd = "" # Set the X11 display parameter if requested if params.get("x11_display"): virt_install_cmd += "DISPLAY=%s " % params.get("x11_display") # Add the qemu binary virt_install_cmd += virt_install_binary # set connect uri virt_install_cmd += add_connect_uri(help, self.connect_uri) # hvm or pv specificed by libvirt switch (pv used by Xen only) hvm_or_pv = params.get("hvm_or_pv") if hvm_or_pv: virt_install_cmd += add_hvm_or_pv(help, hvm_or_pv) # Add the VM's name virt_install_cmd += add_name(help, name) mem = params.get("mem") if mem: virt_install_cmd += add_mem(help, mem) # TODO: should we do the check before we call ? negative case ? check_cpu = params.get("use_check_cpu") if check_cpu: virt_install_cmd += add_check_cpu(help) smp = params.get("smp") if smp: virt_install_cmd += add_smp(help, smp) # libvirt expects --location <path>/images/pxeboot/<vmlinuz|initrd> location = None if params.get("medium") == 'url': location = params.get('url') elif params.get("medium") == 'kernel_initrd': # directory location of kernel/initrd pair (directory layout must # be in format libvirt will recognize) location = params.get("image_dir") elif params.get("medium") == 'nfs': location = "nfs:%s:%s" % (params.get("nfs_server"), params.get("nfs_dir")) elif params.get("medium") == 'cdrom': if params.get("use_libvirt_cdrom_switch") == 'yes': virt_install_cmd += add_cdrom(help, params.get("cdrom_cd1")) elif ((self.driver_type == self.LIBVIRT_XEN) and (params.get('hvm_or_pv') == 'hvm')): virt_install_cmd += add_cdrom(help, params.get("cdrom_unattended")) else: # Fake images/pxeboot using relative symlinks # Assumes kernel and initrd were copied to same dir # TODO: This and cooresponding add_cdrom() in unattended_install test # should be much cleaner. location = os.path.dirname(params.get("kernel")) try: os.symlink(".", os.path.join(location, "images")) os.symlink(".", os.path.join(location, "pxeboot")) except OSError: pass # ignore if already exists if location: virt_install_cmd += add_location(help, location) if params.get("display") == "vnc": if params.get("vnc_port"): vm.vnc_port = int(params.get("vnc_port")) virt_install_cmd += add_vnc(help, vm.vnc_port) if params.get("vnclisten"): vm.vnclisten = params.get("vnclisten") virt_install_cmd += add_vnclisten(help, vm.vnclisten) elif params.get("display") == "sdl": virt_install_cmd += add_sdl(help) elif params.get("display") == "nographic": virt_install_cmd += add_nographic(help) video_device = params.get("video_device") if video_device: virt_install_cmd += add_video(help, video_device) sound_device = params.get("sound_device") if sound_device: virt_install_cmd += add_soundhw(help, sound_device) # if none is given a random UUID will be generated by libvirt if params.get("uuid"): virt_install_cmd += add_uuid(help, params.get("uuid")) # selectable OS type if params.get("use_os_type") == "yes": virt_install_cmd += add_os_type(help, params.get("os_type")) # selectable OS variant if params.get("use_os_variant") == "yes": virt_install_cmd += add_os_variant(help, params.get("os_variant")) # If the PCI assignment step went OK, add each one of the PCI assigned # devices to the command line. if self.pci_devices: for pci_id in self.pci_devices: virt_install_cmd += add_pcidevice(help, pci_id) for image_name in params.objects("images"): image_params = params.object_params(image_name) filename = virt_vm.get_image_filename(image_params, root_dir) if image_params.get("use_storage_pool") == "yes": filename = None if image_params.get("boot_drive") == "no": continue virt_install_cmd += add_drive(help, filename, image_params.get("image_pool"), image_params.get("image_vol"), image_params.get("image_device"), image_params.get("image_bus"), image_params.get("image_perms"), image_params.get("image_size"), image_params.get("drive_sparse"), image_params.get("drive_cache"), image_params.get("image_format")) if self.driver_type == self.LIBVIRT_QEMU: for cdrom in params.objects("cdroms"): cdrom_params = params.object_params(cdrom) iso = cdrom_params.get("cdrom") if params.get("use_libvirt_cdrom_switch") == 'yes': # we don't want to skip the winutils iso if not cdrom == 'winutils': logging.debug( "Using --cdrom instead of --disk for install") logging.debug("Skipping CDROM:%s:%s", cdrom, iso) continue if params.get("medium") == 'cdrom_no_kernel_initrd': if iso == params.get("cdrom_cd1"): logging.debug("Using cdrom or url for install") logging.debug("Skipping CDROM: %s", iso) continue if iso: virt_install_cmd += add_drive( help, virt_utils.get_path(root_dir, iso), image_params.get("iso_image_pool"), image_params.get("iso_image_vol"), 'cdrom', None, None, None, None, None, None) # We may want to add {floppy_otps} parameter for -fda # {fat:floppy:}/path/. However vvfat is not usually recommended. floppy = params.get("floppy") if floppy: floppy = virt_utils.get_path(root_dir, floppy) virt_install_cmd += add_drive(help, floppy, None, None, 'floppy', None, None, None, None, None, None) # FIXME: for now in the pilot always add mac address to virt-install vlan = 0 mac = vm.get_mac_address(vlan) if mac: virt_install_cmd += " --mac %s" % mac self.nic_mac = mac if self.driver_type == self.LIBVIRT_XEN: virt_install_cmd += (" --network=%s" % params.get("virsh_network")) elif self.driver_type == self.LIBVIRT_QEMU: virt_install_cmd += ( " --network=%s,model=%s" % (params.get("virsh_network"), params.get("nic_model"))) if params.get("use_no_reboot") == "yes": virt_install_cmd += " --noreboot" if params.get("use_autostart") == "yes": virt_install_cmd += " --autostart" if params.get("virt_install_debug") == "yes": virt_install_cmd += " --debug" # bz still open, not fully functional yet if params.get("use_virt_install_wait") == "yes": virt_install_cmd += (" --wait %s" % params.get("virt_install_wait_time")) kernel_params = params.get("kernel_params") if kernel_params: virt_install_cmd += " --extra-args '%s'" % kernel_params virt_install_cmd += " --noautoconsole" return virt_install_cmd
def backup_image(self, params, root_dir, action, good=True): """ Backup or restore a disk image, depending on the action chosen. @param params: Dictionary containing the test parameters. @param root_dir: Base directory for relative filenames. @param action: Whether we want to backup or restore the image. @param good: If we are backing up a good image(we want to restore it) or a bad image (we are saving a bad image for posterior analysis). @note: params should contain: image_name -- the name of the image file, without extension image_format -- the format of the image (qcow2, raw etc) """ def backup_raw_device(src, dst): utils.system("dd if=%s of=%s bs=4k conv=sync" % (src, dst)) def backup_image_file(src, dst): logging.debug("Copying %s -> %s", src, dst) shutil.copy(src, dst) def get_backup_name(filename, backup_dir, good): if not os.path.isdir(backup_dir): os.makedirs(backup_dir) basename = os.path.basename(filename) if good: backup_filename = "%s.backup" % basename else: backup_filename = "%s.bad.%s" % (basename, virt_utils.generate_random_string(4)) return os.path.join(backup_dir, backup_filename) image_filename = self.image_filename backup_dir = params.get("backup_dir") if params.get("image_raw_device") == "yes": iname = "raw_device" iformat = params.get("image_format", "qcow2") ifilename = "%s.%s" % (iname, iformat) ifilename = virt_utils.get_path(root_dir, ifilename) image_filename_backup = get_backup_name(ifilename, backup_dir, good) backup_func = backup_raw_device else: image_filename_backup = get_backup_name(image_filename, backup_dir, good) backup_func = backup_image_file if action == "backup": image_dir = os.path.dirname(image_filename) image_dir_disk_free = utils.freespace(image_dir) image_filename_size = os.path.getsize(image_filename) image_filename_backup_size = 0 if os.path.isfile(image_filename_backup): image_filename_backup_size = os.path.getsize(image_filename_backup) disk_free = image_dir_disk_free + image_filename_backup_size minimum_disk_free = 1.2 * image_filename_size if disk_free < minimum_disk_free: image_dir_disk_free_gb = float(image_dir_disk_free) / 10 ** 9 minimum_disk_free_gb = float(minimum_disk_free) / 10 ** 9 logging.error( "Dir %s has %.1f GB free, less than the minimum " "required to store a backup, defined to be 120%% " "of the backup size, %.1f GB. Skipping backup...", image_dir, image_dir_disk_free_gb, minimum_disk_free_gb, ) return if good: # In case of qemu-img check return 1, we will make 2 backups, # one for investigation and other, to use as a 'pristine' # image for further tests state = "good" else: state = "bad" logging.info("Backing up %s image file %s", state, image_filename) src, dst = image_filename, image_filename_backup elif action == "restore": if not os.path.isfile(image_filename_backup): logging.error("Image backup %s not found, skipping restore...", image_filename_backup) return logging.info("Restoring image file %s from backup", image_filename) src, dst = image_filename_backup, image_filename backup_func(src, dst)
def create(self, name=None, params=None, root_dir=None, timeout=5.0, migration_mode=None, mac_source=None): """ Start the VM by running a qemu command. All parameters are optional. If name, params or root_dir are not supplied, the respective values stored as class attributes are used. @param name: The name of the object @param params: A dict containing VM params @param root_dir: Base directory for relative filenames @param migration_mode: If supplied, start VM for incoming migration using this protocol (either 'tcp', 'unix' or 'exec') @param migration_exec_cmd: Command to embed in '-incoming "exec: ..."' (e.g. 'gzip -c -d filename') if migration_mode is 'exec' @param mac_source: A VM object from which to copy MAC addresses. If not specified, new addresses will be generated. @raise VMCreateError: If qemu terminates unexpectedly @raise VMKVMInitError: If KVM initialization fails @raise VMHugePageError: If hugepage initialization fails @raise VMImageMissingError: If a CD image is missing @raise VMHashMismatchError: If a CD image hash has doesn't match the expected hash @raise VMBadPATypeError: If an unsupported PCI assignment type is requested @raise VMPAError: If no PCI assignable devices could be assigned """ error.context("creating '%s'" % self.name) self.destroy(free_mac_addresses=False) if name is not None: self.name = name if params is not None: self.params = params if root_dir is not None: self.root_dir = root_dir name = self.name params = self.params root_dir = self.root_dir # Verify the md5sum of the ISO images for cdrom in params.objects("cdroms"): cdrom_params = params.object_params(cdrom) iso = cdrom_params.get("cdrom") if ((self.driver_type == self.LIBVIRT_XEN) and (params.get('hvm_or_pv') == 'pv') and (os.path.basename(iso) == 'ks.iso')): continue if iso: iso = virt_utils.get_path(root_dir, iso) if not os.path.exists(iso): raise virt_vm.VMImageMissingError(iso) compare = False if cdrom_params.get("md5sum_1m"): logging.debug("Comparing expected MD5 sum with MD5 sum of " "first MB of ISO file...") actual_hash = utils.hash_file(iso, 1048576, method="md5") expected_hash = cdrom_params.get("md5sum_1m") compare = True elif cdrom_params.get("md5sum"): logging.debug("Comparing expected MD5 sum with MD5 sum of " "ISO file...") actual_hash = utils.hash_file(iso, method="md5") expected_hash = cdrom_params.get("md5sum") compare = True elif cdrom_params.get("sha1sum"): logging.debug("Comparing expected SHA1 sum with SHA1 sum " "of ISO file...") actual_hash = utils.hash_file(iso, method="sha1") expected_hash = cdrom_params.get("sha1sum") compare = True if compare: if actual_hash == expected_hash: logging.debug("Hashes match") else: raise virt_vm.VMHashMismatchError( actual_hash, expected_hash) # Make sure the following code is not executed by more than one thread # at the same time lockfile = open("/tmp/libvirt-autotest-vm-create.lock", "w+") fcntl.lockf(lockfile, fcntl.LOCK_EX) try: # Handle port redirections redir_names = params.objects("redirs") host_ports = virt_utils.find_free_ports(5000, 6000, len(redir_names)) self.redirs = {} for i in range(len(redir_names)): redir_params = params.object_params(redir_names[i]) guest_port = int(redir_params.get("guest_port")) self.redirs[guest_port] = host_ports[i] # Generate netdev/device IDs for all NICs self.netdev_id = [] self.device_id = [] for nic in params.objects("nics"): self.netdev_id.append(virt_utils.generate_random_id()) self.device_id.append(virt_utils.generate_random_id()) # Find available PCI devices self.pci_devices = [] for device in params.objects("pci_devices"): self.pci_devices.append(device) # Find available VNC port, if needed if params.get("display") == "vnc": self.vnc_port = virt_utils.find_free_port(5900, 6100) # Find available spice port, if needed if params.get("spice"): self.spice_port = virt_utils.find_free_port(8000, 8100) # Find random UUID if specified 'uuid = random' in config file if params.get("uuid") == "random": f = open("/proc/sys/kernel/random/uuid") self.uuid = f.read().strip() f.close() # Generate or copy MAC addresses for all NICs num_nics = len(params.objects("nics")) for vlan in range(num_nics): nic_name = params.objects("nics")[vlan] nic_params = params.object_params(nic_name) mac = (nic_params.get("nic_mac") or mac_source and mac_source.get_mac_address(vlan)) if mac: virt_utils.set_mac_address(self.instance, vlan, mac) else: virt_utils.generate_mac_address(self.instance, vlan) # Make qemu command install_command = self.__make_libvirt_command() logging.info("Running libvirt command:\n%s", install_command) utils.run(install_command, verbose=False) # Wait for the domain to be created virt_utils.wait_for(func=self.is_alive, timeout=60, text=("waiting for domain %s to start" % self.name)) # Establish a session with the serial console -- requires a version # of netcat that supports -U self.serial_console = aexpect.ShellSession( "nc -U %s" % self.get_serial_console_filename(), auto_close=False, output_func=virt_utils.log_line, output_params=("serial-%s.log" % name, )) finally: fcntl.lockf(lockfile, fcntl.LOCK_UN) lockfile.close()