Exemple #1
0
def get_asset_info(asset):
    asset_path = os.path.join(data_dir.get_download_dir(), '%s.ini' % asset)
    asset_cfg = ConfigParser.ConfigParser()
    asset_cfg.read(asset_path)

    url = asset_cfg.get(asset, 'url')
    sha1_url = asset_cfg.get(asset, 'sha1_url')
    title = asset_cfg.get(asset, 'title')
    destination = asset_cfg.get(asset, 'destination')
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)
    asset_exists = os.path.isfile(destination)

    # Optional fields
    try:
        destination_uncompressed = asset_cfg.get(asset,
                                                 'destination_uncompressed')
        if not os.path.isabs(destination_uncompressed):
            destination_uncompressed = os.path.join(data_dir.get_data_dir(),
                                                    destination)
        uncompress_cmd = asset_cfg.get(asset, 'uncompress_cmd')
    except:
        destination_uncompressed = None
        uncompress_cmd = None


    return {'url': url, 'sha1_url': sha1_url, 'destination': destination,
            'destination_uncompressed': destination_uncompressed,
            'uncompress_cmd': uncompress_cmd, 'shortname': asset,
            'title': title,
            'downloaded': asset_exists}
Exemple #2
0
def print_guest_list(options):
    """
    Helper function to pretty print the guest list.

    This function uses a paginator, if possible (inspired on git).

    @param options: OptParse object with cmdline options.
    @param cartesian_parser: Cartesian parser object with test options.
    """
    cfg = os.path.join(data_dir.get_root_dir(), options.type,
                       "cfg", "guest-os.cfg")
    cartesian_parser = cartesian_config.Parser()
    cartesian_parser.parse_file(cfg)
    pipe = get_paginator()
    index = 0
    pipe.write("Searched %s for guest images\n" %
               os.path.join(data_dir.get_data_dir(), 'images'))
    pipe.write("Available guests:")
    pipe.write("\n\n")
    for params in cartesian_parser.get_dicts():
        index += 1
        image_name = storage.get_image_filename(params, data_dir.get_data_dir())
        shortname = ".".join(params['name'].split(".")[1:])
        if os.path.isfile(image_name):
            out = (bcolors.blue + str(index) + bcolors.end + " " +
                   shortname + "\n")
        else:
            out = (bcolors.blue + str(index) + bcolors.end + " " +
                   shortname + " " + bcolors.yellow +
                   "(missing %s)" % os.path.basename(image_name) +
                   bcolors.end + "\n")
        pipe.write(out)
def print_guest_list(options):
    """
    Helper function to pretty print the guest list.

    This function uses a paginator, if possible (inspired on git).

    :param options: OptParse object with cmdline options.
    :param cartesian_parser: Cartesian parser object with test options.
    """
    pipe = get_paginator()
    # lvsb testing has no concept of guests
    if options.vt_type == 'lvsb':
        pipe.write("No guest types available for lvsb testing")
        return
    index = 0
    pipe.write("Searched %s for guest images\n" %
               os.path.join(data_dir.get_data_dir(), 'images'))
    pipe.write("Available guests:")
    pipe.write("\n\n")
    for params in get_guest_name_parser(options).get_dicts():
        index += 1
        base_dir = params.get("images_base_dir", data_dir.get_data_dir())
        image_name = storage.get_image_filename(params, base_dir)
        name = params['name']
        if os.path.isfile(image_name):
            out = (bcolors.blue + str(index) + bcolors.end + " " +
                   name + "\n")
        else:
            out = (bcolors.blue + str(index) + bcolors.end + " " +
                   name + " " + bcolors.yellow +
                   "(missing %s)" % os.path.basename(image_name) +
                   bcolors.end + "\n")
        pipe.write(out)
def print_guest_list(options):
    """
    Helper function to pretty print the guest list.

    This function uses a paginator, if possible (inspired on git).

    @param options: OptParse object with cmdline options.
    @param cartesian_parser: Cartesian parser object with test options.
    """
    pipe = get_paginator()
    index = 0
    pipe.write("Searched %s for guest images\n" %
               os.path.join(data_dir.get_data_dir(), 'images'))
    pipe.write("Available guests:")
    pipe.write("\n\n")
    for params in get_guest_name_parser(options).get_dicts():
        index += 1
        image_name = storage.get_image_filename(params, data_dir.get_data_dir())
        shortname = params['shortname']
        if os.path.isfile(image_name):
            out = (bcolors.blue + str(index) + bcolors.end + " " +
                   shortname + "\n")
        else:
            out = (bcolors.blue + str(index) + bcolors.end + " " +
                   shortname + " " + bcolors.yellow +
                   "(missing %s)" % os.path.basename(image_name) +
                   bcolors.end + "\n")
        pipe.write(out)
def bootstrap_tests(options):
    """
    Bootstrap process (download the appropriate JeOS file to data dir).

    This function will check whether the JeOS is in the right location of the
    data dir, if not, it will download it non interactively.

    @param options: OptParse object with program command line options.
    """
    test_dir = os.path.dirname(sys.modules[__name__].__file__)

    if options.type:
        test_dir = os.path.abspath(os.path.join(os.path.dirname(test_dir), options.type))
    elif options.config:
        test_dir = os.path.dirname(os.path.dirname(options.config))
        test_dir = os.path.abspath(test_dir)

    if options.type == "qemu":
        check_modules = ["kvm", "kvm-%s" % utils_misc.get_cpu_vendor(verbose=False)]
    else:
        check_modules = None
    online_docs_url = "https://github.com/autotest/virt-test/wiki"

    kwargs = {
        "test_name": options.type,
        "test_dir": test_dir,
        "base_dir": data_dir.get_data_dir(),
        "default_userspace_paths": None,
        "check_modules": check_modules,
        "online_docs_url": online_docs_url,
        "restore_image": options.restore,
        "interactive": False,
    }

    # Tolerance we have without printing a message for the user to wait (3 s)
    tolerance = 3
    failed = False
    wait_message_printed = False

    bg = utils.InterruptedThread(bootstrap.bootstrap, kwargs=kwargs)
    t_begin = time.time()
    bg.start()

    while bg.isAlive():
        t_elapsed = time.time() - t_begin
        if t_elapsed > tolerance and not wait_message_printed:
            print_stdout("Running setup. Please wait...")
            wait_message_printed = True
            sys.stdout.switch()
        time.sleep(0.1)

    if wait_message_printed:
        sys.stdout.switch()

    reason = None
    try:
        bg.join()
    except Exception, e:
        failed = True
        reason = e
Exemple #6
0
def get_file_asset(title, src_path, destination):
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)

    for ext in (".xz", ".gz", ".7z", ".bz2"):
        if os.path.exists(src_path + ext):
            destination = destination + ext
            logging.debug("Found source image %s", destination)
            return {
                "url": None,
                "sha1_url": None,
                "destination": src_path + ext,
                "destination_uncompressed": destination,
                "uncompress_cmd": None,
                "shortname": title,
                "title": title,
                "downloaded": True,
            }

    if os.path.exists(src_path):
        logging.debug("Found source image %s", destination)
        return {
            "url": src_path,
            "sha1_url": None,
            "destination": destination,
            "destination_uncompressed": None,
            "uncompress_cmd": None,
            "shortname": title,
            "title": title,
            "downloaded": os.path.exists(destination),
        }

    return None
Exemple #7
0
def postprocess_image(test, params, image_name):
    """
    Postprocess a single QEMU image according to the instructions in params.

    :param test: An Autotest test object.
    :param params: A dict containing image postprocessing parameters.
    """
    clone_master = params.get("clone_master", None)
    base_dir = data_dir.get_data_dir()
    image = qemu_storage.QemuImg(params, base_dir, image_name)
    if params.get("check_image") == "yes":
        try:
            if clone_master is None:
                image.check_image(params, base_dir)
            elif clone_master == "yes":
                if image_name in params.get("master_images_clone").split():
                    image.check_image(params, base_dir)
            # Allow test to overwrite any pre-testing  automatic backup
            # with a new backup. i.e. assume pre-existing image/backup
            # would not be usable after this test succeeds. The best
            # example for this is when 'unattended_install' is run.
            if params.get("backup_image", "no") == "yes":
                image.backup_image(params, base_dir, "backup", True)
            elif params.get("restore_image", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
        except Exception, e:
            if params.get("restore_image_on_check_error", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
            if params.get("remove_image_on_check_error", "no") == "yes":
                cl_images = params.get("master_images_clone", "")
                if image_name in cl_images.split():
                    image.remove()
            raise e
def bootstrap_tests(options):
    """
    Bootstrap process (download the appropriate JeOS file to data dir).

    This function will check whether the JeOS is in the right location of the
    data dir, if not, it will download it non interactively.

    @param options: OptParse object with program command line options.
    """
    test_dir = os.path.dirname(sys.modules[__name__].__file__)

    if options.type:
        test_dir = os.path.abspath(
            os.path.join(os.path.dirname(test_dir), options.type))
    elif options.config:
        test_dir = os.path.dirname(os.path.dirname(options.config))
        test_dir = os.path.abspath(test_dir)

    check_modules = [
        "kvm", "kvm-%s" % utils_misc.get_cpu_vendor(verbose=False)
    ]
    online_docs_url = "https://github.com/autotest/virt-test/wiki"

    kwargs = {
        'test_name': options.type,
        'test_dir': test_dir,
        'base_dir': data_dir.get_data_dir(),
        'default_userspace_paths': None,
        'check_modules': check_modules,
        'online_docs_url': online_docs_url,
        'restore_image': options.restore,
        'interactive': False
    }

    # Tolerance we have without printing a message for the user to wait (3 s)
    tolerance = 3
    failed = False
    wait_message_printed = False

    bg = utils.InterruptedThread(bootstrap.bootstrap, kwargs=kwargs)
    t_begin = time.time()
    bg.start()

    while bg.isAlive():
        t_elapsed = time.time() - t_begin
        if t_elapsed > tolerance and not wait_message_printed:
            print_stdout("Running setup. Please wait...")
            wait_message_printed = True
            sys.stdout.switch()
        time.sleep(0.1)

    if wait_message_printed:
        sys.stdout.switch()

    reason = None
    try:
        bg.join()
    except Exception, e:
        failed = True
        reason = e
Exemple #9
0
def preprocess_image(test, params, image_name):
    """
    Preprocess a single QEMU image according to the instructions in params.

    :param test: Autotest test object.
    :param params: A dict containing image preprocessing parameters.
    :note: Currently this function just creates an image if requested.
    """
    base_dir = params.get("images_base_dir", data_dir.get_data_dir())

    image_filename = storage.get_image_filename(params,
                                                base_dir)

    create_image = False

    if params.get("force_create_image") == "yes":
        create_image = True
    elif (params.get("create_image") == "yes" and not
          os.path.exists(image_filename)):
        create_image = True

    if params.get("backup_image_before_testing", "no") == "yes":
        image = qemu_storage.QemuImg(params, base_dir, image_name)
        image.backup_image(params, base_dir, "backup", True, True)
    if create_image:
        image = qemu_storage.QemuImg(params, base_dir, image_name)
        image.create(params)
Exemple #10
0
def preprocess_image(test, params, image_name):
    """
    Preprocess a single QEMU image according to the instructions in params.

    :param test: Autotest test object.
    :param params: A dict containing image preprocessing parameters.
    :note: Currently this function just creates an image if requested.
    """
    base_dir = params.get("images_base_dir", data_dir.get_data_dir())

    if not storage.preprocess_image_backend(base_dir, params, image_name):
        logging.error("Backend can't be prepared correctly.")

    image_filename = storage.get_image_filename(params,
                                                base_dir)

    create_image = False
    if params.get("force_create_image") == "yes":
        create_image = True
    elif (params.get("create_image") == "yes" and not
          storage.file_exists(params, image_filename)):
        create_image = True

    if params.get("backup_image_before_testing", "no") == "yes":
        image = qemu_storage.QemuImg(params, base_dir, image_name)
        image.backup_image(params, base_dir, "backup", True, True)
    if create_image:
        image = qemu_storage.QemuImg(params, base_dir, image_name)
        image.create(params)
Exemple #11
0
def get_file_asset(title, src_path, destination):
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)

    for ext in (".xz", ".gz", ".7z", ".bz2"):
        if os.path.exists(src_path + ext):
            destination = destination + ext
            logging.debug('Found source image %s', destination)
            return {
                'url': None,
                'sha1_url': None,
                'destination': src_path + ext,
                'destination_uncompressed': destination,
                'uncompress_cmd': None,
                'shortname': title,
                'title': title,
                'downloaded': True
            }

    if os.path.exists(src_path):
        logging.debug('Found source image %s', destination)
        return {
            'url': src_path,
            'sha1_url': None,
            'destination': destination,
            'destination_uncompressed': None,
            'uncompress_cmd': None,
            'shortname': title,
            'title': title,
            'downloaded': os.path.exists(destination)
        }

    return None
Exemple #12
0
def create_gluster_vol(params):
    vol_name = params.get("gluster_volume_name")
    force = params.get('force_recreate_gluster') == "yes"

    brick_path = params.get("gluster_brick")
    if not os.path.isabs(brick_path):  # do nothing when path is absolute
        base_dir = params.get("images_base_dir", data_dir.get_data_dir())
        brick_path = os.path.join(base_dir, brick_path)

    error.context("Host name lookup failed")
    hostname = socket.gethostname()
    if not hostname or hostname == "(none)":
        if_up = utils_net.get_net_if(state="UP")
        for i in if_up:
            ipv4_value = utils_net.get_net_if_addrs(i)["ipv4"]
            logging.debug("ipv4_value is %s", ipv4_value)
            if ipv4_value != []:
                ip_addr = ipv4_value[0]
                break
        hostname = ip_addr

    # Start the gluster dameon, if not started
    glusterd_start()
    # Check for the volume is already present, if not create one.
    if not is_gluster_vol_avail(vol_name) or force:
        return gluster_vol_create(vol_name, hostname, brick_path, force)
    else:
        return True
Exemple #13
0
def create_gluster_vol(params):
    vol_name = params.get("gluster_volume_name")
    force = params.get('force_recreate_gluster') == "yes"

    brick_path = params.get("gluster_brick")
    if not os.path.isabs(brick_path):  # do nothing when path is absolute
        base_dir = params.get("images_base_dir", data_dir.get_data_dir())
        brick_path = os.path.join(base_dir, brick_path)

    error.context("Host name lookup failed")
    hostname = socket.gethostname()
    if not hostname or hostname == "(none)":
        if_up = utils_net.get_net_if(state="UP")
        for i in if_up:
            ipv4_value = utils_net.get_net_if_addrs(i)["ipv4"]
            logging.debug("ipv4_value is %s", ipv4_value)
            if ipv4_value != []:
                ip_addr = ipv4_value[0]
                break
        hostname = ip_addr

    # Start the gluster dameon, if not started
    glusterd_start()
    # Check for the volume is already present, if not create one.
    if not is_gluster_vol_avail(vol_name) or force:
        return gluster_vol_create(vol_name, hostname, brick_path, force)
    else:
        return True
Exemple #14
0
def postprocess_image(test, params, image_name):
    """
    Postprocess a single QEMU image according to the instructions in params.

    :param test: An Autotest test object.
    :param params: A dict containing image postprocessing parameters.
    """
    clone_master = params.get("clone_master", None)
    base_dir = data_dir.get_data_dir()
    image = qemu_storage.QemuImg(params, base_dir, image_name)
    if params.get("check_image") == "yes":
        try:
            if clone_master is None:
                image.check_image(params, base_dir)
            elif clone_master == "yes":
                if image_name in params.get("master_images_clone").split():
                    image.check_image(params, base_dir)
            # Allow test to overwrite any pre-testing  automatic backup
            # with a new backup. i.e. assume pre-existing image/backup
            # would not be usable after this test succeeds. The best
            # example for this is when 'unattended_install' is run.
            if params.get("backup_image", "no") == "yes":
                image.backup_image(params, base_dir, "backup", True)
            elif params.get("restore_image", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
        except Exception, e:
            if params.get("restore_image_on_check_error", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
            if params.get("remove_image_on_check_error", "no") == "yes":
                cl_images = params.get("master_images_clone", "")
                if image_name in cl_images.split():
                    image.remove()
            raise e
Exemple #15
0
def preprocess_image(test, params, image_name, vm_process_status=None):
    """
    Preprocess a single QEMU image according to the instructions in params.

    :param test: Autotest test object.
    :param params: A dict containing image preprocessing parameters.
    :param vm_process_status: This is needed in postprocess_image. Add it here
                              only for keep it work with process_images()
    :note: Currently this function just creates an image if requested.
    """
    base_dir = params.get("images_base_dir", data_dir.get_data_dir())

    if not storage.preprocess_image_backend(base_dir, params, image_name):
        logging.error("Backend can't be prepared correctly.")

    image_filename = storage.get_image_filename(params, base_dir)

    create_image = False
    if params.get("force_create_image") == "yes":
        create_image = True
    elif (params.get("create_image") == "yes"
          and not storage.file_exists(params, image_filename)):
        create_image = True

    if params.get("backup_image_before_testing", "no") == "yes":
        image = qemu_storage.QemuImg(params, base_dir, image_name)
        image.backup_image(params, base_dir, "backup", True, True)
    if create_image:
        image = qemu_storage.QemuImg(params, base_dir, image_name)
        image.create(params)
def create_config_files(options):
    """
    Check if the appropriate configuration files are present.

    If the files are not present, create them.

    :param options: OptParser object with options.
    """
    shared_dir = os.path.dirname(data_dir.get_data_dir())
    test_dir = os.path.dirname(shared_dir)

    if options.vt_type and options.vt_config:
        test_dir = data_dir.get_backend_dir(options.vt_type)
    elif options.vt_type:
        test_dir = data_dir.get_backend_dir(options.vt_type)
    elif options.vt_config:
        parent_config_dir = os.path.dirname(options.vt_config)
        parent_config_dir = os.path.dirname(parent_config_dir)
        options.vt_type = parent_config_dir
        test_dir = os.path.join(test_dir, parent_config_dir)

    if not os.path.exists(os.path.join(test_dir, "cfg")):
        print_stdout("Setup error: %s does not exist" %
                     os.path.join(test_dir, "cfg"))
        print_stdout("Perhaps you have not specified -t?")
        sys.exit(1)
    # lvsb test doesn't use shared configs
    if options.vt_type != 'lvsb':
        bootstrap.create_config_files(test_dir, shared_dir, interactive=False)
        bootstrap.create_guest_os_cfg(options.vt_type)
    bootstrap.create_subtests_cfg(options.vt_type)
Exemple #17
0
def postprocess_image(test, params, image_name):
    """
    Postprocess a single QEMU image according to the instructions in params.

    :param test: An Autotest test object.
    :param params: A dict containing image postprocessing parameters.
    """
    clone_master = params.get("clone_master", None)
    base_dir = data_dir.get_data_dir()
    image = qemu_storage.QemuImg(params, base_dir, image_name)
    if params.get("check_image") == "yes":
        try:
            if clone_master is None:
                image.check_image(params, base_dir)
            elif clone_master == "yes":
                if image_name in params.get("master_images_clone").split():
                    image.check_image(params, base_dir)
            if params.get("restore_image", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
        except Exception, e:
            if params.get("restore_image_on_check_error", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
            if params.get("remove_image_on_check_error", "no") == "yes":
                cl_images = params.get("master_images_clone", "")
                if image_name in cl_images.split():
                    image.remove()
            raise e
Exemple #18
0
def preprocess_image(test, params, image_name):
    """
    Preprocess a single QEMU image according to the instructions in params.

    @param test: Autotest test object.
    @param params: A dict containing image preprocessing parameters.
    @note: Currently this function just creates an image if requested.
    """
    base_dir = data_dir.get_data_dir()
    if params.get("storage_type") == "iscsi":
        iscsidev = kvm_storage.Iscsidev(params, base_dir, image_name)
        params["image_name"] = iscsidev.setup()
    else:
        image_filename = storage.get_image_filename(params,
                                                    base_dir)

        create_image = False

        if params.get("force_create_image") == "yes":
            create_image = True
        elif (params.get("create_image") == "yes" and not
              os.path.exists(image_filename)):
            create_image = True

        if create_image:
            image = kvm_storage.QemuImg(params, base_dir, image_name)
            if not image.create(params):
                raise error.TestError("Could not create image")
Exemple #19
0
def postprocess_image(test, params, image_name):
    """
    Postprocess a single QEMU image according to the instructions in params.

    :param test: An Autotest test object.
    :param params: A dict containing image postprocessing parameters.
    """
    clone_master = params.get("clone_master", None)
    base_dir = data_dir.get_data_dir()
    image = qemu_storage.QemuImg(params, base_dir, image_name)
    if params.get("check_image") == "yes":
        try:
            if clone_master is None:
                image.check_image(params, base_dir)
            elif clone_master == "yes":
                if image_name in params.get("master_images_clone").split():
                    image.check_image(params, base_dir)
            if params.get("restore_image", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
        except Exception, e:
            if params.get("restore_image_on_check_error", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
            if params.get("remove_image_on_check_error", "no") == "yes":
                cl_images = params.get("master_images_clone", "")
                if image_name in cl_images.split():
                    image.remove()
            raise e
Exemple #20
0
def preprocess_image(test, params, image_name):
    """
    Preprocess a single QEMU image according to the instructions in params.

    @param test: Autotest test object.
    @param params: A dict containing image preprocessing parameters.
    @note: Currently this function just creates an image if requested.
    """
    base_dir = data_dir.get_data_dir()
    if params.get("storage_type") == "iscsi":
        iscsidev = qemu_storage.Iscsidev(params, base_dir, image_name)
        params["image_name"] = iscsidev.setup()
    else:
        image_filename = storage.get_image_filename(params,
                                                    base_dir)

        create_image = False

        if params.get("force_create_image") == "yes":
            create_image = True
        elif (params.get("create_image") == "yes" and not
              os.path.exists(image_filename)):
            create_image = True

        if create_image:
            image = qemu_storage.QemuImg(params, base_dir, image_name)
            if not image.create(params):
                raise error.TestError("Could not create image")
Exemple #21
0
def postprocess_image(test, params, image_name, vm_process_status=None):
    """
    Postprocess a single QEMU image according to the instructions in params.

    :param test: An Autotest test object.
    :param params: A dict containing image postprocessing parameters.
    :param vm_process_status: (optional) vm process status like running, dead
                              or None for no vm exist.
    """
    clone_master = params.get("clone_master", None)
    base_dir = data_dir.get_data_dir()
    image = qemu_storage.QemuImg(params, base_dir, image_name)

    check_image_flag = params.get("check_image") == "yes"
    if vm_process_status == "running" and check_image_flag:
        if params.get("skip_image_check_during_running") == "yes":
            logging.debug("Guest is still running, skip the image check.")
            check_image_flag = False
        else:
            image_info_output = image.info()
            image_info = {}
            for image_info_item in image_info_output.splitlines():
                option = image_info_item.split(":")
                if len(option) == 2:
                    image_info[option[0].strip()] = option[1].strip()
            if ("lazy refcounts" in image_info
                    and image_info["lazy refcounts"] == "true"):
                logging.debug("Should not check image while guest is alive"
                              " when the image is create with lazy refcounts."
                              " Skip the image check.")
                check_image_flag = False

    if check_image_flag:
        try:
            if clone_master is None:
                image.check_image(params, base_dir)
            elif clone_master == "yes":
                if image_name in params.get("master_images_clone").split():
                    image.check_image(params, base_dir)
            # Allow test to overwrite any pre-testing  automatic backup
            # with a new backup. i.e. assume pre-existing image/backup
            # would not be usable after this test succeeds. The best
            # example for this is when 'unattended_install' is run.
            if params.get("backup_image", "no") == "yes":
                image.backup_image(params, base_dir, "backup", True)
            elif params.get("restore_image", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
        except Exception, e:
            if params.get("restore_image_on_check_error", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
            if params.get("remove_image_on_check_error", "no") == "yes":
                cl_images = params.get("master_images_clone", "")
                if image_name in cl_images.split():
                    image.remove()
            if (params.get("skip_cluster_leak_warn") == "yes"
                    and "Leaked clusters" in e.message):
                logging.warn(e.message)
            else:
                raise e
Exemple #22
0
def postprocess_image(test, params, image_name, vm_process_status=None):
    """
    Postprocess a single QEMU image according to the instructions in params.

    :param test: An Autotest test object.
    :param params: A dict containing image postprocessing parameters.
    :param vm_process_status: (optional) vm process status like running, dead
                              or None for no vm exist.
    """
    clone_master = params.get("clone_master", None)
    base_dir = data_dir.get_data_dir()
    image = qemu_storage.QemuImg(params, base_dir, image_name)

    check_image_flag = params.get("check_image") == "yes"
    if vm_process_status == "running" and check_image_flag:
        if params.get("skip_image_check_during_running") == "yes":
            logging.debug("Guest is still running, skip the image check.")
            check_image_flag = False
        else:
            image_info_output = image.info()
            image_info = {}
            for image_info_item in image_info_output.splitlines():
                option = image_info_item.split(":")
                if len(option) == 2:
                    image_info[option[0].strip()] = option[1].strip()
            if ("lazy refcounts" in image_info
                    and image_info["lazy refcounts"] == "true"):
                logging.debug("Should not check image while guest is alive"
                              " when the image is create with lazy refcounts."
                              " Skip the image check.")
                check_image_flag = False

    if check_image_flag:
        try:
            if clone_master is None:
                image.check_image(params, base_dir)
            elif clone_master == "yes":
                if image_name in params.get("master_images_clone").split():
                    image.check_image(params, base_dir)
            # Allow test to overwrite any pre-testing  automatic backup
            # with a new backup. i.e. assume pre-existing image/backup
            # would not be usable after this test succeeds. The best
            # example for this is when 'unattended_install' is run.
            if params.get("backup_image", "no") == "yes":
                image.backup_image(params, base_dir, "backup", True)
            elif params.get("restore_image", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
        except Exception, e:
            if params.get("restore_image_on_check_error", "no") == "yes":
                image.backup_image(params, base_dir, "restore", True)
            if params.get("remove_image_on_check_error", "no") == "yes":
                cl_images = params.get("master_images_clone", "")
                if image_name in cl_images.split():
                    image.remove()
            if (params.get("skip_cluster_leak_warn") == "yes"
                    and "Leaked clusters" in e.message):
                logging.warn(e.message)
            else:
                raise e
Exemple #23
0
def get_asset_info(asset):
    asset_path = os.path.join(data_dir.get_download_dir(), '%s.ini' % asset)
    asset_cfg = ConfigParser.ConfigParser()
    asset_cfg.read(asset_path)

    url = asset_cfg.get(asset, 'url')
    try:
        sha1_url = asset_cfg.get(asset, 'sha1_url')
    except ConfigParser.NoOptionError:
        sha1_url = None
    title = asset_cfg.get(asset, 'title')
    destination = asset_cfg.get(asset, 'destination')
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)
    asset_exists = os.path.isfile(destination)

    # Optional fields
    try:
        destination_uncompressed = asset_cfg.get(asset,
                                                 'destination_uncompressed')
        if not os.path.isabs(destination_uncompressed):
            destination_uncompressed = os.path.join(data_dir.get_data_dir(),
                                                    destination_uncompressed)
    except:
        destination_uncompressed = None

    try:
        uncompress_cmd = asset_cfg.get(asset, 'uncompress_cmd')
    except:
        uncompress_cmd = None

    return {
        'url': url,
        'sha1_url': sha1_url,
        'destination': destination,
        'destination_uncompressed': destination_uncompressed,
        'uncompress_cmd': uncompress_cmd,
        'shortname': asset,
        'title': title,
        'downloaded': asset_exists
    }
Exemple #24
0
 def check_disk_params(self, params):
     """
     Check gathered info from qtree/block with params
     @param params: autotest params
     @return: number of errors
     """
     err = 0
     disks = {}
     for disk in self.disks:
         if isinstance(disk, QtreeDisk):
             disks[disk.get_qname()] = disk.get_params().copy()
     # We don't have the params name so we need to map file_names instead
     qname = None
     for name in params.objects('images'):
         current = None
         image_params = params.object_params(name)
         image_name = os.path.realpath(
             storage.get_image_filename(image_params,
                                        data_dir.get_data_dir()))
         for (qname, disk) in disks.iteritems():
             if disk.get('image_name') == image_name:
                 current = disk
                 # autotest params might use relative path
                 current['image_name'] = image_params.get('image_name')
                 break
         if not current:
             logging.error("Disk %s is not in qtree but is in params.",
                           name)
             err += 1
             continue
         for prop in current.iterkeys():
             handled = False
             if prop == "drive_format":
                 # HOOK: params to qemu translation
                 if current.get(prop).startswith(image_params.get(prop)):
                     handled = True
             elif (image_params.get(prop)
                   and image_params.get(prop) == current.get(prop)):
                 handled = True
             if not handled:
                 logging.error(
                     "Disk %s property %s=%s doesn't match params"
                     " %s", qname, prop, current.get(prop),
                     image_params.get(prop))
                 err += 1
         disks.pop(qname)
     if disks:
         logging.error(
             'Some disks were in qtree but not in autotest params'
             ': %s', disks)
         err += 1
     return err
Exemple #25
0
def get_asset_info(asset):
    asset_info = {}
    asset_path = os.path.join(data_dir.get_download_dir(), '%s.ini' % asset)
    asset_cfg = test_config.config_loader(asset_path)

    asset_info['url'] = asset_cfg.get(asset, 'url')
    asset_info['sha1_url'] = asset_cfg.get(asset, 'sha1_url')
    asset_info['title'] = asset_cfg.get(asset, 'title')
    destination = asset_cfg.get(asset, 'destination')
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)
    asset_info['destination'] = destination
    asset_info['asset_exists'] = os.path.isfile(destination)

    # Optional fields
    d_uncompressed = asset_cfg.get(asset, 'destination_uncompressed')
    if d_uncompressed is not None and not os.path.isabs(d_uncompressed):
        d_uncompressed = os.path.join(data_dir.get_data_dir(), d_uncompressed)
    asset_info['destination_uncompressed'] = d_uncompressed
    asset_info['uncompress_cmd'] = asset_cfg.get(asset, 'uncompress_cmd')

    return asset_info
Exemple #26
0
def get_asset_info(asset):
    asset_info = {}
    asset_path = os.path.join(data_dir.get_download_dir(), "%s.ini" % asset)
    asset_cfg = test_config.config_loader(asset_path)

    asset_info["url"] = asset_cfg.get(asset, "url")
    asset_info["sha1_url"] = asset_cfg.get(asset, "sha1_url")
    asset_info["title"] = asset_cfg.get(asset, "title")
    destination = asset_cfg.get(asset, "destination")
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)
    asset_info["destination"] = destination
    asset_info["asset_exists"] = os.path.isfile(destination)

    # Optional fields
    d_uncompressed = asset_cfg.get(asset, "destination_uncompressed")
    if d_uncompressed is not None and not os.path.isabs(d_uncompressed):
        d_uncompressed = os.path.join(data_dir.get_data_dir(), d_uncompressed)
    asset_info["destination_uncompressed"] = d_uncompressed
    asset_info["uncompress_cmd"] = asset_cfg.get(asset, "uncompress_cmd")

    return asset_info
Exemple #27
0
def get_asset_info(asset):
    asset_info = {}
    asset_path = os.path.join(data_dir.get_download_dir(), '%s.ini' % asset)
    asset_cfg = test_config.config_loader(asset_path)

    asset_info['url'] = asset_cfg.get(asset, 'url')
    asset_info['sha1_url'] = asset_cfg.get(asset, 'sha1_url')
    asset_info['title'] = asset_cfg.get(asset, 'title')
    destination = asset_cfg.get(asset, 'destination')
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)
    asset_info['destination'] = destination
    asset_info['asset_exists'] = os.path.isfile(destination)

    # Optional fields
    d_uncompressed = asset_cfg.get(asset, 'destination_uncompressed')
    if d_uncompressed is not None and not os.path.isabs(d_uncompressed):
        d_uncompressed = os.path.join(data_dir.get_data_dir(),
                                      d_uncompressed)
    asset_info['destination_uncompressed'] = d_uncompressed
    asset_info['uncompress_cmd'] = asset_cfg.get(asset, 'uncompress_cmd')

    return asset_info
Exemple #28
0
 def check_disk_params(self, params):
     """
     Check gathered info from qtree/block with params
     @param params: autotest params
     @return: number of errors
     """
     err = 0
     disks = {}
     for disk in self.disks:
         if isinstance(disk, QtreeDisk):
             disks[disk.get_qname()] = disk.get_params().copy()
     # We don't have the params name so we need to map file_names instead
     qname = None
     for name in params.objects('images'):
         current = None
         image_params = params.object_params(name)
         image_name = os.path.realpath(
                     storage.get_image_filename(image_params,
                                                data_dir.get_data_dir()))
         for (qname, disk) in disks.iteritems():
             if disk.get('image_name') == image_name:
                 current = disk
                 # autotest params might use relative path
                 current['image_name'] = image_params.get('image_name')
                 break
         if not current:
             logging.error("Disk %s is not in qtree but is in params.", name)
             err += 1
             continue
         for prop in current.iterkeys():
             handled = False
             if prop == "drive_format":
                 # HOOK: params to qemu translation
                 if current.get(prop).startswith(image_params.get(prop)):
                     handled = True
             elif (image_params.get(prop) and
                     image_params.get(prop) == current.get(prop)):
                 handled = True
             if not handled:
                 logging.error("Disk %s property %s=%s doesn't match params"
                               " %s", qname, prop, current.get(prop),
                               image_params.get(prop))
                 err += 1
         disks.pop(qname)
     if disks:
         logging.error('Some disks were in qtree but not in autotest params'
                       ': %s', disks)
         err += 1
     return err
Exemple #29
0
def get_asset_info(asset):
    asset_path = os.path.join(data_dir.get_download_dir(), "%s.ini" % asset)
    asset_cfg = ConfigParser.ConfigParser()
    asset_cfg.read(asset_path)

    url = asset_cfg.get(asset, "url")
    try:
        sha1_url = asset_cfg.get(asset, "sha1_url")
    except ConfigParser.NoOptionError:
        sha1_url = None
    title = asset_cfg.get(asset, "title")
    destination = asset_cfg.get(asset, "destination")
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)
    asset_exists = os.path.isfile(destination)

    # Optional fields
    try:
        destination_uncompressed = asset_cfg.get(asset, "destination_uncompressed")
        if not os.path.isabs(destination_uncompressed):
            destination_uncompressed = os.path.join(data_dir.get_data_dir(), destination)
        uncompress_cmd = asset_cfg.get(asset, "uncompress_cmd")
    except:
        destination_uncompressed = None
        uncompress_cmd = None

    return {
        "url": url,
        "sha1_url": sha1_url,
        "destination": destination,
        "destination_uncompressed": destination_uncompressed,
        "uncompress_cmd": uncompress_cmd,
        "shortname": asset,
        "title": title,
        "downloaded": asset_exists,
    }
Exemple #30
0
def download_asset(asset, interactive=True, restore_image=False):
    """
    Download an asset defined on an asset file.

    Asset files are located under /shared/download.d, are .ini files with the
    following keys defined:
        title: Title string to display in the download progress bar.
        url = URL of the resource
        sha1_url = URL with SHA1 information for the resource, in the form
            sha1sum file_basename
        destination = Location of your file relative to the data directory
            (TEST_SUITE_ROOT/shared/data)
        destination = Location of the uncompressed file relative to the data
            directory (TEST_SUITE_ROOT/shared/data)
        uncompress_cmd = Command that needs to be executed with the compressed
            file as a parameter

    @param asset: String describing an asset file.
    @param interactive: Whether to ask the user before downloading the file.
    @param restore_image: If the asset is a compressed image, we can uncompress
                          in order to restore the image.
    """
    asset_info = get_asset_info(asset)
    destination = os.path.join(data_dir.get_data_dir(),
                               asset_info['destination'])

    if (interactive and not os.path.isfile(destination)):
        answer = utils.ask("File %s not present. Do you want to download it?" %
                           asset_info['title'])
    else:
        answer = "y"

    if answer == "y":
        had_to_download = download_file(asset=asset, interactive=interactive)

        requires_uncompress = asset_info['uncompress_cmd'] is not None
        if requires_uncompress:
            destination_uncompressed = asset_info['destination_uncompressed']
            uncompressed_file_exists = os.path.exists(destination_uncompressed)

            restore_image = (restore_image or had_to_download or not
                             uncompressed_file_exists)

            if os.path.isfile(destination) and restore_image:
                os.chdir(os.path.dirname(destination))
                uncompress_cmd = asset_info['uncompress_cmd']
                utils.run("%s %s" % (uncompress_cmd, destination))
Exemple #31
0
def get_file_asset(title, src_path, destination):
    if not os.path.isabs(destination):
        destination = os.path.join(data_dir.get_data_dir(), destination)

    for ext in (".xz", ".gz", ".7z", ".bz2"):
        if os.path.exists(src_path + ext):
            destination = destination + ext
            logging.debug('Found source image %s', destination)
            return {'url': None, 'sha1_url': None, 'destination': src_path + ext,
                    'destination_uncompressed': destination,
                    'uncompress_cmd': None, 'shortname': title, 'title': title,
                    'downloaded': True}

    if os.path.exists(src_path):
        logging.debug('Found source image %s', destination)
        return {'url': src_path, 'sha1_url': None, 'destination': destination,
                'destination_uncompressed': None, 'uncompress_cmd': None,
                'shortname': title, 'title': title,
                'downloaded': os.path.exists(destination)}

    return None
Exemple #32
0
    def _parse_params(self):
        '''
        Parses the params items for entries related to guest kernel
        '''
        configure_opt_key = '%s_config' % self.prefix
        self.config = self.params.get(configure_opt_key, '')

        build_image_key = '%s_build_image' % self.prefix
        self.build_image = self.params.get(build_image_key,
                                           'arch/x86/boot/bzImage')

        build_target_key = '%s_build_target' % self.prefix
        self.build_target = self.params.get(build_target_key, 'bzImage')

        kernel_path_key = '%s_kernel_path' % self.prefix
        image_dir = os.path.join(data_dir.get_data_dir(), 'images')
        default_kernel_path = os.path.join(image_dir, self.build_target)
        self.kernel_path = self.params.get(kernel_path_key,
                                           default_kernel_path)

        logging.info('Parsing Linux kernel build parameters for %s',
                     self.prefix)
def create_config_files(options):
    """
    Check if the appropriate configuration files are present.

    If the files are not present, create them.

    @param options: OptParser object with options.
    """
    shared_dir = os.path.dirname(data_dir.get_data_dir())
    test_dir = os.path.dirname(shared_dir)
    shared_dir = os.path.join(shared_dir, "cfg")

    if (options.type and options.config):
        test_dir = os.path.join(test_dir, options.type)
    elif options.type:
        test_dir = os.path.join(test_dir, options.type)
    elif options.config:
        parent_config_dir = os.path.dirname(options.config)
        parent_config_dir = os.path.basename(parent_config_dir)
        test_dir = os.path.join(test_dir, parent_config_dir)

    bootstrap.create_config_files(test_dir, shared_dir, interactive=False)
def create_config_files(options):
    """
    Check if the appropriate configuration files are present.

    If the files are not present, create them.

    @param options: OptParser object with options.
    """
    shared_dir = os.path.dirname(data_dir.get_data_dir())
    test_dir = os.path.dirname(shared_dir)
    shared_dir = os.path.join(shared_dir, "cfg")

    if (options.type and options.config):
        test_dir = os.path.join(test_dir, options.type)
    elif options.type:
        test_dir = os.path.join(test_dir, options.type)
    elif options.config:
        parent_config_dir = os.path.dirname(options.config)
        parent_config_dir = os.path.basename(parent_config_dir)
        test_dir = os.path.join(test_dir, parent_config_dir)

    utils_misc.create_config_files(test_dir, shared_dir, interactive=False)
Exemple #35
0
    def _parse_params(self):
        '''
        Parses the params items for entries related to guest kernel
        '''
        configure_opt_key = '%s_config' % self.prefix
        self.config = self.params.get(configure_opt_key, '')

        build_image_key = '%s_build_image' % self.prefix
        self.build_image = self.params.get(build_image_key,
                                           'arch/x86/boot/bzImage')

        build_target_key = '%s_build_target' % self.prefix
        self.build_target = self.params.get(build_target_key, 'bzImage')

        kernel_path_key = '%s_kernel_path' % self.prefix
        image_dir = os.path.join(data_dir.get_data_dir(), 'images')
        default_kernel_path = os.path.join(image_dir,
                                           self.build_target)
        self.kernel_path = self.params.get(kernel_path_key,
                                           default_kernel_path)

        logging.info('Parsing Linux kernel build parameters for %s',
                     self.prefix)
Exemple #36
0
def create_config_files(options):
    """
    Check if the appropriate configuration files are present.

    If the files are not present, create them.

    @param options: OptParser object with options.
    """
    shared_dir = os.path.dirname(data_dir.get_data_dir())
    test_dir = os.path.dirname(shared_dir)

    if (options.type and options.config):
        test_dir = os.path.join(test_dir, options.type)
    elif options.type:
        test_dir = os.path.join(test_dir, options.type)
    elif options.config:
        parent_config_dir = os.path.dirname(options.config)
        parent_config_dir = os.path.dirname(parent_config_dir)
        options.type = parent_config_dir
        test_dir = os.path.join(test_dir, parent_config_dir)

    bootstrap.create_config_files(test_dir, shared_dir, interactive=False)
    bootstrap.create_subtests_cfg(options.type)
    bootstrap.create_guest_os_cfg(options.type)
Exemple #37
0
def preprocess(test, params, env):
    """
    Preprocess all VMs and images according to the instructions in params.
    Also, collect some host information, such as the KVM version.

    :param test: An Autotest test object.
    :param params: A dict containing all VM and image parameters.
    :param env: The environment (a dict-like object).
    """
    error.context("preprocessing")
    # First, let's verify if this test does require root or not. If it
    # does and the test suite is running as a regular user, we shall just
    # throw a TestNAError exception, which will skip the test.
    if params.get('requires_root', 'no') == 'yes':
        utils_misc.verify_running_as_root()

    port = params.get('shell_port')
    prompt = params.get('shell_prompt')
    address = params.get('ovirt_node_address')
    username = params.get('ovirt_node_user')
    password = params.get('ovirt_node_password')

    setup_pb = False
    for nic in params.get('nics', "").split():
        nic_params = params.object_params(nic)
        if nic_params.get('netdst') == 'private':
            setup_pb = True
            params_pb = nic_params
            params['netdst_%s' % nic] = nic_params.get("priv_brname", 'atbr0')

    if setup_pb:
        brcfg = test_setup.PrivateBridgeConfig(params_pb)
        brcfg.setup()

    base_dir = data_dir.get_data_dir()
    if params.get("storage_type") == "iscsi":
        iscsidev = qemu_storage.Iscsidev(params, base_dir, "iscsi")
        params["image_name"] = iscsidev.setup()
        params["image_raw_device"] = "yes"

    if params.get("storage_type") == "lvm":
        lvmdev = qemu_storage.LVMdev(params, base_dir, "lvm")
        params["image_name"] = lvmdev.setup()
        params["image_raw_device"] = "yes"
        env.register_lvmdev("lvm_%s" % params["main_vm"], lvmdev)

    if params.get("storage_type") == "nfs":
        image_nfs = nfs.Nfs(params)
        image_nfs.setup()
        image_name_only = os.path.basename(params["image_name"])
        params['image_name'] = os.path.join(image_nfs.mount_dir,
                                            image_name_only)
        for image_name in params.objects("images"):
            name_tag = "image_name_%s" % image_name
            if params.get(name_tag):
                image_name_only = os.path.basename(params[name_tag])
                params[name_tag] = os.path.join(image_nfs.mount_dir,
                                                image_name_only)

    # Start tcpdump if it isn't already running
    # The fact it has to be started here is so that the test params
    # have to be honored.
    env.start_tcpdump(params)

    # Destroy and remove VMs that are no longer needed in the environment
    requested_vms = params.objects("vms")
    for key in env.keys():
        vm = env[key]
        if not isinstance(vm, virt_vm.BaseVM):
            continue
        if vm.name not in requested_vms:
            vm.destroy()
            del env[key]

    if (params.get("auto_cpu_model") == "yes"
            and params.get("vm_type") == "qemu"):
        if not env.get("cpu_model"):
            env["cpu_model"] = utils_misc.get_qemu_best_cpu_model(params)
        params["cpu_model"] = env.get("cpu_model")

    kvm_ver_cmd = params.get("kvm_ver_cmd", "")

    if kvm_ver_cmd:
        try:
            cmd_result = utils.run(kvm_ver_cmd)
            kvm_version = cmd_result.stdout.strip()
        except error.CmdError:
            kvm_version = "Unknown"
    else:
        # Get the KVM kernel module version and write it as a keyval
        if os.path.exists("/dev/kvm"):
            try:
                kvm_version = open("/sys/module/kvm/version").read().strip()
            except Exception:
                kvm_version = os.uname()[2]
        else:
            logging.warning("KVM module not loaded")
            kvm_version = "Unknown"

    logging.debug("KVM version: %s" % kvm_version)
    test.write_test_keyval({"kvm_version": kvm_version})

    # Get the KVM userspace version and write it as a keyval
    kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")

    if kvm_userspace_ver_cmd:
        try:
            cmd_result = utils.run(kvm_userspace_ver_cmd)
            kvm_userspace_version = cmd_result.stdout.strip()
        except error.CmdError:
            kvm_userspace_version = "Unknown"
    else:
        qemu_path = utils_misc.get_qemu_binary(params)
        version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
        matches = re.findall("[Vv]ersion .*?,", version_line)
        if matches:
            kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
        else:
            kvm_userspace_version = "Unknown"

    logging.debug("KVM userspace version: %s" % kvm_userspace_version)
    test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})

    if params.get("setup_hugepages") == "yes":
        h = test_setup.HugePageConfig(params)
        suggest_mem = h.setup()
        if suggest_mem is not None:
            params['mem'] = suggest_mem
        if params.get("vm_type") == "libvirt":
            utils_libvirtd.libvirtd_restart()

    if params.get("setup_thp") == "yes":
        thp = test_setup.TransparentHugePageConfig(test, params)
        thp.setup()

    if params.get("setup_ksm") == "yes":
        ksm = test_setup.KSMConfig(params, env)
        ksm.setup(env)

    # Execute any pre_commands
    if params.get("pre_command"):
        process_command(test, params, env, params.get("pre_command"),
                        int(params.get("pre_command_timeout", "600")),
                        params.get("pre_command_noncritical") == "yes")

    # if you want set "pci=nomsi" before test, set "disable_pci_msi = yes"
    # and pci_msi_sensitive = "yes"
    if params.get("pci_msi_sensitive", "no") == "yes":
        disable_pci_msi = params.get("disable_pci_msi", "no")
        image_filename = storage.get_image_filename(params,
                                                    data_dir.get_data_dir())
        grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
        kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
                                        r".*vmlinuz-\d+.*")
        msi_keyword = params.get("msi_keyword", " pci=nomsi")

        disk_obj = utils_disk.GuestFSModiDisk(image_filename)
        kernel_config_ori = disk_obj.read_file(grub_file)
        kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
        if not kernel_config:
            raise error.TestError("Cannot find the kernel config, reg is %s" %
                                  kernel_cfg_pos_reg)
        kernel_config_line = kernel_config[0]

        kernel_need_modify = False
        if disable_pci_msi == "yes":
            if not re.findall(msi_keyword, kernel_config_line):
                kernel_config_set = kernel_config_line + msi_keyword
                kernel_need_modify = True
        else:
            if re.findall(msi_keyword, kernel_config_line):
                kernel_config_set = re.sub(msi_keyword, "", kernel_config_line)
                kernel_need_modify = True

        if kernel_need_modify:
            for vm in env.get_all_vms():
                if vm:
                    vm.destroy()
                    env.unregister_vm(vm.name)
            disk_obj.replace_image_file_content(grub_file, kernel_config_line,
                                                kernel_config_set)
        logging.debug("Guest cmdline 'pci=nomsi' setting is: [ %s ]" %
                      disable_pci_msi)

    kernel_extra_params = params.get("kernel_extra_params")
    if kernel_extra_params:
        image_filename = storage.get_image_filename(params,
                                                    data_dir.get_data_dir())
        grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
        kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
                                        r".*vmlinuz-\d+.*")

        disk_obj = utils_disk.GuestFSModiDisk(image_filename)
        kernel_config_ori = disk_obj.read_file(grub_file)
        kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
        if not kernel_config:
            raise error.TestError("Cannot find the kernel config, reg is %s" %
                                  kernel_cfg_pos_reg)
        kernel_config_line = kernel_config[0]

        kernel_need_modify = False
        if not re.findall(kernel_extra_params, kernel_config_line):
            kernel_config_set = kernel_config_line + kernel_extra_params
            kernel_need_modify = True

        if kernel_need_modify:
            for vm in env.get_all_vms():
                if vm:
                    vm.destroy()
                    env.unregister_vm(vm.name)
            disk_obj.replace_image_file_content(grub_file, kernel_config_line,
                                                kernel_config_set)
        logging.debug("Guest cmdline extra_params setting is: [ %s ]" %
                      kernel_extra_params)

    # Clone master image from vms.
    base_dir = data_dir.get_data_dir()
    if params.get("master_images_clone"):
        for vm_name in params.get("vms").split():
            vm = env.get_vm(vm_name)
            if vm:
                vm.destroy()
                env.unregister_vm(vm_name)

            vm_params = params.object_params(vm_name)
            for image in vm_params.get("master_images_clone").split():
                image_obj = qemu_storage.QemuImg(params, base_dir, image)
                image_obj.clone_image(params, vm_name, image, base_dir)

    # Preprocess all VMs and images
    if params.get("not_preprocess", "no") == "no":
        process(test, params, env, preprocess_image, preprocess_vm)

    # Start the screendump thread
    if params.get("take_regular_screendumps") == "yes":
        global _screendump_thread, _screendump_thread_termination_event
        _screendump_thread_termination_event = threading.Event()
        _screendump_thread = threading.Thread(target=_take_screendumps,
                                              name='ScreenDump',
                                              args=(test, params, env))
        _screendump_thread.start()

    # Start the register query thread
    if params.get("store_vm_register") == "yes":
        global _vm_register_thread, _vm_register_thread_termination_event
        _vm_register_thread_termination_event = threading.Event()
        _vm_register_thread = threading.Thread(target=_store_vm_register,
                                               name='VmRegister',
                                               args=(test, params, env))
        _vm_register_thread.start()

    return params
Exemple #38
0
    def create(self, name=None, params=None, root_dir=None, timeout=5.0,
               migration_mode=None, mac_source=None):
        """
        Start the VM by running a qemu command.
        All parameters are optional. If name, params or root_dir are not
        supplied, the respective values stored as class attributes are used.

        @param name: The name of the object
        @param params: A dict containing VM params
        @param root_dir: Base directory for relative filenames
        @param migration_mode: If supplied, start VM for incoming migration
                using this protocol (either 'tcp', 'unix' or 'exec')
        @param migration_exec_cmd: Command to embed in '-incoming "exec: ..."'
                (e.g. 'gzip -c -d filename') if migration_mode is 'exec'
        @param mac_source: A VM object from which to copy MAC addresses. If not
                specified, new addresses will be generated.

        @raise VMCreateError: If qemu terminates unexpectedly
        @raise VMKVMInitError: If KVM initialization fails
        @raise VMHugePageError: If hugepage initialization fails
        @raise VMImageMissingError: If a CD image is missing
        @raise VMHashMismatchError: If a CD image hash has doesn't match the
                expected hash
        @raise VMBadPATypeError: If an unsupported PCI assignment type is
                requested
        @raise VMPAError: If no PCI assignable devices could be assigned
        """
        error.context("creating '%s'" % self.name)
        self.destroy(free_mac_addresses=False)

        if name is not None:
            self.name = name
        if params is not None:
            self.params = params
        if root_dir is not None:
            self.root_dir = root_dir
        name = self.name
        params = self.params
        root_dir = self.root_dir

        # Verify the md5sum of the ISO images
        for cdrom in params.objects("cdroms"):
            if params.get("medium") == "import":
                break
            cdrom_params = params.object_params(cdrom)
            iso = cdrom_params.get("cdrom")
            if ((self.driver_type == 'xen') and
                (params.get('hvm_or_pv') == 'pv') and
                (os.path.basename(iso) == 'ks.iso')):
                continue
            if iso:
                iso = utils_misc.get_path(data_dir.get_data_dir(), iso)
                if not os.path.exists(iso):
                    raise virt_vm.VMImageMissingError(iso)
                compare = False
                if cdrom_params.get("md5sum_1m"):
                    logging.debug("Comparing expected MD5 sum with MD5 sum of "
                                  "first MB of ISO file...")
                    actual_hash = utils.hash_file(iso, 1048576, method="md5")
                    expected_hash = cdrom_params.get("md5sum_1m")
                    compare = True
                elif cdrom_params.get("md5sum"):
                    logging.debug("Comparing expected MD5 sum with MD5 sum of "
                                  "ISO file...")
                    actual_hash = utils.hash_file(iso, method="md5")
                    expected_hash = cdrom_params.get("md5sum")
                    compare = True
                elif cdrom_params.get("sha1sum"):
                    logging.debug("Comparing expected SHA1 sum with SHA1 sum "
                                  "of ISO file...")
                    actual_hash = utils.hash_file(iso, method="sha1")
                    expected_hash = cdrom_params.get("sha1sum")
                    compare = True
                if compare:
                    if actual_hash == expected_hash:
                        logging.debug("Hashes match")
                    else:
                        raise virt_vm.VMHashMismatchError(actual_hash,
                                                          expected_hash)

        # Make sure the following code is not executed by more than one thread
        # at the same time
        lockfile = open("/tmp/libvirt-autotest-vm-create.lock", "w+")
        fcntl.lockf(lockfile, fcntl.LOCK_EX)

        try:
            # Handle port redirections
            redir_names = params.objects("redirs")
            host_ports = utils_misc.find_free_ports(5000, 6000, len(redir_names))
            self.redirs = {}
            for i in range(len(redir_names)):
                redir_params = params.object_params(redir_names[i])
                guest_port = int(redir_params.get("guest_port"))
                self.redirs[guest_port] = host_ports[i]

            # Find available PCI devices
            self.pci_devices = []
            for device in params.objects("pci_devices"):
                self.pci_devices.append(device)

            # Find available VNC port, if needed
            if params.get("display") == "vnc":
                if params.get("vnc_autoport") == "yes":
                    self.vnc_port = None
                    self.vnc_autoport = True
                else:
                    self.vnc_port = utils_misc.find_free_port(5900, 6100)
                    self.vnc_autoport = False

            # Find available spice port, if needed
            if params.get("spice"):
                self.spice_port = utils_misc.find_free_port(8000, 8100)

            # Find random UUID if specified 'uuid = random' in config file
            if params.get("uuid") == "random":
                f = open("/proc/sys/kernel/random/uuid")
                self.uuid = f.read().strip()
                f.close()

            # Generate or copy MAC addresses for all NICs
            for nic in self.virtnet:
                nic_params = dict(nic)
                if mac_source:
                    # Will raise exception if source doesn't
                    # have cooresponding nic
                    logging.debug("Copying mac for nic %s from VM %s"
                                    % (nic.nic_name, mac_source.nam))
                    nic_params['mac'] = mac_source.get_mac_address(nic.nic_name)
                # make_create_command() calls vm.add_nic (i.e. on a copy)
                nic = self.add_nic(**nic_params)
                logging.debug('VM.create activating nic %s' % nic)
                self.activate_nic(nic.nic_name)

            # Make qemu command
            install_command = self.make_create_command()

            logging.info("Running libvirt command (reformatted):")
            for item in install_command.replace(" -", " \n    -").splitlines():
                logging.info("%s", item)
            utils.run(install_command, verbose=False)
            # Wait for the domain to be created
            utils_misc.wait_for(func=self.is_alive, timeout=60,
                                text=("waiting for domain %s to start" %
                                      self.name))
            self.uuid = virsh.domuuid(self.name, uri=self.connect_uri)

            # Establish a session with the serial console
            if self.only_pty == True:
                self.serial_console = aexpect.ShellSession(
                    "virsh console %s" % self.name,
                    auto_close=False,
                    output_func=utils_misc.log_line,
                    output_params=("serial-%s.log" % name,))
            else:
                self.serial_console = aexpect.ShellSession(
                    "tail -f %s" % self.get_serial_console_filename(),
                    auto_close=False,
                    output_func=utils_misc.log_line,
                    output_params=("serial-%s.log" % name,))

        finally:
            fcntl.lockf(lockfile, fcntl.LOCK_UN)
            lockfile.close()
Exemple #39
0
def run_tests(parser, options):
    """
    Runs the sequence of KVM tests based on the list of dctionaries
    generated by the configuration system, handling dependencies.

    @param parser: Config parser object.
    @return: True, if all tests ran passed, False if any of them failed.
    """
    debugdir = os.path.join(data_dir.get_root_dir(), 'logs',
                            'run-%s' % time.strftime('%Y-%m-%d-%H.%M.%S'))
    if not os.path.isdir(debugdir):
        os.makedirs(debugdir)
    debuglog = os.path.join(debugdir, "debug.log")
    configure_file_logging(debuglog)

    print_stdout(bcolors.HEADER +
                 "DATA DIR: %s" % data_dir.get_backing_data_dir() +
                 bcolors.ENDC)

    print_header("DEBUG LOG: %s" % debuglog)

    last_index = -1

    logging.info("Starting test job at %s" % time.strftime('%Y-%m-%d %H:%M:%S'))
    logging.info("")
    logging.debug("Options received from the command line:")
    utils_misc.display_attributes(options)
    logging.debug("")

    logging.debug("Cleaning up previous job tmp files")
    d = parser.get_dicts().next()
    env_filename = os.path.join(data_dir.get_root_dir(),
                                options.type, d.get("env", "env"))
    env = utils_env.Env(env_filename, Test.env_version)
    env.destroy()
    try:
        address_pool_files = glob.glob("/tmp/address_pool*")
        for address_pool_file in address_pool_files:
            os.remove(address_pool_file)
        aexpect_tmp = "/tmp/aexpect_spawn/"
        if os.path.isdir(aexpect_tmp):
            shutil.rmtree("/tmp/aexpect_spawn/")
    except (IOError, OSError):
        pass
    logging.debug("")

    if options.restore_image_between_tests:
        logging.debug("Creating first backup of guest image")
        qemu_img = storage.QemuImg(d, data_dir.get_data_dir(), "image")
        qemu_img.backup_image(d, data_dir.get_data_dir(), 'backup', True)
        logging.debug("")

    if options.type == 'qemu':
        logging.info("We're running the qemu test with:")
        logging.info("qemu binary: %s" % d.get('qemu_binary'))
        logging.info("qemu img binary: %s" % d.get('qemu_img_binary'))
        logging.info("qemu io binary: %s" % d.get('qemu_io_binary'))
        logging.info("")

    logging.info("Defined test set:")
    for i, d in enumerate(parser.get_dicts()):
        if options.config is None and options.type in TEST_TYPES_STRIP_NAMES:
            shortname = ".".join(d['name'].split(".")[12:])
        else:
            shortname = ".".join(d['shortname'].split("."))

        logging.info("Test %4d:  %s" % (i + 1, shortname))
        last_index += 1

    if last_index == -1:
        print_stdout("No tests generated by config file %s" % parser.filename)
        print_stdout("Please check the file for errors (bad variable names, "
                     "wrong indentation)")
        sys.exit(-1)
    logging.info("")

    n_tests = last_index + 1
    print_header("TESTS: %s" % n_tests)

    status_dct = {}
    failed = False
    # Add the parameter decide if setup host env in the test case
    # For some special tests we only setup host in the first and last case
    # When we need to setup host env we need the host_setup_flag as following:
    #    0(00): do nothing
    #    1(01): setup env
    #    2(10): cleanup env
    #    3(11): setup and cleanup env
    index = 0
    setup_flag = 1
    cleanup_flag = 2
    for dct in parser.get_dicts():
        if options.config is None and options.type in TEST_TYPES_STRIP_NAMES:
            shortname = ".".join(d['name'].split(".")[12:])
        else:
            shortname = ".".join(d['shortname'].split("."))

        if index == 0:
            if dct.get("host_setup_flag", None) is not None:
                flag = int(dct["host_setup_flag"])
                dct["host_setup_flag"] = flag | setup_flag
            else:
                dct["host_setup_flag"] = setup_flag
        if index == last_index:
            if dct.get("host_setup_flag", None) is not None:
                flag = int(dct["host_setup_flag"])
                dct["host_setup_flag"] = flag | cleanup_flag
            else:
                dct["host_setup_flag"] = cleanup_flag
        index += 1

        # Add kvm module status
        dct["kvm_default"] = utils_misc.get_module_params(
                                             dct.get("sysfs_dir", "sys"), "kvm")

        if dct.get("skip") == "yes":
            continue

        dependencies_satisfied = True
        for dep in dct.get("dep"):
            for test_name in status_dct.keys():
                if not dep in test_name:
                    continue

                if not status_dct[test_name]:
                    dependencies_satisfied = False
                    break

        current_status = False
        if dependencies_satisfied:
            t = Test(dct, options)
            t.set_debugdir(debugdir)

            pretty_index = "(%d/%d)" % (index, n_tests)
            print_stdout("%s %s:" % (pretty_index, t.tag), end=False)

            try:
                try:
                    t_begin = time.time()
                    t.start_file_logging()
                    current_status = t.run_once()
                    logging.info("PASS %s" % t.tag)
                    logging.info("")
                    t.stop_file_logging()
                finally:
                    t_end = time.time()
                    t_elapsed = t_end - t_begin
            except error.TestNAError, reason:
                logging.info("SKIP %s -> %s: %s", t.tag,
                             reason.__class__.__name__, reason)
                logging.info("")
                t.stop_file_logging()
                print_skip()
                status_dct[dct.get("name")] = False
                continue
            except error.TestWarn, reason:
                logging.info("WARN %s -> %s: %s", t.tag,
                             reason.__class__.__name__,
                             reason)
                logging.info("")
                t.stop_file_logging()
                print_warn(t_elapsed)
                status_dct[dct.get("name")] = True
                continue
            except Exception, reason:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.error("")
                tb_info = traceback.format_exception(exc_type, exc_value,
                                                     exc_traceback.tb_next)
                tb_info = "".join(tb_info)
                for e_line in tb_info.splitlines():
                    logging.error(e_line)
                logging.error("")
                logging.error("FAIL %s -> %s: %s", t.tag,
                              reason.__class__.__name__,
                              reason)
                logging.info("")
                t.stop_file_logging()
                current_status = False
Exemple #40
0
def preprocess(test, params, env):
    """
    Preprocess all VMs and images according to the instructions in params.
    Also, collect some host information, such as the KVM version.

    :param test: An Autotest test object.
    :param params: A dict containing all VM and image parameters.
    :param env: The environment (a dict-like object).
    """
    error.context("preprocessing")
    # First, let's verify if this test does require root or not. If it
    # does and the test suite is running as a regular user, we shall just
    # throw a TestNAError exception, which will skip the test.
    if params.get('requires_root', 'no') == 'yes':
        utils_misc.verify_running_as_root()

    port = params.get('shell_port')
    prompt = params.get('shell_prompt')
    address = params.get('ovirt_node_address')
    username = params.get('ovirt_node_user')
    password = params.get('ovirt_node_password')

    setup_pb = False
    for nic in params.get('nics', "").split():
        nic_params = params.object_params(nic)
        if nic_params.get('netdst') == 'private':
            setup_pb = True
            params_pb = nic_params
            params['netdst_%s' % nic] = nic_params.get("priv_brname", 'atbr0')

    if setup_pb:
        brcfg = test_setup.PrivateBridgeConfig(params_pb)
        brcfg.setup()

    base_dir = data_dir.get_data_dir()
    if params.get("storage_type") == "iscsi":
        iscsidev = qemu_storage.Iscsidev(params, base_dir, "iscsi")
        params["image_name"] = iscsidev.setup()
        params["image_raw_device"] = "yes"

    if params.get("storage_type") == "lvm":
        lvmdev = qemu_storage.LVMdev(params, base_dir, "lvm")
        params["image_name"] = lvmdev.setup()
        params["image_raw_device"] = "yes"
        env.register_lvmdev("lvm_%s" % params["main_vm"], lvmdev)

    if params.get("storage_type") == "nfs":
        image_nfs = nfs.Nfs(params)
        image_nfs.setup()
        image_name_only = os.path.basename(params["image_name"])
        params['image_name'] = os.path.join(image_nfs.mount_dir,
                                            image_name_only)
        for image_name in params.objects("images"):
            name_tag = "image_name_%s" % image_name
            if params.get(name_tag):
                image_name_only = os.path.basename(params[name_tag])
                params[name_tag] = os.path.join(image_nfs.mount_dir,
                                                image_name_only)

    # Start tcpdump if it isn't already running
    # The fact it has to be started here is so that the test params
    # have to be honored.
    env.start_tcpdump(params)

    # Destroy and remove VMs that are no longer needed in the environment
    requested_vms = params.objects("vms")
    for key in env.keys():
        vm = env[key]
        if not isinstance(vm, virt_vm.BaseVM):
            continue
        if not vm.name in requested_vms:
            vm.destroy()
            del env[key]

    if (params.get("auto_cpu_model") == "yes" and
            params.get("vm_type") == "qemu"):
        if not env.get("cpu_model"):
            env["cpu_model"] = utils_misc.get_qemu_best_cpu_model(params)
        params["cpu_model"] = env.get("cpu_model")

    kvm_ver_cmd = params.get("kvm_ver_cmd", "")

    if kvm_ver_cmd:
        try:
            cmd_result = utils.run(kvm_ver_cmd)
            kvm_version = cmd_result.stdout.strip()
        except error.CmdError:
            kvm_version = "Unknown"
    else:
        # Get the KVM kernel module version and write it as a keyval
        if os.path.exists("/dev/kvm"):
            try:
                kvm_version = open("/sys/module/kvm/version").read().strip()
            except Exception:
                kvm_version = os.uname()[2]
        else:
            logging.warning("KVM module not loaded")
            kvm_version = "Unknown"

    logging.debug("KVM version: %s" % kvm_version)
    test.write_test_keyval({"kvm_version": kvm_version})

    # Get the KVM userspace version and write it as a keyval
    kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")

    if kvm_userspace_ver_cmd:
        try:
            cmd_result = utils.run(kvm_userspace_ver_cmd)
            kvm_userspace_version = cmd_result.stdout.strip()
        except error.CmdError:
            kvm_userspace_version = "Unknown"
    else:
        qemu_path = utils_misc.get_qemu_binary(params)
        version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
        matches = re.findall("[Vv]ersion .*?,", version_line)
        if matches:
            kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
        else:
            kvm_userspace_version = "Unknown"

    logging.debug("KVM userspace version: %s" % kvm_userspace_version)
    test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})

    if params.get("setup_hugepages") == "yes":
        h = test_setup.HugePageConfig(params)
        suggest_mem = h.setup()
        if suggest_mem is not None:
            params['mem'] = suggest_mem
        if params.get("vm_type") == "libvirt":
            utils_libvirtd.libvirtd_restart()

    if params.get("setup_thp") == "yes":
        thp = test_setup.TransparentHugePageConfig(test, params)
        thp.setup()

    if params.get("setup_ksm") == "yes":
        ksm = test_setup.KSMConfig(params, env)
        ksm.setup(env)

    # Execute any pre_commands
    if params.get("pre_command"):
        process_command(test, params, env, params.get("pre_command"),
                        int(params.get("pre_command_timeout", "600")),
                        params.get("pre_command_noncritical") == "yes")

    # if you want set "pci=nomsi" before test, set "disable_pci_msi = yes"
    # and pci_msi_sensitive = "yes"
    if params.get("pci_msi_sensitive", "no") == "yes":
        disable_pci_msi = params.get("disable_pci_msi", "no")
        image_filename = storage.get_image_filename(params,
                                                    data_dir.get_data_dir())
        grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
        kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
                                        r".*vmlinuz-\d+.*")
        msi_keyword = params.get("msi_keyword", " pci=nomsi")

        disk_obj = utils_disk.GuestFSModiDisk(image_filename)
        kernel_config_ori = disk_obj.read_file(grub_file)
        kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
        if not kernel_config:
            raise error.TestError("Cannot find the kernel config, reg is %s" %
                                  kernel_cfg_pos_reg)
        kernel_config_line = kernel_config[0]

        kernel_need_modify = False
        if disable_pci_msi == "yes":
            if not re.findall(msi_keyword, kernel_config_line):
                kernel_config_set = kernel_config_line + msi_keyword
                kernel_need_modify = True
        else:
            if re.findall(msi_keyword, kernel_config_line):
                kernel_config_set = re.sub(msi_keyword, "", kernel_config_line)
                kernel_need_modify = True

        if kernel_need_modify:
            for vm in env.get_all_vms():
                if vm:
                    vm.destroy()
                    env.unregister_vm(vm.name)
            disk_obj.replace_image_file_content(grub_file, kernel_config_line,
                                                kernel_config_set)
        logging.debug("Guest cmdline 'pci=nomsi' setting is: [ %s ]" %
                      disable_pci_msi)

    kernel_extra_params = params.get("kernel_extra_params")
    if kernel_extra_params:
        image_filename = storage.get_image_filename(params,
                                                    data_dir.get_data_dir())
        grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
        kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
                                        r".*vmlinuz-\d+.*")

        disk_obj = utils_disk.GuestFSModiDisk(image_filename)
        kernel_config_ori = disk_obj.read_file(grub_file)
        kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
        if not kernel_config:
            raise error.TestError("Cannot find the kernel config, reg is %s" %
                                  kernel_cfg_pos_reg)
        kernel_config_line = kernel_config[0]

        kernel_need_modify = False
        if not re.findall(kernel_extra_params, kernel_config_line):
            kernel_config_set = kernel_config_line + kernel_extra_params
            kernel_need_modify = True

        if kernel_need_modify:
            for vm in env.get_all_vms():
                if vm:
                    vm.destroy()
                    env.unregister_vm(vm.name)
            disk_obj.replace_image_file_content(grub_file, kernel_config_line,
                                                kernel_config_set)
        logging.debug("Guest cmdline extra_params setting is: [ %s ]" %
                      kernel_extra_params)

    # Clone master image from vms.
    base_dir = data_dir.get_data_dir()
    if params.get("master_images_clone"):
        for vm_name in params.get("vms").split():
            vm = env.get_vm(vm_name)
            if vm:
                vm.destroy(free_mac_addresses=False)
                env.unregister_vm(vm_name)

            vm_params = params.object_params(vm_name)
            for image in vm_params.get("master_images_clone").split():
                image_obj = qemu_storage.QemuImg(params, base_dir, image)
                image_obj.clone_image(params, vm_name, image, base_dir)

    # Preprocess all VMs and images
    if params.get("not_preprocess", "no") == "no":
        process(test, params, env, preprocess_image, preprocess_vm)

    # Start the screendump thread
    if params.get("take_regular_screendumps") == "yes":
        global _screendump_thread, _screendump_thread_termination_event
        _screendump_thread_termination_event = threading.Event()
        _screendump_thread = threading.Thread(target=_take_screendumps,
                                              name='ScreenDump',
                                              args=(test, params, env))
        _screendump_thread.start()

    return params
Exemple #41
0
    def create(self, name=None, params=None, root_dir=None, timeout=5.0, migration_mode=None, mac_source=None):
        """
        Start the VM by running a qemu command.
        All parameters are optional. If name, params or root_dir are not
        supplied, the respective values stored as class attributes are used.

        @param name: The name of the object
        @param params: A dict containing VM params
        @param root_dir: Base directory for relative filenames
        @param migration_mode: If supplied, start VM for incoming migration
                using this protocol (either 'tcp', 'unix' or 'exec')
        @param migration_exec_cmd: Command to embed in '-incoming "exec: ..."'
                (e.g. 'gzip -c -d filename') if migration_mode is 'exec'
        @param mac_source: A VM object from which to copy MAC addresses. If not
                specified, new addresses will be generated.

        @raise VMCreateError: If qemu terminates unexpectedly
        @raise VMKVMInitError: If KVM initialization fails
        @raise VMHugePageError: If hugepage initialization fails
        @raise VMImageMissingError: If a CD image is missing
        @raise VMHashMismatchError: If a CD image hash has doesn't match the
                expected hash
        @raise VMBadPATypeError: If an unsupported PCI assignment type is
                requested
        @raise VMPAError: If no PCI assignable devices could be assigned
        """
        error.context("creating '%s'" % self.name)
        self.destroy(free_mac_addresses=False)

        if name is not None:
            self.name = name
        if params is not None:
            self.params = params
        if root_dir is not None:
            self.root_dir = root_dir
        name = self.name
        params = self.params
        root_dir = self.root_dir

        # Verify the md5sum of the ISO images
        for cdrom in params.objects("cdroms"):
            if params.get("medium") == "import":
                break
            cdrom_params = params.object_params(cdrom)
            iso = cdrom_params.get("cdrom")
            if (
                (self.driver_type == "xen")
                and (params.get("hvm_or_pv") == "pv")
                and (os.path.basename(iso) == "ks.iso")
            ):
                continue
            if iso:
                iso = utils_misc.get_path(data_dir.get_data_dir(), iso)
                if not os.path.exists(iso):
                    raise virt_vm.VMImageMissingError(iso)
                compare = False
                if cdrom_params.get("md5sum_1m"):
                    logging.debug("Comparing expected MD5 sum with MD5 sum of " "first MB of ISO file...")
                    actual_hash = utils.hash_file(iso, 1048576, method="md5")
                    expected_hash = cdrom_params.get("md5sum_1m")
                    compare = True
                elif cdrom_params.get("md5sum"):
                    logging.debug("Comparing expected MD5 sum with MD5 sum of " "ISO file...")
                    actual_hash = utils.hash_file(iso, method="md5")
                    expected_hash = cdrom_params.get("md5sum")
                    compare = True
                elif cdrom_params.get("sha1sum"):
                    logging.debug("Comparing expected SHA1 sum with SHA1 sum " "of ISO file...")
                    actual_hash = utils.hash_file(iso, method="sha1")
                    expected_hash = cdrom_params.get("sha1sum")
                    compare = True
                if compare:
                    if actual_hash == expected_hash:
                        logging.debug("Hashes match")
                    else:
                        raise virt_vm.VMHashMismatchError(actual_hash, expected_hash)

        # Make sure the following code is not executed by more than one thread
        # at the same time
        lockfile = open("/tmp/libvirt-autotest-vm-create.lock", "w+")
        fcntl.lockf(lockfile, fcntl.LOCK_EX)

        try:
            # Handle port redirections
            redir_names = params.objects("redirs")
            host_ports = utils_misc.find_free_ports(5000, 6000, len(redir_names))
            self.redirs = {}
            for i in range(len(redir_names)):
                redir_params = params.object_params(redir_names[i])
                guest_port = int(redir_params.get("guest_port"))
                self.redirs[guest_port] = host_ports[i]

            # Find available PCI devices
            self.pci_devices = []
            for device in params.objects("pci_devices"):
                self.pci_devices.append(device)

            # Find available VNC port, if needed
            if params.get("display") == "vnc":
                if params.get("vnc_autoport") == "yes":
                    self.vnc_port = None
                    self.vnc_autoport = True
                else:
                    self.vnc_port = utils_misc.find_free_port(5900, 6100)
                    self.vnc_autoport = False

            # Find available spice port, if needed
            if params.get("spice"):
                self.spice_port = utils_misc.find_free_port(8000, 8100)

            # Find random UUID if specified 'uuid = random' in config file
            if params.get("uuid") == "random":
                f = open("/proc/sys/kernel/random/uuid")
                self.uuid = f.read().strip()
                f.close()

            # Generate or copy MAC addresses for all NICs
            for nic in self.virtnet:
                nic_params = dict(nic)
                if mac_source:
                    # Will raise exception if source doesn't
                    # have cooresponding nic
                    logging.debug("Copying mac for nic %s from VM %s" % (nic.nic_name, mac_source.nam))
                    nic_params["mac"] = mac_source.get_mac_address(nic.nic_name)
                # make_create_command() calls vm.add_nic (i.e. on a copy)
                nic = self.add_nic(**nic_params)
                logging.debug("VM.create activating nic %s" % nic)
                self.activate_nic(nic.nic_name)

            # Make qemu command
            install_command = self.make_create_command()

            logging.info("Running libvirt command (reformatted):")
            for item in install_command.replace(" -", " \n    -").splitlines():
                logging.info("%s", item)
            utils.run(install_command, verbose=False)
            # Wait for the domain to be created
            utils_misc.wait_for(func=self.is_alive, timeout=60, text=("waiting for domain %s to start" % self.name))
            self.uuid = virsh.domuuid(self.name, uri=self.connect_uri)

            # Establish a session with the serial console
            if self.only_pty == True:
                self.serial_console = aexpect.ShellSession(
                    "virsh console %s" % self.name,
                    auto_close=False,
                    output_func=utils_misc.log_line,
                    output_params=("serial-%s.log" % name,),
                )
            else:
                self.serial_console = aexpect.ShellSession(
                    "tail -f %s" % self.get_serial_console_filename(),
                    auto_close=False,
                    output_func=utils_misc.log_line,
                    output_params=("serial-%s.log" % name,),
                )

        finally:
            fcntl.lockf(lockfile, fcntl.LOCK_UN)
            lockfile.close()
Exemple #42
0
    def check_disk_params(self, params):
        """
        Check gathered info from qtree/block with params
        :param params: autotest params
        :return: number of errors
        """
        def check_drive_format(node, params):
            """ checks the drive format according to qtree info """
            expected = params.get('drive_format')
            if expected == 'scsi':
                if arch.ARCH == 'ppc64':
                    expected = 'spapr-vscsi'
                else:
                    expected = 'lsi53c895a'
            elif expected.startswith('scsi'):
                expected = params.get('scsi_hba', 'virtio-scsi-pci')
            elif expected.startswith('usb'):
                expected = 'usb-storage'
            try:
                if expected == 'virtio':
                    actual = node.qtree['type']
                else:
                    actual = node.parent.parent.qtree.get('type')
            except AttributeError:
                logging.error(
                    "Failed to check drive format, can't get parent"
                    "of:\n%s", node)
            if actual == 'virtio-scsi-device':  # new name for virtio-scsi
                actual = 'virtio-scsi-pci'
            if expected not in actual:
                return ("drive format in qemu is %s, in autotest %s" %
                        (actual, expected))

        err = 0
        disks = {}
        for disk in self.disks:
            if isinstance(disk, QtreeDisk):
                disks[disk.get_qname()] = (disk.get_params().copy(), disk)
        # We don't have the params name so we need to map file_names instead
        qname = None
        for name in params.objects('cdroms'):
            image_name = utils_misc.get_path(
                data_dir.get_data_dir(),
                params.object_params(name).get('cdrom', ''))
            image_name = os.path.realpath(image_name)
            for (qname, disk) in disks.iteritems():
                if disk[0].get('image_name') == image_name:
                    break
            else:
                continue  # Not /proc/scsi cdrom device
            disks.pop(qname)
        for name in params.objects('images'):
            current = None
            image_params = params.object_params(name)

            base_dir = image_params.get("images_base_dir",
                                        data_dir.get_data_dir())

            image_name = os.path.realpath(
                storage.get_image_filename(image_params, base_dir))
            for (qname, disk) in disks.iteritems():
                if disk[0].get('image_name') == image_name:
                    current = disk[0]
                    current_node = disk[1]
                    # autotest params might use relative path
                    current['image_name'] = image_params.get('image_name')
                    break
            if not current:
                logging.error("Disk %s is not in qtree but is in params.",
                              name)
                err += 1
                continue
            for prop in current.iterkeys():
                handled = False
                if prop == "drive_format":
                    out = check_drive_format(current_node, image_params)
                    if out:
                        logging.error("Disk %s %s", qname, out)
                        err += 1
                    handled = True
                elif (image_params.get(prop)
                      and image_params.get(prop) == current.get(prop)):
                    handled = True
                if not handled:
                    logging.error(
                        "Disk %s property %s=%s doesn't match params"
                        " %s", qname, prop, current.get(prop),
                        image_params.get(prop))
                    err += 1
            disks.pop(qname)
        if disks:
            logging.error(
                'Some disks were in qtree but not in autotest params'
                ': %s', disks)
            err += 1
        return err
    last_index = -1

    logging.info("Starting test job at %s", test_start_time)
    logging.info("")

    logging.info(version.get_pretty_version_info())
    logging.info("")

    cleanup_env(parser, options)

    d = parser.get_dicts().next()

    if not options.vt_config:
        if not options.vt_keep_image_between_tests:
            logging.debug("Creating first backup of guest image")
            qemu_img = storage.QemuImg(d, data_dir.get_data_dir(), "image")
            qemu_img.backup_image(d, data_dir.get_data_dir(), 'backup', True)
            logging.debug("")

    for line in get_cartesian_parser_details(parser).splitlines():
        logging.info(line)

    logging.info("Defined test set:")
    for count, dic in enumerate(parser.get_dicts()):
        shortname = d.get("_name_map_file")["subtests.cfg"]

        logging.info("Test %4d:  %s", count + 1, shortname)
        last_index += 1

    if last_index == -1:
        print_stdout("No tests generated by config file %s" % parser.filename)
def bootstrap_tests(options):
    """
    Bootstrap process (download the appropriate JeOS file to data dir).

    This function will check whether the JeOS is in the right location of the
    data dir, if not, it will download it non interactively.

    :param options: OptParse object with program command line options.
    """
    test_dir = os.path.dirname(sys.modules[__name__].__file__)

    if options.type:
        test_dir = os.path.abspath(os.path.join(os.path.dirname(test_dir),
                                                options.type))
    elif options.config:
        parent_config_dir = os.path.dirname(os.path.dirname(options.config))
        parent_config_dir = os.path.dirname(parent_config_dir)
        options.type = parent_config_dir
        test_dir = os.path.abspath(parent_config_dir)

    if options.type == 'qemu':
        check_modules = arch.get_kvm_module_list()
    else:
        check_modules = None
    online_docs_url = "https://github.com/autotest/virt-test/wiki"

    kwargs = {'test_name': options.type,
              'test_dir': test_dir,
              'base_dir': data_dir.get_data_dir(),
              'default_userspace_paths': None,
              'check_modules': check_modules,
              'online_docs_url': online_docs_url,
              'download_image': not options.no_downloads,
              'restore_image': not options.keep_image,
              'interactive': False}

    # Tolerance we have without printing a message for the user to wait (3 s)
    tolerance = 3
    failed = False
    wait_message_printed = False

    bg = utils.InterruptedThread(bootstrap.bootstrap, kwargs=kwargs)
    t_begin = time.time()
    bg.start()

    while bg.isAlive():
        t_elapsed = time.time() - t_begin
        if t_elapsed > tolerance and not wait_message_printed:
            print_stdout("Running setup. Please wait...")
            wait_message_printed = True
            # if bootstrap takes too long, we temporarily make stdout verbose
            # again, so the user can see what's taking so long
            sys.stdout.restore()
        time.sleep(0.1)

    # in case stdout was restored above, redirect it again
    sys.stdout.redirect()

    reason = None
    try:
        bg.join()
    except Exception, e:
        failed = True
        reason = e
Exemple #45
0
            except Exception, e:
                logging.error("Unexpected error:" % e)
            libvirtd_inst.restart()

    # Execute any pre_commands
    if params.get("pre_command"):
        process_command(test, params, env, params.get("pre_command"),
                        int(params.get("pre_command_timeout", "600")),
                        params.get("pre_command_noncritical") == "yes")

    # if you want set "pci=nomsi" before test, set "disable_pci_msi = yes"
    # and pci_msi_sensitive = "yes"
    if params.get("pci_msi_sensitive", "no") == "yes":
        disable_pci_msi = params.get("disable_pci_msi", "no")
        image_filename = storage.get_image_filename(params,
                                                    data_dir.get_data_dir())
        grub_file = params.get("grub_file", "/boot/grub2/grub.cfg")
        kernel_cfg_pos_reg = params.get("kernel_cfg_pos_reg",
                                        r".*vmlinuz-\d+.*")
        msi_keyword = params.get("msi_keyword", " pci=nomsi")

        disk_obj = utils_disk.GuestFSModiDisk(image_filename)
        kernel_config_ori = disk_obj.read_file(grub_file)
        kernel_config = re.findall(kernel_cfg_pos_reg, kernel_config_ori)
        if not kernel_config:
            raise error.TestError("Cannot find the kernel config, reg is %s" %
                                  kernel_cfg_pos_reg)
        kernel_config_line = kernel_config[0]

        kernel_need_modify = False
        if disable_pci_msi == "yes":
Exemple #46
0
        verify_mandatory_programs(options.vt_type, guest_os)
    except Exception, details:
        logging.info(details)
        logging.info('Install the missing programs and/or headers and '
                     're-run boostrap')
        sys.exit(1)

    logging.info("")
    step += 1
    logging.info("%d - Checking the recommended programs", step)
    verify_recommended_programs(options.vt_type)

    logging.info("")
    step += 1
    logging.info("%d - Verifying directories", step)
    datadir = data_dir.get_data_dir()
    shared_dir = os.path.dirname(datadir)
    sub_dir_list = ["images", "isos", "steps_data", "gpg"]
    for sub_dir in sub_dir_list:
        sub_dir_path = os.path.join(datadir, sub_dir)
        if not os.path.isdir(sub_dir_path):
            logging.debug("Creating %s", sub_dir_path)
            os.makedirs(sub_dir_path)
        else:
            logging.debug("Dir %s exists, not creating",
                          sub_dir_path)

    test_dir = data_dir.get_backend_dir(options.vt_type)
    if options.vt_type == 'libvirt':
        step = create_config_files(test_dir, shared_dir, interactive, step,
                                   force_update=options.vt_update_config)
Exemple #47
0
def bootstrap(test_name, test_dir, base_dir, default_userspace_paths,
              check_modules, online_docs_url, restore_image=False,
              download_image=True, interactive=True, selinux=False,
              verbose=False, update_providers=False,
              guest_os=defaults.DEFAULT_GUEST_OS):
    """
    Common virt test assistant module.

    :param test_name: Test name, such as "qemu".
    :param test_dir: Path with the test directory.
    :param base_dir: Base directory used to hold images and isos.
    :param default_userspace_paths: Important programs for a successful test
            execution.
    :param check_modules: Whether we want to verify if a given list of modules
            is loaded in the system.
    :param online_docs_url: URL to an online documentation system, such as a
            wiki page.
    :param restore_image: Whether to restore the image from the pristine.
    :param interactive: Whether to ask for confirmation.
    :param verbose: Verbose output.
    :param selinux: Whether setup SELinux contexts for shared/data.
    :param update_providers: Whether to update test providers if they are already
            downloaded.
    :param guest_os: Specify the guest image used for bootstrapping. By default
            the JeOS image is used.

    :raise error.CmdError: If JeOS image failed to uncompress
    :raise ValueError: If 7za was not found
    """
    if interactive:
        logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
                                          verbose=verbose)
    logging.info("%s test config helper", test_name)
    step = 0

    logging.info("")
    step += 1
    logging.info("%d - Updating all test providers", step)
    asset.download_all_test_providers(update_providers)

    logging.info("")
    step += 1
    logging.info("%d - Checking the mandatory programs and headers", step)
    verify_mandatory_programs(test_name)

    logging.info("")
    step += 1
    logging.info("%d - Checking the recommended programs", step)
    verify_recommended_programs(test_name)

    logging.info("")
    step += 1
    logging.info("%d - Verifying directories", step)
    shared_dir = os.path.dirname(data_dir.get_data_dir())
    sub_dir_list = ["images", "isos", "steps_data", "gpg"]
    for sub_dir in sub_dir_list:
        sub_dir_path = os.path.join(base_dir, sub_dir)
        if not os.path.isdir(sub_dir_path):
            logging.debug("Creating %s", sub_dir_path)
            os.makedirs(sub_dir_path)
        else:
            logging.debug("Dir %s exists, not creating",
                          sub_dir_path)

    datadir = data_dir.get_data_dir()
    if test_name == 'libvirt':
        create_config_files(test_dir, shared_dir, interactive, step)
        create_subtests_cfg(test_name)
        create_guest_os_cfg(test_name)
        # Don't bother checking if changes can't be made
        if os.getuid() == 0:
            verify_selinux(datadir,
                           os.path.join(datadir, 'images'),
                           os.path.join(datadir, 'isos'),
                           data_dir.get_tmp_dir(),
                           interactive, selinux)

    # lvsb test doesn't use any shared configs
    elif test_name == 'lvsb':
        create_subtests_cfg(test_name)
        if os.getuid() == 0:
            # Don't bother checking if changes can't be made
            verify_selinux(datadir,
                           os.path.join(datadir, 'images'),
                           os.path.join(datadir, 'isos'),
                           data_dir.get_tmp_dir(),
                           interactive, selinux)
    else:  # Some other test
        create_config_files(test_dir, shared_dir, interactive, step)
        create_subtests_cfg(test_name)
        create_guest_os_cfg(test_name)

    if download_image or restore_image:
        logging.info("")
        step += 2
        logging.info("%s - Verifying (and possibly downloading) guest image",
                     step)
        for os_info in get_guest_os_info_list(test_name, guest_os):
            os_asset = os_info['asset']
            asset.download_asset(os_asset, interactive=interactive,
                                 restore_image=restore_image)

    if check_modules:
        logging.info("")
        step += 1
        logging.info("%d - Checking for modules %s", step,
                     ", ".join(check_modules))
        for module in check_modules:
            if not utils.module_is_loaded(module):
                logging.warning("Module %s is not loaded. You might want to "
                                "load it", module)
            else:
                logging.debug("Module %s loaded", module)

    if online_docs_url:
        logging.info("")
        step += 1
        logging.info("%d - If you wish, take a look at the online docs for "
                     "more info", step)
        logging.info("")
        logging.info(online_docs_url)
def bootstrap_tests(options):
    """
    Bootstrap process (download the appropriate JeOS file to data dir).

    This function will check whether the JeOS is in the right location of the
    data dir, if not, it will download it non interactively.

    :param options: OptParse object with program command line options.
    """
    if options.type:
        test_dir = data_dir.get_backend_dir(options.type)
    elif options.config:
        parent_config_dir = os.path.dirname(os.path.dirname(options.config))
        parent_config_dir = os.path.dirname(parent_config_dir)
        options.type = parent_config_dir
        test_dir = os.path.abspath(parent_config_dir)

    if options.type == 'qemu':
        check_modules = arch.get_kvm_module_list()
    else:
        check_modules = None
    online_docs_url = "https://github.com/autotest/virt-test/wiki"

    if not options.config:
        restore_image = not options.keep_image
    else:
        restore_image = False

    kwargs = {'test_name': options.type,
              'test_dir': test_dir,
              'base_dir': data_dir.get_data_dir(),
              'default_userspace_paths': None,
              'check_modules': check_modules,
              'online_docs_url': online_docs_url,
              'download_image': not options.no_downloads,
              'selinux': options.selinux_setup,
              'restore_image': restore_image,
              'interactive': False,
              'update_providers': options.update_providers}

    # Tolerance we have without printing a message for the user to wait (3 s)
    tolerance = 3
    failed = False
    wait_message_printed = False

    bg = utils.InterruptedThread(bootstrap.bootstrap, kwargs=kwargs)
    t_begin = time.time()
    bg.start()

    while bg.isAlive():
        t_elapsed = time.time() - t_begin
        if t_elapsed > tolerance and not wait_message_printed:
            print_stdout("Running setup. Please wait...")
            wait_message_printed = True
            # if bootstrap takes too long, we temporarily make stdout verbose
            # again, so the user can see what's taking so long
            sys.stdout.restore()
        time.sleep(0.1)

    # in case stdout was restored above, redirect it again
    sys.stdout.redirect()

    reason = None
    try:
        bg.join()
    except Exception, e:
        failed = True
        reason = e
Exemple #49
0
def preprocess(test, params, env):
    """
    Preprocess all VMs and images according to the instructions in params.
    Also, collect some host information, such as the KVM version.

    @param test: An Autotest test object.
    @param params: A dict containing all VM and image parameters.
    @param env: The environment (a dict-like object).
    """
    error.context("preprocessing")
    # First, let's verify if this test does require root or not. If it
    # does and the test suite is running as a regular user, we shall just
    # throw a TestNAError exception, which will skip the test.
    if params.get('requires_root', 'no') == 'yes':
        utils_test.verify_running_as_root()

    port = params.get('shell_port')
    prompt = params.get('shell_prompt')
    address = params.get('ovirt_node_address')
    username = params.get('ovirt_node_user')
    password = params.get('ovirt_node_password')

    # Start tcpdump if it isn't already running
    if "address_cache" not in env:
        env["address_cache"] = {}
    if "tcpdump" in env and not env["tcpdump"].is_alive():
        env["tcpdump"].close()
        del env["tcpdump"]
    if "tcpdump" not in env and params.get("run_tcpdump", "yes") == "yes":
        cmd = "%s -npvi any 'dst port 68'" % utils_misc.find_command("tcpdump")
        if params.get("remote_preprocess") == "yes":
            login_cmd = ("ssh -o UserKnownHostsFile=/dev/null -o \
                         PreferredAuthentications=password -p %s %s@%s" %
                         (port, username, address))
            env["tcpdump"] = aexpect.ShellSession(
                login_cmd,
                output_func=_update_address_cache,
                output_params=(env["address_cache"],))
            remote._remote_login(env["tcpdump"], username, password, prompt)
            env["tcpdump"].sendline(cmd)
        else:
            env["tcpdump"] = aexpect.Tail(
                command=cmd,
                output_func=_update_address_cache,
                output_params=(env["address_cache"],))

        if utils_misc.wait_for(lambda: not env["tcpdump"].is_alive(),
                              0.1, 0.1, 1.0):
            logging.warn("Could not start tcpdump")
            logging.warn("Status: %s" % env["tcpdump"].get_status())
            logging.warn("Output:" + utils_misc.format_str_for_message(
                env["tcpdump"].get_output()))

    # Destroy and remove VMs that are no longer needed in the environment
    requested_vms = params.objects("vms")
    for key in env.keys():
        vm = env[key]
        if not isinstance(vm, virt_vm.BaseVM):
            continue
        if not vm.name in requested_vms:
            vm.destroy()
            del env[key]

    # Get Host cpu type
    if params.get("auto_cpu_model") == "yes":
        if not env.get("cpu_model"):
            env["cpu_model"] = utils_misc.get_cpu_model()
        params["cpu_model"] = env.get("cpu_model")

    kvm_ver_cmd = params.get("kvm_ver_cmd", "")

    if kvm_ver_cmd:
        try:
            cmd_result = utils.run(kvm_ver_cmd)
            kvm_version = cmd_result.stdout.strip()
        except error.CmdError:
            kvm_version = "Unknown"
    else:
        # Get the KVM kernel module version and write it as a keyval
        if os.path.exists("/dev/kvm"):
            try:
                kvm_version = open("/sys/module/kvm/version").read().strip()
            except Exception:
                kvm_version = os.uname()[2]
        else:
            logging.warning("KVM module not loaded")
            kvm_version = "Unknown"

    logging.debug("KVM version: %s" % kvm_version)
    test.write_test_keyval({"kvm_version": kvm_version})

    # Get the KVM userspace version and write it as a keyval
    kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")

    if kvm_userspace_ver_cmd:
        try:
            cmd_result = utils.run(kvm_userspace_ver_cmd)
            kvm_userspace_version = cmd_result.stdout.strip()
        except error.CmdError:
            kvm_userspace_version = "Unknown"
    else:
        qemu_path = utils_misc.get_path(test.bindir,
                                        params.get("qemu_binary", "qemu"))
        version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
        matches = re.findall("[Vv]ersion .*?,", version_line)
        if matches:
            kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
        else:
            kvm_userspace_version = "Unknown"

    logging.debug("KVM userspace version: %s" % kvm_userspace_version)
    test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})

    if params.get("setup_hugepages") == "yes":
        h = test_setup.HugePageConfig(params)
        h.setup()
        if params.get("vm_type") == "libvirt":
            libvirt_vm.libvirtd_restart()

    if params.get("setup_thp") == "yes":
        thp = test_setup.TransparentHugePageConfig(test, params)
        thp.setup()

    # Execute any pre_commands
    if params.get("pre_command"):
        process_command(test, params, env, params.get("pre_command"),
                        int(params.get("pre_command_timeout", "600")),
                        params.get("pre_command_noncritical") == "yes")

    #Clone master image from vms.
    base_dir = data_dir.get_data_dir()
    if params.get("master_images_clone"):
        for vm_name in params.get("vms").split():
            vm = env.get_vm(vm_name)
            if vm:
                vm.destroy(free_mac_addresses=False)
                env.unregister_vm(vm_name)

            vm_params = params.object_params(vm_name)
            for image in vm_params.get("master_images_clone").split():
                image_obj = qemu_storage.QemuImg(params, base_dir, image)
                image_obj.clone_image(params, vm_name, image, base_dir)

    # Preprocess all VMs and images
    if params.get("not_preprocess","no") == "no":
        process(test, params, env, preprocess_image, preprocess_vm)

    # Start the screendump thread
    if params.get("take_regular_screendumps") == "yes":
        global _screendump_thread, _screendump_thread_termination_event
        _screendump_thread_termination_event = threading.Event()
        _screendump_thread = threading.Thread(target=_take_screendumps,
                                              args=(test, params, env))
        _screendump_thread.start()
    def __init__(self, vm):
        """ Initialize worker for use (including port init on guest) """
        self.vm = vm
        self.session = self.vm.wait_for_login()
        self.__cmd_execute_worker = None

        # Detect the OS version
        guest_script_py = "virtio_console_guest.py"
        out = self.session.cmd_output("echo on")
        if "on" in out:
            self.os_linux = True
            guest_script_path = "/tmp/%s" % guest_script_py
            cmd_guest_size = ("du -b %s | cut -f1"
                              % guest_script_path)
            cmd_already_compiled_chck = "ls %so" % guest_script_path
            cmd_compile = ("python -OO %s -c "
                           "&& echo -n 'PASS: Compile virtio_guest finished' "
                           "|| echo -n 'FAIL: Compile virtio_guest failed'"
                           % guest_script_path)
            self.__cmd_execute_worker = ("python %so"
                                         "&& echo -n 'PASS: virtio_guest finished' "
                                         "|| echo -n 'FAIL: virtio_guest failed'"
                                         % guest_script_path)
        else:
            self.os_linux = False
            guest_script_path = "C:\\%s" % guest_script_py
            cmd_guest_size = ("for %%I in (%s) do @echo %%~zI"
                              % guest_script_path)
            cmd_already_compiled_chck = "dir %so" % guest_script_path
            cmd_compile = ("%s -c "
                           "&& echo PASS: Compile virtio_guest finished "
                           "|| echo FAIL: Compile virtio_guest failed"
                           % guest_script_path)
            self.__cmd_execute_worker = ("%so "
                                         "&& echo PASS: virtio_guest finished "
                                         "|| echo FAIL: virtio_guest failed"
                                         % guest_script_path)

        # Copy, compile and run the worker
        timeout = 10
        base_path = os.path.dirname(data_dir.get_data_dir())
        guest_script_src = os.path.join(base_path, 'scripts',
                                        'virtio_console_guest.py')
        script_size = utils.system_output("du -b %s | cut -f1"
                                          % guest_script_src).strip()
        script_size_guest = self.session.cmd_output(cmd_guest_size).strip()
        if (script_size != script_size_guest or
                self.session.cmd_status(cmd_already_compiled_chck)):
            if self.os_linux:
                # Disable [email protected] on systemd-like hosts
                self.session.cmd_status('systemctl mask '
                                        '*****@*****.**')
                self.session.cmd_status('systemctl stop '
                                        '*****@*****.**')
            # Copy virtio_console_guest.py into guests
            self.vm.copy_files_to(guest_script_src, guest_script_path)

            # set echo off (self.cmd() musn't contain C:)
            self.session.sendline("echo off")
            # Compile worker
            logging.debug("Compile %s on guest %s", guest_script_py,
                          self.vm.name)
            try:
                self.cmd(cmd_compile, timeout)
            except VirtioPortException:
                if not self.os_linux:
                    logging.error("Script execution failed, do you have python"
                                  " and pywin32 installed? Currently this "
                                  "needs to be done manually!")
                raise
            self.session.sendline()

        # set echo off (self.cmd() musn't contain C:)
        self.session.sendline("echo off")
        logging.debug("Starting %so on guest %s", guest_script_py,
                      self.vm.name)
        self._execute_worker(timeout)
        self._init_guest(timeout)
Exemple #51
0
def bootstrap(test_name,
              test_dir,
              base_dir,
              default_userspace_paths,
              check_modules,
              online_docs_url,
              restore_image=False,
              download_image=True,
              interactive=True,
              verbose=False):
    """
    Common virt test assistant module.

    @param test_name: Test name, such as "qemu".
    @param test_dir: Path with the test directory.
    @param base_dir: Base directory used to hold images and isos.
    @param default_userspace_paths: Important programs for a successful test
            execution.
    @param check_modules: Whether we want to verify if a given list of modules
            is loaded in the system.
    @param online_docs_url: URL to an online documentation system, such as a
            wiki page.
    @param restore_image: Whether to restore the image from the pristine.
    @param interactive: Whether to ask for confirmation.

    @raise error.CmdError: If JeOS image failed to uncompress
    @raise ValueError: If 7za was not found
    """
    if interactive:
        logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
                                          verbose=verbose)
    logging.info("%s test config helper", test_name)
    step = 0

    logging.info("")
    step += 1
    logging.info("%d - Checking the mandatory programs and headers", step)
    verify_mandatory_programs(test_name)

    logging.info("")
    step += 1
    logging.info("%d - Checking the recommended programs", step)
    verify_recommended_programs(test_name)

    logging.info("")
    step += 1
    logging.info("%d - Verifying directories", step)
    shared_dir = os.path.dirname(data_dir.get_data_dir())
    sub_dir_list = ["images", "isos", "steps_data"]
    for sub_dir in sub_dir_list:
        sub_dir_path = os.path.join(base_dir, sub_dir)
        if not os.path.isdir(sub_dir_path):
            logging.debug("Creating %s", sub_dir_path)
            os.makedirs(sub_dir_path)
        else:
            logging.debug("Dir %s exists, not creating", sub_dir_path)

    create_config_files(test_dir, shared_dir, interactive, step)
    create_subtests_cfg(test_name)
    create_guest_os_cfg(test_name)

    if download_image or restore_image:
        logging.info("")
        step += 2
        logging.info("%s - Verifying (and possibly downloading) guest image",
                     step)
        asset.download_asset('jeos-17-64',
                             interactive=interactive,
                             restore_image=restore_image)

    if check_modules:
        logging.info("")
        step += 1
        logging.info("%d - Checking for modules %s", step,
                     ", ".join(check_modules))
        for module in check_modules:
            if not utils.module_is_loaded(module):
                logging.warning(
                    "Module %s is not loaded. You might want to "
                    "load it", module)
            else:
                logging.debug("Module %s loaded", module)

    if online_docs_url:
        logging.info("")
        step += 1
        logging.info(
            "%d - If you wish, take a look at the online docs for "
            "more info", step)
        logging.info("")
        logging.info(online_docs_url)
Exemple #52
0
    def make_create_command(self, name=None, params=None, root_dir=None):
        """
        Generate a libvirt command line. All parameters are optional. If a
        parameter is not supplied, the corresponding value stored in the
        class attributes is used.

        @param name: The name of the object
        @param params: A dict containing VM params
        @param root_dir: Base directory for relative filenames

        @note: The params dict should contain:
               mem -- memory size in MBs
               cdrom -- ISO filename to use with the qemu -cdrom parameter
               extra_params -- a string to append to the qemu command
               shell_port -- port of the remote shell daemon on the guest
               (SSH, Telnet or the home-made Remote Shell Server)
               shell_client -- client program to use for connecting to the
               remote shell daemon on the guest (ssh, telnet or nc)
               x11_display -- if specified, the DISPLAY environment variable
               will be be set to this value for the qemu process (useful for
               SDL rendering)
               images -- a list of image object names, separated by spaces
               nics -- a list of NIC object names, separated by spaces

               For each image in images:
               drive_format -- string to pass as 'if' parameter for this
               image (e.g. ide, scsi)
               image_snapshot -- if yes, pass 'snapshot=on' to qemu for
               this image
               image_boot -- if yes, pass 'boot=on' to qemu for this image
               In addition, all parameters required by get_image_filename.

               For each NIC in nics:
               nic_model -- string to pass as 'model' parameter for this
               NIC (e.g. e1000)
        """
        # helper function for command line option wrappers
        def has_option(help_text, option):
            return bool(re.search(r"--%s" % option, help_text, re.MULTILINE))

        # Wrappers for all supported libvirt command line parameters.
        # This is meant to allow support for multiple libvirt versions.
        # Each of these functions receives the output of 'libvirt --help' as a
        # parameter, and should add the requested command line option
        # accordingly.

        def add_name(help_text, name):
            return " --name '%s'" % name

        def add_machine_type(help_text, machine_type):
            if has_option(help_text, "machine"):
                return " --machine %s" % machine_type
            else:
                return ""

        def add_hvm_or_pv(help_text, hvm_or_pv):
            if hvm_or_pv == "hvm":
                return " --hvm --accelerate"
            elif hvm_or_pv == "pv":
                return " --paravirt"
            else:
                logging.warning("Unknown virt type hvm_or_pv, using default.")
                return ""

        def add_mem(help_text, mem):
            return " --ram=%s" % mem

        def add_check_cpu(help_text):
            if has_option(help_text, "check-cpu"):
                return " --check-cpu"
            else:
                return ""

        def add_smp(help_text, smp):
            return " --vcpu=%s" % smp

        def add_location(help_text, location):
            if has_option(help_text, "location"):
                return " --location %s" % location
            else:
                return ""

        def add_cdrom(help_text, filename, index=None):
            if has_option(help_text, "cdrom"):
                return " --cdrom %s" % filename
            else:
                return ""

        def add_pxe(help_text):
            if has_option(help_text, "pxe"):
                return " --pxe"
            else:
                return ""

        def add_import(help_text):
            if has_option(help_text, "import"):
                return " --import"
            else:
                return ""

        def add_drive(
            help_text,
            filename,
            pool=None,
            vol=None,
            device=None,
            bus=None,
            perms=None,
            size=None,
            sparse=False,
            cache=None,
            fmt=None,
        ):
            cmd = " --disk"
            if filename:
                cmd += " path=%s" % filename
            elif pool:
                if vol:
                    cmd += " vol=%s/%s" % (pool, vol)
                else:
                    cmd += " pool=%s" % pool
            if device:
                cmd += ",device=%s" % device
            if bus:
                cmd += ",bus=%s" % bus
            if perms:
                cmd += ",%s" % perms
            if size:
                cmd += ",size=%s" % size.rstrip("Gg")
            if sparse:
                cmd += ",sparse=false"
            if fmt:
                cmd += ",format=%s" % fmt
            if cache:
                cmd += ",cache=%s" % cache
            return cmd

        def add_floppy(help_text, filename):
            return " --disk path=%s,device=floppy,ro" % filename

        def add_vnc(help_text, vnc_port=None):
            if vnc_port:
                return " --vnc --vncport=%d" % (vnc_port)
            else:
                return " --vnc"

        def add_vnclisten(help_text, vnclisten):
            if has_option(help_text, "vnclisten"):
                return " --vnclisten=%s" % (vnclisten)
            else:
                return ""

        def add_sdl(help_text):
            if has_option(help_text, "sdl"):
                return " --sdl"
            else:
                return ""

        def add_nographic(help_text):
            return " --nographics"

        def add_video(help_text, video_device):
            if has_option(help_text, "video"):
                return " --video=%s" % (video_device)
            else:
                return ""

        def add_uuid(help_text, uuid):
            if has_option(help_text, "uuid"):
                return " --uuid %s" % uuid
            else:
                return ""

        def add_os_type(help_text, os_type):
            if has_option(help_text, "os-type"):
                return " --os-type %s" % os_type
            else:
                return ""

        def add_os_variant(help_text, os_variant):
            if has_option(help_text, "os-variant"):
                return " --os-variant %s" % os_variant
            else:
                return ""

        def add_pcidevice(help_text, pci_device):
            if has_option(help_text, "host-device"):
                return " --host-device %s" % pci_device
            else:
                return ""

        def add_soundhw(help_text, sound_device):
            if has_option(help_text, "soundhw"):
                return " --soundhw %s" % sound_device
            else:
                return ""

        def add_serial(help_text, filename):
            if has_option(help_text, "serial"):
                return "  --serial file,path=%s --serial pty" % filename
            else:
                self.only_pty = True
                return ""

        def add_kernel_cmdline(help_text, cmdline):
            return " -append %s" % cmdline

        def add_connect_uri(help_text, uri):
            if uri and has_option(help_text, "connect"):
                return " --connect=%s" % uri
            else:
                return ""

        def add_nic(help_text, nic_params):
            """
            Return additional command line params based on dict-like nic_params
            """
            mac = nic_params.get("mac")
            nettype = nic_params.get("nettype")
            netdst = nic_params.get("netdst")
            nic_model = nic_params.get("nic_model")
            if nettype:
                result = " --network=%s" % nettype
            else:
                result = ""
            if has_option(help_text, "bridge"):
                # older libvirt (--network=NATdev --bridge=bridgename --mac=mac)
                if nettype != "user":
                    result += ":%s" % netdst
                if mac:  # possible to specify --mac w/o --network
                    result += " --mac=%s" % mac
            else:
                # newer libvirt (--network=mynet,model=virtio,mac=00:11)
                if nettype != "user":
                    result += "=%s" % netdst
                if nettype and nic_model:  # only supported along with nettype
                    result += ",model=%s" % nic_model
                if nettype and mac:
                    result += ",mac=%s" % mac
                elif mac:  # possible to specify --mac w/o --network
                    result += " --mac=%s" % mac
            logging.debug("vm.make_create_command.add_nic returning: %s" % result)
            return result

        # End of command line option wrappers

        if name is None:
            name = self.name
        if params is None:
            params = self.params
        if root_dir is None:
            root_dir = self.root_dir

        # Clone this VM using the new params
        vm = self.clone(name, params, root_dir, copy_state=True)

        virt_install_binary = utils_misc.get_path(root_dir, params.get("virt_install_binary", "virt-install"))

        help_text = utils.system_output("%s --help" % virt_install_binary)

        # Find all supported machine types, so we can rule out an unsupported
        # machine type option passed in the configuration.
        hvm_or_pv = params.get("hvm_or_pv", "hvm")
        # default to 'uname -m' output
        arch_name = params.get("vm_arch_name", utils.get_current_kernel_arch())
        capabs = libvirt_xml.LibvirtXML()
        support_machine_type = capabs.os_arch_machine_map[hvm_or_pv][arch_name]
        logging.debug("Machine types supported for %s\%s: %s" % (hvm_or_pv, arch_name, support_machine_type))

        # Start constructing the qemu command
        virt_install_cmd = ""
        # Set the X11 display parameter if requested
        if params.get("x11_display"):
            virt_install_cmd += "DISPLAY=%s " % params.get("x11_display")
        # Add the qemu binary
        virt_install_cmd += virt_install_binary

        # set connect uri
        virt_install_cmd += add_connect_uri(help_text, self.connect_uri)

        # hvm or pv specificed by libvirt switch (pv used  by Xen only)
        if hvm_or_pv:
            virt_install_cmd += add_hvm_or_pv(help_text, hvm_or_pv)

        # Add the VM's name
        virt_install_cmd += add_name(help_text, name)

        machine_type = params.get("machine_type")
        if machine_type:
            if machine_type in support_machine_type:
                virt_install_cmd += add_machine_type(help_text, machine_type)
            else:
                raise error.TestNAError("Unsupported machine type %s." % (machine_type))

        mem = params.get("mem")
        if mem:
            virt_install_cmd += add_mem(help_text, mem)

        # TODO: should we do the check before we call ? negative case ?
        check_cpu = params.get("use_check_cpu")
        if check_cpu:
            virt_install_cmd += add_check_cpu(help_text)

        smp = params.get("smp")
        if smp:
            virt_install_cmd += add_smp(help_text, smp)

        # TODO: directory location for vmlinuz/kernel for cdrom install ?
        location = None
        if params.get("medium") == "url":
            location = params.get("url")

        elif params.get("medium") == "kernel_initrd":
            # directory location of kernel/initrd pair (directory layout must
            # be in format libvirt will recognize)
            location = params.get("image_dir")

        elif params.get("medium") == "nfs":
            location = "nfs:%s:%s" % (params.get("nfs_server"), params.get("nfs_dir"))

        elif params.get("medium") == "cdrom":
            if params.get("use_libvirt_cdrom_switch") == "yes":
                virt_install_cmd += add_cdrom(help_text, params.get("cdrom_cd1"))
            elif params.get("unattended_delivery_method") == "integrated":
                virt_install_cmd += add_cdrom(
                    help_text, os.path.join(data_dir.get_data_dir(), params.get("cdrom_unattended"))
                )
            else:
                location = params.get("image_dir")
                kernel_dir = os.path.dirname(params.get("kernel"))
                kernel_parent_dir = os.path.dirname(kernel_dir)
                pxeboot_link = os.path.join(kernel_parent_dir, "pxeboot")
                if os.path.islink(pxeboot_link):
                    os.unlink(pxeboot_link)
                if os.path.isdir(pxeboot_link):
                    logging.info("Removed old %s leftover directory", pxeboot_link)
                    shutil.rmtree(pxeboot_link)
                os.symlink(kernel_dir, pxeboot_link)

        elif params.get("medium") == "import":
            virt_install_cmd += add_import(help_text)

        if location:
            virt_install_cmd += add_location(help_text, location)

        if params.get("display") == "vnc":
            if params.get("vnc_autoport") == "yes":
                vm.vnc_autoport = True
            else:
                vm.vnc_autoport = False
            if not vm.vnc_autoport and params.get("vnc_port"):
                vm.vnc_port = int(params.get("vnc_port"))
            virt_install_cmd += add_vnc(help_text, vm.vnc_port)
            if params.get("vnclisten"):
                vm.vnclisten = params.get("vnclisten")
            virt_install_cmd += add_vnclisten(help_text, vm.vnclisten)
        elif params.get("display") == "sdl":
            virt_install_cmd += add_sdl(help_text)
        elif params.get("display") == "nographic":
            virt_install_cmd += add_nographic(help_text)

        video_device = params.get("video_device")
        if video_device:
            virt_install_cmd += add_video(help_text, video_device)

        sound_device = params.get("sound_device")
        if sound_device:
            virt_install_cmd += add_soundhw(help_text, sound_device)

        # if none is given a random UUID will be generated by libvirt
        if params.get("uuid"):
            virt_install_cmd += add_uuid(help_text, params.get("uuid"))

        # selectable OS type
        if params.get("use_os_type") == "yes":
            virt_install_cmd += add_os_type(help_text, params.get("os_type"))

        # selectable OS variant
        if params.get("use_os_variant") == "yes":
            virt_install_cmd += add_os_variant(help_text, params.get("os_variant"))

        # Add serial console
        virt_install_cmd += add_serial(help_text, self.get_serial_console_filename())

        # If the PCI assignment step went OK, add each one of the PCI assigned
        # devices to the command line.
        if self.pci_devices:
            for pci_id in self.pci_devices:
                virt_install_cmd += add_pcidevice(help_text, pci_id)

        for image_name in params.objects("images"):
            image_params = params.object_params(image_name)
            filename = storage.get_image_filename(image_params, data_dir.get_data_dir())
            if image_params.get("use_storage_pool") == "yes":
                filename = None
                virt_install_cmd += add_drive(
                    help_text,
                    filename,
                    image_params.get("image_pool"),
                    image_params.get("image_vol"),
                    image_params.get("image_device"),
                    image_params.get("image_bus"),
                    image_params.get("image_perms"),
                    image_params.get("image_size"),
                    image_params.get("drive_sparse"),
                    image_params.get("drive_cache"),
                    image_params.get("image_format"),
                )

            if image_params.get("boot_drive") == "no":
                continue
            if filename:
                virt_install_cmd += add_drive(
                    help_text,
                    filename,
                    None,
                    None,
                    None,
                    image_params.get("drive_format"),
                    None,
                    image_params.get("image_size"),
                    image_params.get("drive_sparse"),
                    image_params.get("drive_cache"),
                    image_params.get("image_format"),
                )

        if params.get("unattended_delivery_method") != "integrated" and not (
            self.driver_type == "xen" and params.get("hvm_or_pv") == "pv"
        ):
            for cdrom in params.objects("cdroms"):
                cdrom_params = params.object_params(cdrom)
                iso = cdrom_params.get("cdrom")
                if params.get("use_libvirt_cdrom_switch") == "yes":
                    # we don't want to skip the winutils iso
                    if not cdrom == "winutils":
                        logging.debug("Using --cdrom instead of --disk for install")
                        logging.debug("Skipping CDROM:%s:%s", cdrom, iso)
                        continue
                if params.get("medium") == "cdrom_no_kernel_initrd":
                    if iso == params.get("cdrom_cd1"):
                        logging.debug("Using cdrom or url for install")
                        logging.debug("Skipping CDROM: %s", iso)
                        continue

                if iso:
                    virt_install_cmd += add_drive(
                        help_text,
                        utils_misc.get_path(root_dir, iso),
                        image_params.get("iso_image_pool"),
                        image_params.get("iso_image_vol"),
                        "cdrom",
                        None,
                        None,
                        None,
                        None,
                        None,
                        None,
                    )

        # We may want to add {floppy_otps} parameter for -fda
        # {fat:floppy:}/path/. However vvfat is not usually recommended.
        # Only support to add the main floppy if you want to add the second
        # one please modify this part.
        floppy = params.get("floppy_name")
        if floppy:
            floppy = utils_misc.get_path(data_dir.get_data_dir(), floppy)
            virt_install_cmd += add_drive(help_text, floppy, None, None, "floppy", None, None, None, None, None, None)

        # setup networking parameters
        for nic in vm.virtnet:
            # make_create_command can be called w/o vm.create()
            nic = vm.add_nic(**dict(nic))
            logging.debug("make_create_command() setting up command for" " nic: %s" % str(nic))
            virt_install_cmd += add_nic(help_text, nic)

        if params.get("use_no_reboot") == "yes":
            virt_install_cmd += " --noreboot"

        if params.get("use_autostart") == "yes":
            virt_install_cmd += " --autostart"

        if params.get("virt_install_debug") == "yes":
            virt_install_cmd += " --debug"

        # bz still open, not fully functional yet
        if params.get("use_virt_install_wait") == "yes":
            virt_install_cmd += " --wait %s" % params.get("virt_install_wait_time")

        kernel_params = params.get("kernel_params")
        if kernel_params:
            virt_install_cmd += " --extra-args '%s'" % kernel_params

        virt_install_cmd += " --noautoconsole"

        return virt_install_cmd
Exemple #53
0
def bootstrap(test_name,
              test_dir,
              base_dir,
              default_userspace_paths,
              check_modules,
              online_docs_url,
              restore_image=False,
              interactive=True,
              selinux=False,
              verbose=False,
              update_providers=False,
              guest_os=defaults.DEFAULT_GUEST_OS,
              force_update=False):
    """
    Common virt test assistant module.

    :param test_name: Test name, such as "qemu".
    :param test_dir: Path with the test directory.
    :param base_dir: Base directory used to hold images and isos.
    :param default_userspace_paths: Important programs for a successful test
            execution.
    :param check_modules: Whether we want to verify if a given list of modules
            is loaded in the system.
    :param online_docs_url: URL to an online documentation system, such as a
            wiki page.
    :param restore_image: Whether to restore the image from the pristine.
    :param interactive: Whether to ask for confirmation.
    :param verbose: Verbose output.
    :param selinux: Whether setup SELinux contexts for shared/data.
    :param update_providers: Whether to update test providers if they are already
            downloaded.
    :param guest_os: Specify the guest image used for bootstrapping. By default
            the JeOS image is used.

    :raise error.CmdError: If JeOS image failed to uncompress
    :raise ValueError: If 7za was not found
    """
    if interactive:
        logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
                                          verbose=verbose)
    logging.info("%s test config helper", test_name)
    step = 0

    logging.info("")
    step += 1
    logging.info("%d - Updating all test providers", step)
    asset.download_all_test_providers(update_providers)

    logging.info("")
    step += 1
    logging.info("%d - Checking the mandatory programs and headers", step)
    verify_mandatory_programs(test_name, guest_os)

    logging.info("")
    step += 1
    logging.info("%d - Checking the recommended programs", step)
    verify_recommended_programs(test_name)

    logging.info("")
    step += 1
    logging.info("%d - Verifying directories", step)
    shared_dir = os.path.dirname(data_dir.get_data_dir())
    sub_dir_list = ["images", "isos", "steps_data", "gpg"]
    for sub_dir in sub_dir_list:
        sub_dir_path = os.path.join(base_dir, sub_dir)
        if not os.path.isdir(sub_dir_path):
            logging.debug("Creating %s", sub_dir_path)
            os.makedirs(sub_dir_path)
        else:
            logging.debug("Dir %s exists, not creating", sub_dir_path)

    datadir = data_dir.get_data_dir()
    if test_name == 'libvirt':
        create_config_files(test_dir, shared_dir, interactive, step,
                            force_update)
        create_subtests_cfg(test_name)
        create_guest_os_cfg(test_name)
        # Don't bother checking if changes can't be made
        if os.getuid() == 0:
            verify_selinux(datadir, os.path.join(datadir, 'images'),
                           os.path.join(datadir, 'isos'),
                           data_dir.get_tmp_dir(), interactive, selinux)

    # lvsb test doesn't use any shared configs
    elif test_name == 'lvsb':
        create_subtests_cfg(test_name)
        if os.getuid() == 0:
            # Don't bother checking if changes can't be made
            verify_selinux(datadir, os.path.join(datadir, 'images'),
                           os.path.join(datadir, 'isos'),
                           data_dir.get_tmp_dir(), interactive, selinux)
    else:  # Some other test
        create_config_files(test_dir, shared_dir, interactive, step,
                            force_update)
        create_subtests_cfg(test_name)
        create_guest_os_cfg(test_name)

    if restore_image:
        logging.info("")
        step += 1
        logging.info("%s - Verifying (and possibly downloading) guest image",
                     step)
        for os_info in get_guest_os_info_list(test_name, guest_os):
            os_asset = os_info['asset']
            asset.download_asset(os_asset,
                                 interactive=interactive,
                                 restore_image=restore_image)

    if check_modules:
        logging.info("")
        step += 1
        logging.info("%d - Checking for modules %s", step,
                     ", ".join(check_modules))
        for module in check_modules:
            if not utils.module_is_loaded(module):
                logging.warning(
                    "Module %s is not loaded. You might want to "
                    "load it", module)
            else:
                logging.debug("Module %s loaded", module)

    if online_docs_url:
        logging.info("")
        step += 1
        logging.info(
            "%d - If you wish, take a look at the online docs for "
            "more info", step)
        logging.info("")
        logging.info(online_docs_url)
Exemple #54
0
        except Exception, details:
            err += "\nKSM cleanup: %s" % str(details).replace('\\n', '\n  ')
            logging.error(details)

    # Execute any post_commands
    if params.get("post_command"):
        try:
            process_command(test, params, env, params.get("post_command"),
                            int(params.get("post_command_timeout", "600")),
                            params.get("post_command_noncritical") == "yes")
        except Exception, details:
            err += "\nPostprocess command: %s" % str(details).replace(
                '\n', '\n  ')
            logging.error(details)

    base_dir = data_dir.get_data_dir()
    if params.get("storage_type") == "iscsi":
        try:
            iscsidev = qemu_storage.Iscsidev(params, base_dir, "iscsi")
            iscsidev.cleanup()
        except Exception, details:
            err += "\niscsi cleanup: %s" % str(details).replace('\\n', '\n  ')
            logging.error(details)

    if params.get("storage_type") == "lvm":
        try:
            lvmdev = env.get_lvmdev("lvm_%s" % params["main_vm"])
            lvmdev.cleanup()
        except Exception, details:
            err += "\nLVM cleanup: %s" % str(details).replace('\\n', '\n  ')
            logging.error(details)
Exemple #55
0
def preprocess(test, params, env):
    """
    Preprocess all VMs and images according to the instructions in params.
    Also, collect some host information, such as the KVM version.

    :param test: An Autotest test object.
    :param params: A dict containing all VM and image parameters.
    :param env: The environment (a dict-like object).
    """
    error.context("preprocessing")
    # First, let's verify if this test does require root or not. If it
    # does and the test suite is running as a regular user, we shall just
    # throw a TestNAError exception, which will skip the test.
    if params.get('requires_root', 'no') == 'yes':
        utils_misc.verify_running_as_root()

    port = params.get('shell_port')
    prompt = params.get('shell_prompt')
    address = params.get('ovirt_node_address')
    username = params.get('ovirt_node_user')
    password = params.get('ovirt_node_password')

    setup_pb = False
    for nic in params.get('nics', "").split():
        nic_params = params.object_params(nic)
        if nic_params.get('netdst') == 'private':
            setup_pb = True
            params_pb = nic_params
            params['netdst_%s' % nic] = nic_params.get("priv_brname", 'atbr0')

    if setup_pb:
        brcfg = test_setup.PrivateBridgeConfig(params_pb)
        brcfg.setup()

    base_dir = data_dir.get_data_dir()
    if params.get("storage_type") == "iscsi":
        iscsidev = qemu_storage.Iscsidev(params, base_dir, "iscsi")
        params["image_name"] = iscsidev.setup()
        params["image_raw_device"] = "yes"

    if params.get("storage_type") == "lvm":
        lvmdev = qemu_storage.LVMdev(params, base_dir, "lvm")
        params["image_name"] = lvmdev.setup()
        params["image_raw_device"] = "yes"
        env.register_lvmdev("lvm_%s" % params["main_vm"], lvmdev)

    if params.get("storage_type") == "nfs":
        image_nfs = nfs.Nfs(params)
        image_nfs.setup()
        image_name_only = os.path.basename(params["image_name"])
        params['image_name'] = os.path.join(image_nfs.mount_dir,
                                            image_name_only)
        for image_name in params.objects("images"):
            name_tag = "image_name_%s" % image_name
            if params.get(name_tag):
                image_name_only = os.path.basename(params[name_tag])
                params[name_tag] = os.path.join(image_nfs.mount_dir,
                                                image_name_only)

    # Start tcpdump if it isn't already running
    # The fact it has to be started here is so that the test params
    # have to be honored.
    env.start_tcpdump(params)

    # Destroy and remove VMs that are no longer needed in the environment
    requested_vms = params.objects("vms")
    for key in env.keys():
        vm = env[key]
        if not isinstance(vm, virt_vm.BaseVM):
            continue
        if vm.name not in requested_vms:
            vm.destroy()
            del env[key]

    if (params.get("auto_cpu_model") == "yes"
            and params.get("vm_type") == "qemu"):
        if not env.get("cpu_model"):
            env["cpu_model"] = utils_misc.get_qemu_best_cpu_model(params)
        params["cpu_model"] = env.get("cpu_model")

    kvm_ver_cmd = params.get("kvm_ver_cmd", "")

    if kvm_ver_cmd:
        try:
            cmd_result = utils.run(kvm_ver_cmd)
            kvm_version = cmd_result.stdout.strip()
        except error.CmdError:
            kvm_version = "Unknown"
    else:
        # Get the KVM kernel module version and write it as a keyval
        if os.path.exists("/dev/kvm"):
            try:
                kvm_version = open("/sys/module/kvm/version").read().strip()
            except Exception:
                kvm_version = os.uname()[2]
        else:
            logging.warning("KVM module not loaded")
            kvm_version = "Unknown"

    logging.debug("KVM version: %s" % kvm_version)
    test.write_test_keyval({"kvm_version": kvm_version})

    # Get the KVM userspace version and write it as a keyval
    kvm_userspace_ver_cmd = params.get("kvm_userspace_ver_cmd", "")

    if kvm_userspace_ver_cmd:
        try:
            cmd_result = utils.run(kvm_userspace_ver_cmd)
            kvm_userspace_version = cmd_result.stdout.strip()
        except error.CmdError:
            kvm_userspace_version = "Unknown"
    else:
        qemu_path = utils_misc.get_qemu_binary(params)
        version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
        matches = re.findall("[Vv]ersion .*?,", version_line)
        if matches:
            kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
        else:
            kvm_userspace_version = "Unknown"

    logging.debug("KVM userspace version: %s" % kvm_userspace_version)
    test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})

    libvirtd_inst = utils_libvirtd.Libvirtd()

    if params.get("setup_hugepages") == "yes":
        h = test_setup.HugePageConfig(params)
        suggest_mem = h.setup()
        if suggest_mem is not None:
            params['mem'] = suggest_mem
        if params.get("vm_type") == "libvirt":
            libvirtd_inst.restart()

    if params.get("setup_thp") == "yes":
        thp = test_setup.TransparentHugePageConfig(test, params)
        thp.setup()

    if params.get("setup_ksm") == "yes":
        ksm = test_setup.KSMConfig(params, env)
        ksm.setup(env)

    if params.get("vm_type") == "libvirt":
        if params.get("setup_libvirt_polkit") == "yes":
            pol = test_setup.LibvirtPolkitConfig(params)
            try:
                pol.setup()
            except test_setup.PolkitWriteLibvirtdConfigError, e:
                logging.error("e")
            except test_setup.PolkitRulesSetupError, e:
                logging.error("e")
            except Exception, e:
                logging.error("Unexpected error:" % e)