def create_config_files(test_dir, shared_dir, interactive, step=None, force_update=False): def is_file_tracked(fl): tracked_result = utils.run("git ls-files %s --error-unmatch" % fl, ignore_status=True, verbose=False) return (tracked_result.exit_status == 0) if step is None: step = 0 logging.info("") step += 1 logging.info("%d - Generating config set", step) config_file_list = data_dir.SubdirGlobList(os.path.join(test_dir, "cfg"), "*.cfg", config_filter) config_file_list = [cf for cf in config_file_list if is_file_tracked(cf)] config_file_list_shared = glob.glob(os.path.join(shared_dir, "cfg", "*.cfg")) # Handle overrides of cfg files. Let's say a test provides its own # subtest.cfg.sample, this file takes precedence over the shared # subtest.cfg.sample. So, yank this file from the cfg file list. config_file_list_shared_keep = [] for cf in config_file_list_shared: basename = os.path.basename(cf) target = os.path.join(test_dir, "cfg", basename) if target not in config_file_list: config_file_list_shared_keep.append(cf) config_file_list += config_file_list_shared_keep for config_file in config_file_list: src_file = config_file dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file)) if not os.path.isfile(dst_file): logging.debug("Creating config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: diff_cmd = "diff -Naur %s %s" % (dst_file, src_file) diff_result = utils.run( diff_cmd, ignore_status=True, verbose=False) if diff_result.exit_status != 0: logging.info("%s result:\n %s", diff_result.command, diff_result.stdout) if interactive: answer = utils.ask("Config file %s differs from %s." "Overwrite?" % (dst_file, src_file)) elif force_update: answer = "y" else: answer = "n" if answer == "y": logging.debug("Restoring config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: logging.debug("Preserving existing %s file", dst_file) else: logging.debug("Config file %s exists, not touching", dst_file)
def download_asset(asset, interactive=True, restore_image=False): """ Download an asset defined on an asset file. Asset files are located under /shared/downloads, are .ini files with the following keys defined: title: Title string to display in the download progress bar. url = URL of the resource sha1_url = URL with SHA1 information for the resource, in the form sha1sum file_basename destination = Location of your file relative to the data directory (TEST_SUITE_ROOT/shared/data) destination = Location of the uncompressed file relative to the data directory (TEST_SUITE_ROOT/shared/data) uncompress_cmd = Command that needs to be executed with the compressed file as a parameter :param asset: String describing an asset file. :param interactive: Whether to ask the user before downloading the file. :param restore_image: If the asset is a compressed image, we can uncompress in order to restore the image. """ asset_info = get_asset_info(asset) destination = asset_info['destination'] if (interactive and not os.path.isfile(destination)): answer = utils.ask("File %s not present. Do you want to download it?" % asset_info['title']) else: answer = "y" if answer == "y": download_file(asset_info=asset_info, interactive=interactive, force=restore_image)
def create_config_files(test_dir, shared_dir, interactive, step=None): if step is None: step = 0 logging.info("") step += 1 logging.info("%d - Creating config files from samples", step) config_file_list = glob.glob(os.path.join(test_dir, "cfg", "*.cfg.sample")) config_file_list_shared = glob.glob(os.path.join(shared_dir, "*.cfg.sample")) # Handle overrides of cfg files. Let's say a test provides its own # subtest.cfg.sample, this file takes precedence over the shared # subtest.cfg.sample. So, yank this file from the cfg file list. idx = 0 for cf in config_file_list_shared: basename = os.path.basename(cf) target = os.path.join(test_dir, "cfg", basename) if target in config_file_list: config_file_list_shared.pop(idx) idx += 1 config_file_list += config_file_list_shared for config_file in config_file_list: src_file = config_file dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file)) dst_file = dst_file.rstrip(".sample") if not os.path.isfile(dst_file): logging.debug("Creating config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: diff_result = utils.run("diff -Naur %s %s" % (dst_file, src_file), ignore_status=True, verbose=False) if diff_result.exit_status != 0: logging.info("%s result:\n %s" % (diff_result.command, diff_result.stdout)) if interactive: answer = utils.ask("Config file %s differs from %s." "Overwrite?" % (dst_file,src_file)) else: answer = "n" if answer == "y": logging.debug("Restoring config file %s from sample" % dst_file) shutil.copyfile(src_file, dst_file) else: logging.debug("Preserving existing %s file" % dst_file) else: logging.debug("Config file %s exists, not touching" % dst_file)
def download_asset(asset, interactive=True, restore_image=False): """ Download an asset defined on an asset file. Asset files are located under /shared/download.d, are .ini files with the following keys defined: title: Title string to display in the download progress bar. url = URL of the resource sha1_url = URL with SHA1 information for the resource, in the form sha1sum file_basename destination = Location of your file relative to the data directory (TEST_SUITE_ROOT/shared/data) destination = Location of the uncompressed file relative to the data directory (TEST_SUITE_ROOT/shared/data) uncompress_cmd = Command that needs to be executed with the compressed file as a parameter @param asset: String describing an asset file. @param interactive: Whether to ask the user before downloading the file. @param restore_image: If the asset is a compressed image, we can uncompress in order to restore the image. """ asset_info = get_asset_info(asset) destination = os.path.join(data_dir.get_data_dir(), asset_info['destination']) if (interactive and not os.path.isfile(destination)): answer = utils.ask("File %s not present. Do you want to download it?" % asset_info['title']) else: answer = "y" if answer == "y": had_to_download = download_file(asset=asset, interactive=interactive) requires_uncompress = asset_info['uncompress_cmd'] is not None if requires_uncompress: destination_uncompressed = asset_info['destination_uncompressed'] uncompressed_file_exists = os.path.exists(destination_uncompressed) restore_image = (restore_image or had_to_download or not uncompressed_file_exists) if os.path.isfile(destination) and restore_image: os.chdir(os.path.dirname(destination)) uncompress_cmd = asset_info['uncompress_cmd'] utils.run("%s %s" % (uncompress_cmd, destination))
sha1_contents = sha1_file.read() sha1 = sha1_contents.split(" ")[0] logging.info("Expected SHA1 sum: %s", sha1) except Exception, e: logging.error("Failed to get SHA1 from file: %s", e) else: sha1 = None destination_dir = os.path.dirname(destination) if not os.path.isdir(destination_dir): os.makedirs(destination_dir) if not os.path.isfile(destination): logging.warning("File %s not found", destination) if interactive: answer = utils.ask("Would you like to download it from %s?" % url) else: answer = 'y' if answer == 'y': utils.interactive_download( url, destination, "Downloading %s" % title) had_to_download = True else: logging.warning("Missing file %s", destination) else: logging.info("Found %s", destination) if sha1 is None: answer = 'n' else: answer = 'y'
def verify_selinux(datadir, imagesdir, isosdir, tmpdir, interactive, selinux=False): """ Verify/Set/Warn about SELinux and default file contexts for testing. :param datadir: Abs. path to data-directory symlink :param imagesdir: Abs. path to data/images directory :param isosdir: Abs. path to data/isos directory :param tmpdir: Abs. path to virt-test tmp dir :param interactive: True if running from console :param selinux: Whether setup SELinux contexts for shared/data """ # datadir can be a symlink, but these must not have any imagesdir = os.path.realpath(imagesdir) isosdir = os.path.realpath(isosdir) tmpdir = os.path.realpath(tmpdir) needs_relabel = None try: # Raise SeCmdError if selinux not installed if utils_selinux.get_status() == 'enforcing': # Check if default contexts are set if not haz_defcon(datadir, imagesdir, isosdir, tmpdir): if selinux: answer = "y" else: if interactive: answer = utils.ask("Setup all undefined default SE" "Linux contexts for shared/data/?") else: answer = "n" else: answer = "n" if answer.lower() == "y": # Assume relabeling is needed if changes made needs_relabel = set_defcon(datadir, imagesdir, isosdir, tmpdir) # Only relabel if files/dirs don't match default labels_ok = utils_selinux.verify_defcon(datadir, False) labels_ok &= utils_selinux.verify_defcon(imagesdir, True) labels_ok &= utils_selinux.verify_defcon(isosdir, True) labels_ok &= utils_selinux.verify_defcon(tmpdir, True) if labels_ok: needs_relabel = False else: logging.warning("On-disk SELinux labels do not match defaults") needs_relabel = True # Disabled or Permissive mode is same result as not installed else: logging.info("SELinux in permissive or disabled, testing" "in enforcing mode is highly encourraged.") except utils_selinux.SemanageError: logging.info("Could not set default SELinux contexts. Please") logging.info("consider installing the semanage program then ") logging.info("verifying and/or running running:") # Paths must be transmogrified (changed) into regular expressions logging.info("semanage fcontext --add -t virt_var_lib_t '%s'", utils_selinux.transmogrify_usr_local(datadir)) logging.info("semanage fcontext --add -t virt_image_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(imagesdir))) logging.info("semanage fcontext --add -t virt_content_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(isosdir))) logging.info("semanage fcontext --add -t user_tmp_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(tmpdir))) needs_relabel = None # Next run will catch if relabeling needed except utils_selinux.SelinuxError: # Catchall SELinux related logging.info("SELinux not available, or error in command/setup.") logging.info("Please manually verify default file contexts before") logging.info("testing with SELinux enabled and enforcing.") if needs_relabel: if selinux: answer = "y" else: if interactive: answer = utils.ask("Relabel from default contexts?") else: answer = "n" if answer.lower() == 'y': changes = utils_selinux.apply_defcon(datadir, False) changes += utils_selinux.apply_defcon(imagesdir, True) changes += utils_selinux.apply_defcon(isosdir, True) changes += utils_selinux.apply_defcon(tmpdir, True) logging.info("Corrected contexts on %d files/dirs", len(changes))
def bootstrap(test_name, test_dir, base_dir, default_userspace_paths, check_modules, online_docs_url, restore_image=False, interactive=True, verbose=False): """ Common virt test assistant module. @param test_name: Test name, such as "kvm". @param test_dir: Path with the test directory. @param base_dir: Base directory used to hold images and isos. @param default_userspace_paths: Important programs for a successful test execution. @param check_modules: Whether we want to verify if a given list of modules is loaded in the system. @param online_docs_url: URL to an online documentation system, such as a wiki page. @param restore_image: Whether to restore the image from the pristine. @param interactive: Whether to ask for confirmation. @raise error.CmdError: If JeOS image failed to uncompress @raise ValueError: If 7za was not found """ if interactive: logging_manager.configure_logging(utils_misc.VirtLoggingConfig(), verbose=verbose) logging.info("%s test config helper", test_name) step = 0 logging.info("") step += 1 logging.info("%d - Checking the mandatory programs and headers", step) verify_mandatory_programs(test_name) logging.info("") step += 1 logging.info("%d - Checking the recommended programs", step) verify_recommended_programs(test_name) logging.info("") step += 1 logging.info("%d - Verifying directories", step) shared_dir = os.path.dirname(data_dir.get_data_dir()) shared_dir = os.path.join(shared_dir, "cfg") sub_dir_list = ["images", "isos", "steps_data"] for sub_dir in sub_dir_list: sub_dir_path = os.path.join(base_dir, sub_dir) if not os.path.isdir(sub_dir_path): logging.debug("Creating %s", sub_dir_path) os.makedirs(sub_dir_path) else: logging.debug("Dir %s exists, not creating" % sub_dir_path) create_config_files(test_dir, shared_dir, interactive, step) logging.info("") step += 2 logging.info("%s - Verifying (and possibly downloading) guest image", step) sha1_file = "SHA1SUM" guest_tarball = "jeos-17-64.qcow2.7z" base_location = "http://lmr.fedorapeople.org/jeos/" url = os.path.join(base_location, guest_tarball) tarball_sha1_url = os.path.join(base_location, sha1_file) destination = os.path.join(base_dir, 'images') uncompressed_file_path = os.path.join(base_dir, 'images', 'jeos-17-64.qcow2') uncompressed_file_exists = os.path.isfile(uncompressed_file_path) if (interactive and not os.path.isfile(os.path.join(destination, guest_tarball))): answer = utils.ask("Minimal basic guest image (JeOS) not present. " "Do you want to download it (~ 180MB)?") else: answer = "y" if answer == "y": had_to_download = download_file(url, destination, tarball_sha1_url, title="Downloading JeOS x86_64", interactive=interactive) restore_image = (restore_image or had_to_download or not uncompressed_file_exists) tarball_path = os.path.join(destination, guest_tarball) if os.path.isfile(tarball_path) and restore_image: os.chdir(destination) utils.run("7za -y e %s" % tarball_path) if check_modules: logging.info("") step += 1 logging.info("%d - Checking for modules %s", step, ", ".join(check_modules)) for module in check_modules: if not utils.module_is_loaded(module): logging.warning("Module %s is not loaded. You might want to " "load it", module) else: logging.debug("Module %s loaded", module) if online_docs_url: logging.info("") step += 1 logging.info("%d - If you wish, take a look at the online docs for " "more info", step) logging.info("") logging.info(online_docs_url)
logging.info("Verifying expected SHA1 sum from %s", sha1_url) sha1_file = urllib2.urlopen(sha1_url) sha1_contents = sha1_file.read() sha1 = sha1_contents.split(" ")[0] logging.info("Expected SHA1 sum: %s", sha1) except Exception, e: logging.error("Failed to get SHA1 from file: %s", e) if not os.path.isdir(destination): os.makedirs(destination) path = os.path.join(destination, os.path.basename(url)) if not os.path.isfile(path): logging.warning("File %s not found", path) if interactive: answer = utils.ask("Would you like to download it from %s?" % url) else: answer = 'y' if answer == 'y': utils.interactive_download(url, path, "JeOS x86_64 image") had_to_download = True else: logging.warning("Missing file %s", path) else: logging.info("Found %s", path) if sha1 is None: answer = 'n' else: answer = 'y' if answer == 'y':
def create_config_files(test_dir, shared_dir, interactive, step=None, force_update=False): def is_file_tracked(fl): tracked_result = utils.run("git ls-files %s --error-unmatch" % fl, ignore_status=True, verbose=False) return (tracked_result.exit_status == 0) if step is None: step = 0 logging.info("") step += 1 logging.info("%d - Generating config set", step) config_file_list = data_dir.SubdirGlobList(os.path.join(test_dir, "cfg"), "*.cfg", config_filter) config_file_list = [cf for cf in config_file_list if is_file_tracked(cf)] config_file_list_shared = glob.glob( os.path.join(shared_dir, "cfg", "*.cfg")) # Handle overrides of cfg files. Let's say a test provides its own # subtest.cfg.sample, this file takes precedence over the shared # subtest.cfg.sample. So, yank this file from the cfg file list. config_file_list_shared_keep = [] for cf in config_file_list_shared: basename = os.path.basename(cf) target = os.path.join(test_dir, "cfg", basename) if target not in config_file_list: config_file_list_shared_keep.append(cf) config_file_list += config_file_list_shared_keep for config_file in config_file_list: src_file = config_file dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file)) if not os.path.isfile(dst_file): logging.debug("Creating config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: diff_cmd = "diff -Naur %s %s" % (dst_file, src_file) diff_result = utils.run(diff_cmd, ignore_status=True, verbose=False) if diff_result.exit_status != 0: logging.info("%s result:\n %s", diff_result.command, diff_result.stdout) if interactive: answer = utils.ask("Config file %s differs from %s." "Overwrite?" % (dst_file, src_file)) elif force_update: answer = "y" else: answer = "n" if answer == "y": logging.debug("Restoring config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: logging.debug("Preserving existing %s file", dst_file) else: logging.debug("Config file %s exists, not touching", dst_file)
def verify_selinux(datadir, imagesdir, isosdir, tmpdir, interactive, selinux=False): """ Verify/Set/Warn about SELinux and default file contexts for testing. :param datadir: Abs. path to data-directory symlink :param imagesdir: Abs. path to data/images directory :param isosdir: Abs. path to data/isos directory :param tmpdir: Abs. path to virt-test tmp dir :param interactive: True if running from console :param selinux: Whether setup SELinux contexts for shared/data """ # datadir can be a symlink, but these must not have any imagesdir = os.path.realpath(imagesdir) isosdir = os.path.realpath(isosdir) tmpdir = os.path.realpath(tmpdir) needs_relabel = None try: # Raise SeCmdError if selinux not installed if utils_selinux.get_status() == 'enforcing': # Check if default contexts are set if not haz_defcon(datadir, imagesdir, isosdir, tmpdir): if selinux: answer = "y" else: if interactive: answer = utils.ask("Setup all undefined default SE" "Linux contexts for shared/data/?") else: answer = "n" else: answer = "n" if answer.lower() == "y": # Assume relabeling is needed if changes made needs_relabel = set_defcon(datadir, imagesdir, isosdir, tmpdir) # Only relabel if files/dirs don't match default labels_ok = utils_selinux.verify_defcon(datadir, False) labels_ok &= utils_selinux.verify_defcon(imagesdir, True) labels_ok &= utils_selinux.verify_defcon(isosdir, True) labels_ok &= utils_selinux.verify_defcon(tmpdir, True) if labels_ok: needs_relabel = False else: logging.warning("On-disk SELinux labels do not match defaults") needs_relabel = True # Disabled or Permissive mode is same result as not installed else: logging.info("SELinux in permissive or disabled, testing" "in enforcing mode is highly encourraged.") except utils_selinux.SemanageError: logging.info("Could not set default SELinux contexts. Please") logging.info("consider installing the semanage program then ") logging.info("verifying and/or running running:") # Paths must be transmogrified (changed) into regular expressions logging.info("semanage fcontext --add -t virt_var_lib_t '%s'", utils_selinux.transmogrify_usr_local(datadir)) logging.info( "semanage fcontext --add -t virt_image_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(imagesdir))) logging.info( "semanage fcontext --add -t virt_content_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(isosdir))) logging.info( "semanage fcontext --add -t user_tmp_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(tmpdir))) needs_relabel = None # Next run will catch if relabeling needed except utils_selinux.SelinuxError: # Catchall SELinux related logging.info("SELinux not available, or error in command/setup.") logging.info("Please manually verify default file contexts before") logging.info("testing with SELinux enabled and enforcing.") if needs_relabel: if selinux: answer = "y" else: if interactive: answer = utils.ask("Relabel from default contexts?") else: answer = "n" if answer.lower() == 'y': changes = utils_selinux.apply_defcon(datadir, False) changes += utils_selinux.apply_defcon(imagesdir, True) changes += utils_selinux.apply_defcon(isosdir, True) changes += utils_selinux.apply_defcon(tmpdir, True) logging.info("Corrected contexts on %d files/dirs", len(changes))