def create_config_files(test_dir, shared_dir, interactive, step=None,
force_update=False):
def is_file_tracked(fl):
tracked_result = utils.run("git ls-files %s --error-unmatch" % fl,
ignore_status=True, verbose=False)
return (tracked_result.exit_status == 0)
if step is None:
step = 0
logging.info("")
step += 1
logging.info("%d - Generating config set", step)
config_file_list = data_dir.SubdirGlobList(os.path.join(test_dir, "cfg"),
"*.cfg",
config_filter)
config_file_list = [cf for cf in config_file_list if is_file_tracked(cf)]
config_file_list_shared = glob.glob(os.path.join(shared_dir, "cfg",
"*.cfg"))
# Handle overrides of cfg files. Let's say a test provides its own
# subtest.cfg.sample, this file takes precedence over the shared
# subtest.cfg.sample. So, yank this file from the cfg file list.
config_file_list_shared_keep = []
for cf in config_file_list_shared:
basename = os.path.basename(cf)
target = os.path.join(test_dir, "cfg", basename)
if target not in config_file_list:
config_file_list_shared_keep.append(cf)
config_file_list += config_file_list_shared_keep
for config_file in config_file_list:
src_file = config_file
dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file))
if not os.path.isfile(dst_file):
logging.debug("Creating config file %s from sample", dst_file)
shutil.copyfile(src_file, dst_file)
else:
diff_cmd = "diff -Naur %s %s" % (dst_file, src_file)
diff_result = utils.run(
diff_cmd, ignore_status=True, verbose=False)
if diff_result.exit_status != 0:
logging.info("%s result:\n %s",
diff_result.command, diff_result.stdout)
if interactive:
answer = utils.ask("Config file %s differs from %s."
"Overwrite?" % (dst_file, src_file))
elif force_update:
answer = "y"
else:
answer = "n"
if answer == "y":
logging.debug("Restoring config file %s from sample",
dst_file)
shutil.copyfile(src_file, dst_file)
else:
logging.debug("Preserving existing %s file", dst_file)
else:
logging.debug("Config file %s exists, not touching", dst_file)
def download_asset(asset, interactive=True, restore_image=False):
"""
Download an asset defined on an asset file.
Asset files are located under /shared/downloads, are .ini files with the
following keys defined:
title: Title string to display in the download progress bar.
url = URL of the resource
sha1_url = URL with SHA1 information for the resource, in the form
sha1sum file_basename
destination = Location of your file relative to the data directory
(TEST_SUITE_ROOT/shared/data)
destination = Location of the uncompressed file relative to the data
directory (TEST_SUITE_ROOT/shared/data)
uncompress_cmd = Command that needs to be executed with the compressed
file as a parameter
:param asset: String describing an asset file.
:param interactive: Whether to ask the user before downloading the file.
:param restore_image: If the asset is a compressed image, we can uncompress
in order to restore the image.
"""
asset_info = get_asset_info(asset)
destination = asset_info['destination']
if (interactive and not os.path.isfile(destination)):
answer = utils.ask("File %s not present. Do you want to download it?" %
asset_info['title'])
else:
answer = "y"
if answer == "y":
download_file(asset_info=asset_info, interactive=interactive,
force=restore_image)
def create_config_files(test_dir, shared_dir, interactive, step=None):
if step is None:
step = 0
logging.info("")
step += 1
logging.info("%d - Creating config files from samples", step)
config_file_list = glob.glob(os.path.join(test_dir, "cfg", "*.cfg.sample"))
config_file_list_shared = glob.glob(os.path.join(shared_dir,
"*.cfg.sample"))
# Handle overrides of cfg files. Let's say a test provides its own
# subtest.cfg.sample, this file takes precedence over the shared
# subtest.cfg.sample. So, yank this file from the cfg file list.
idx = 0
for cf in config_file_list_shared:
basename = os.path.basename(cf)
target = os.path.join(test_dir, "cfg", basename)
if target in config_file_list:
config_file_list_shared.pop(idx)
idx += 1
config_file_list += config_file_list_shared
for config_file in config_file_list:
src_file = config_file
dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file))
dst_file = dst_file.rstrip(".sample")
if not os.path.isfile(dst_file):
logging.debug("Creating config file %s from sample", dst_file)
shutil.copyfile(src_file, dst_file)
else:
diff_result = utils.run("diff -Naur %s %s" % (dst_file, src_file),
ignore_status=True, verbose=False)
if diff_result.exit_status != 0:
logging.info("%s result:\n %s" %
(diff_result.command, diff_result.stdout))
if interactive:
answer = utils.ask("Config file %s differs from %s."
"Overwrite?" % (dst_file,src_file))
else:
answer = "n"
if answer == "y":
logging.debug("Restoring config file %s from sample" %
dst_file)
shutil.copyfile(src_file, dst_file)
else:
logging.debug("Preserving existing %s file" % dst_file)
else:
logging.debug("Config file %s exists, not touching" % dst_file)
def download_asset(asset, interactive=True, restore_image=False):
"""
Download an asset defined on an asset file.
Asset files are located under /shared/download.d, are .ini files with the
following keys defined:
title: Title string to display in the download progress bar.
url = URL of the resource
sha1_url = URL with SHA1 information for the resource, in the form
sha1sum file_basename
destination = Location of your file relative to the data directory
(TEST_SUITE_ROOT/shared/data)
destination = Location of the uncompressed file relative to the data
directory (TEST_SUITE_ROOT/shared/data)
uncompress_cmd = Command that needs to be executed with the compressed
file as a parameter
@param asset: String describing an asset file.
@param interactive: Whether to ask the user before downloading the file.
@param restore_image: If the asset is a compressed image, we can uncompress
in order to restore the image.
"""
asset_info = get_asset_info(asset)
destination = os.path.join(data_dir.get_data_dir(),
asset_info['destination'])
if (interactive and not os.path.isfile(destination)):
answer = utils.ask("File %s not present. Do you want to download it?" %
asset_info['title'])
else:
answer = "y"
if answer == "y":
had_to_download = download_file(asset=asset, interactive=interactive)
requires_uncompress = asset_info['uncompress_cmd'] is not None
if requires_uncompress:
destination_uncompressed = asset_info['destination_uncompressed']
uncompressed_file_exists = os.path.exists(destination_uncompressed)
restore_image = (restore_image or had_to_download or not
uncompressed_file_exists)
if os.path.isfile(destination) and restore_image:
os.chdir(os.path.dirname(destination))
uncompress_cmd = asset_info['uncompress_cmd']
utils.run("%s %s" % (uncompress_cmd, destination))
sha1_contents = sha1_file.read()
sha1 = sha1_contents.split(" ")[0]
logging.info("Expected SHA1 sum: %s", sha1)
except Exception, e:
logging.error("Failed to get SHA1 from file: %s", e)
else:
sha1 = None
destination_dir = os.path.dirname(destination)
if not os.path.isdir(destination_dir):
os.makedirs(destination_dir)
if not os.path.isfile(destination):
logging.warning("File %s not found", destination)
if interactive:
answer = utils.ask("Would you like to download it from %s?" % url)
else:
answer = 'y'
if answer == 'y':
utils.interactive_download(
url, destination, "Downloading %s" % title)
had_to_download = True
else:
logging.warning("Missing file %s", destination)
else:
logging.info("Found %s", destination)
if sha1 is None:
answer = 'n'
else:
answer = 'y'
def verify_selinux(datadir, imagesdir, isosdir, tmpdir,
interactive, selinux=False):
"""
Verify/Set/Warn about SELinux and default file contexts for testing.
:param datadir: Abs. path to data-directory symlink
:param imagesdir: Abs. path to data/images directory
:param isosdir: Abs. path to data/isos directory
:param tmpdir: Abs. path to virt-test tmp dir
:param interactive: True if running from console
:param selinux: Whether setup SELinux contexts for shared/data
"""
# datadir can be a symlink, but these must not have any
imagesdir = os.path.realpath(imagesdir)
isosdir = os.path.realpath(isosdir)
tmpdir = os.path.realpath(tmpdir)
needs_relabel = None
try:
# Raise SeCmdError if selinux not installed
if utils_selinux.get_status() == 'enforcing':
# Check if default contexts are set
if not haz_defcon(datadir, imagesdir, isosdir, tmpdir):
if selinux:
answer = "y"
else:
if interactive:
answer = utils.ask("Setup all undefined default SE"
"Linux contexts for shared/data/?")
else:
answer = "n"
else:
answer = "n"
if answer.lower() == "y":
# Assume relabeling is needed if changes made
needs_relabel = set_defcon(datadir, imagesdir, isosdir, tmpdir)
# Only relabel if files/dirs don't match default
labels_ok = utils_selinux.verify_defcon(datadir, False)
labels_ok &= utils_selinux.verify_defcon(imagesdir, True)
labels_ok &= utils_selinux.verify_defcon(isosdir, True)
labels_ok &= utils_selinux.verify_defcon(tmpdir, True)
if labels_ok:
needs_relabel = False
else:
logging.warning("On-disk SELinux labels do not match defaults")
needs_relabel = True
# Disabled or Permissive mode is same result as not installed
else:
logging.info("SELinux in permissive or disabled, testing"
"in enforcing mode is highly encourraged.")
except utils_selinux.SemanageError:
logging.info("Could not set default SELinux contexts. Please")
logging.info("consider installing the semanage program then ")
logging.info("verifying and/or running running:")
# Paths must be transmogrified (changed) into regular expressions
logging.info("semanage fcontext --add -t virt_var_lib_t '%s'",
utils_selinux.transmogrify_usr_local(datadir))
logging.info("semanage fcontext --add -t virt_image_t '%s'",
utils_selinux.transmogrify_usr_local(
utils_selinux.transmogrify_sub_dirs(imagesdir)))
logging.info("semanage fcontext --add -t virt_content_t '%s'",
utils_selinux.transmogrify_usr_local(
utils_selinux.transmogrify_sub_dirs(isosdir)))
logging.info("semanage fcontext --add -t user_tmp_t '%s'",
utils_selinux.transmogrify_usr_local(
utils_selinux.transmogrify_sub_dirs(tmpdir)))
needs_relabel = None # Next run will catch if relabeling needed
except utils_selinux.SelinuxError: # Catchall SELinux related
logging.info("SELinux not available, or error in command/setup.")
logging.info("Please manually verify default file contexts before")
logging.info("testing with SELinux enabled and enforcing.")
if needs_relabel:
if selinux:
answer = "y"
else:
if interactive:
answer = utils.ask("Relabel from default contexts?")
else:
answer = "n"
if answer.lower() == 'y':
changes = utils_selinux.apply_defcon(datadir, False)
changes += utils_selinux.apply_defcon(imagesdir, True)
changes += utils_selinux.apply_defcon(isosdir, True)
changes += utils_selinux.apply_defcon(tmpdir, True)
logging.info("Corrected contexts on %d files/dirs",
len(changes))
def bootstrap(test_name, test_dir, base_dir, default_userspace_paths,
check_modules, online_docs_url, restore_image=False,
interactive=True, verbose=False):
"""
Common virt test assistant module.
@param test_name: Test name, such as "kvm".
@param test_dir: Path with the test directory.
@param base_dir: Base directory used to hold images and isos.
@param default_userspace_paths: Important programs for a successful test
execution.
@param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
@param online_docs_url: URL to an online documentation system, such as a
wiki page.
@param restore_image: Whether to restore the image from the pristine.
@param interactive: Whether to ask for confirmation.
@raise error.CmdError: If JeOS image failed to uncompress
@raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
shared_dir = os.path.join(shared_dir, "cfg")
sub_dir_list = ["images", "isos", "steps_data"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating" %
sub_dir_path)
create_config_files(test_dir, shared_dir, interactive, step)
logging.info("")
step += 2
logging.info("%s - Verifying (and possibly downloading) guest image", step)
sha1_file = "SHA1SUM"
guest_tarball = "jeos-17-64.qcow2.7z"
base_location = "http://lmr.fedorapeople.org/jeos/"
url = os.path.join(base_location, guest_tarball)
tarball_sha1_url = os.path.join(base_location, sha1_file)
destination = os.path.join(base_dir, 'images')
uncompressed_file_path = os.path.join(base_dir, 'images',
'jeos-17-64.qcow2')
uncompressed_file_exists = os.path.isfile(uncompressed_file_path)
if (interactive and not
os.path.isfile(os.path.join(destination, guest_tarball))):
answer = utils.ask("Minimal basic guest image (JeOS) not present. "
"Do you want to download it (~ 180MB)?")
else:
answer = "y"
if answer == "y":
had_to_download = download_file(url, destination, tarball_sha1_url,
title="Downloading JeOS x86_64",
interactive=interactive)
restore_image = (restore_image or had_to_download or not
uncompressed_file_exists)
tarball_path = os.path.join(destination, guest_tarball)
if os.path.isfile(tarball_path) and restore_image:
os.chdir(destination)
utils.run("7za -y e %s" % tarball_path)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning("Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info("%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
logging.info("Verifying expected SHA1 sum from %s", sha1_url)
sha1_file = urllib2.urlopen(sha1_url)
sha1_contents = sha1_file.read()
sha1 = sha1_contents.split(" ")[0]
logging.info("Expected SHA1 sum: %s", sha1)
except Exception, e:
logging.error("Failed to get SHA1 from file: %s", e)
if not os.path.isdir(destination):
os.makedirs(destination)
path = os.path.join(destination, os.path.basename(url))
if not os.path.isfile(path):
logging.warning("File %s not found", path)
if interactive:
answer = utils.ask("Would you like to download it from %s?" % url)
else:
answer = 'y'
if answer == 'y':
utils.interactive_download(url, path, "JeOS x86_64 image")
had_to_download = True
else:
logging.warning("Missing file %s", path)
else:
logging.info("Found %s", path)
if sha1 is None:
answer = 'n'
else:
answer = 'y'
if answer == 'y':
sha1_contents = sha1_file.read()
sha1 = sha1_contents.split(" ")[0]
logging.info("Expected SHA1 sum: %s", sha1)
except Exception, e:
logging.error("Failed to get SHA1 from file: %s", e)
else:
sha1 = None
destination_dir = os.path.dirname(destination)
if not os.path.isdir(destination_dir):
os.makedirs(destination_dir)
if not os.path.isfile(destination):
logging.warning("File %s not found", destination)
if interactive:
answer = utils.ask("Would you like to download it from %s?" % url)
else:
answer = 'y'
if answer == 'y':
utils.interactive_download(
url, destination, "Downloading %s" % title)
had_to_download = True
else:
logging.warning("Missing file %s", destination)
else:
logging.info("Found %s", destination)
if sha1 is None:
answer = 'n'
else:
answer = 'y'
def create_config_files(test_dir,
shared_dir,
interactive,
step=None,
force_update=False):
def is_file_tracked(fl):
tracked_result = utils.run("git ls-files %s --error-unmatch" % fl,
ignore_status=True,
verbose=False)
return (tracked_result.exit_status == 0)
if step is None:
step = 0
logging.info("")
step += 1
logging.info("%d - Generating config set", step)
config_file_list = data_dir.SubdirGlobList(os.path.join(test_dir, "cfg"),
"*.cfg", config_filter)
config_file_list = [cf for cf in config_file_list if is_file_tracked(cf)]
config_file_list_shared = glob.glob(
os.path.join(shared_dir, "cfg", "*.cfg"))
# Handle overrides of cfg files. Let's say a test provides its own
# subtest.cfg.sample, this file takes precedence over the shared
# subtest.cfg.sample. So, yank this file from the cfg file list.
config_file_list_shared_keep = []
for cf in config_file_list_shared:
basename = os.path.basename(cf)
target = os.path.join(test_dir, "cfg", basename)
if target not in config_file_list:
config_file_list_shared_keep.append(cf)
config_file_list += config_file_list_shared_keep
for config_file in config_file_list:
src_file = config_file
dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file))
if not os.path.isfile(dst_file):
logging.debug("Creating config file %s from sample", dst_file)
shutil.copyfile(src_file, dst_file)
else:
diff_cmd = "diff -Naur %s %s" % (dst_file, src_file)
diff_result = utils.run(diff_cmd,
ignore_status=True,
verbose=False)
if diff_result.exit_status != 0:
logging.info("%s result:\n %s", diff_result.command,
diff_result.stdout)
if interactive:
answer = utils.ask("Config file %s differs from %s."
"Overwrite?" % (dst_file, src_file))
elif force_update:
answer = "y"
else:
answer = "n"
if answer == "y":
logging.debug("Restoring config file %s from sample",
dst_file)
shutil.copyfile(src_file, dst_file)
else:
logging.debug("Preserving existing %s file", dst_file)
else:
logging.debug("Config file %s exists, not touching", dst_file)
def verify_selinux(datadir,
imagesdir,
isosdir,
tmpdir,
interactive,
selinux=False):
"""
Verify/Set/Warn about SELinux and default file contexts for testing.
:param datadir: Abs. path to data-directory symlink
:param imagesdir: Abs. path to data/images directory
:param isosdir: Abs. path to data/isos directory
:param tmpdir: Abs. path to virt-test tmp dir
:param interactive: True if running from console
:param selinux: Whether setup SELinux contexts for shared/data
"""
# datadir can be a symlink, but these must not have any
imagesdir = os.path.realpath(imagesdir)
isosdir = os.path.realpath(isosdir)
tmpdir = os.path.realpath(tmpdir)
needs_relabel = None
try:
# Raise SeCmdError if selinux not installed
if utils_selinux.get_status() == 'enforcing':
# Check if default contexts are set
if not haz_defcon(datadir, imagesdir, isosdir, tmpdir):
if selinux:
answer = "y"
else:
if interactive:
answer = utils.ask("Setup all undefined default SE"
"Linux contexts for shared/data/?")
else:
answer = "n"
else:
answer = "n"
if answer.lower() == "y":
# Assume relabeling is needed if changes made
needs_relabel = set_defcon(datadir, imagesdir, isosdir, tmpdir)
# Only relabel if files/dirs don't match default
labels_ok = utils_selinux.verify_defcon(datadir, False)
labels_ok &= utils_selinux.verify_defcon(imagesdir, True)
labels_ok &= utils_selinux.verify_defcon(isosdir, True)
labels_ok &= utils_selinux.verify_defcon(tmpdir, True)
if labels_ok:
needs_relabel = False
else:
logging.warning("On-disk SELinux labels do not match defaults")
needs_relabel = True
# Disabled or Permissive mode is same result as not installed
else:
logging.info("SELinux in permissive or disabled, testing"
"in enforcing mode is highly encourraged.")
except utils_selinux.SemanageError:
logging.info("Could not set default SELinux contexts. Please")
logging.info("consider installing the semanage program then ")
logging.info("verifying and/or running running:")
# Paths must be transmogrified (changed) into regular expressions
logging.info("semanage fcontext --add -t virt_var_lib_t '%s'",
utils_selinux.transmogrify_usr_local(datadir))
logging.info(
"semanage fcontext --add -t virt_image_t '%s'",
utils_selinux.transmogrify_usr_local(
utils_selinux.transmogrify_sub_dirs(imagesdir)))
logging.info(
"semanage fcontext --add -t virt_content_t '%s'",
utils_selinux.transmogrify_usr_local(
utils_selinux.transmogrify_sub_dirs(isosdir)))
logging.info(
"semanage fcontext --add -t user_tmp_t '%s'",
utils_selinux.transmogrify_usr_local(
utils_selinux.transmogrify_sub_dirs(tmpdir)))
needs_relabel = None # Next run will catch if relabeling needed
except utils_selinux.SelinuxError: # Catchall SELinux related
logging.info("SELinux not available, or error in command/setup.")
logging.info("Please manually verify default file contexts before")
logging.info("testing with SELinux enabled and enforcing.")
if needs_relabel:
if selinux:
answer = "y"
else:
if interactive:
answer = utils.ask("Relabel from default contexts?")
else:
answer = "n"
if answer.lower() == 'y':
changes = utils_selinux.apply_defcon(datadir, False)
changes += utils_selinux.apply_defcon(imagesdir, True)
changes += utils_selinux.apply_defcon(isosdir, True)
changes += utils_selinux.apply_defcon(tmpdir, True)
logging.info("Corrected contexts on %d files/dirs", len(changes))
