def sync_download_dir(interactive): base_download_dir = data_dir.get_base_download_dir() download_dir = data_dir.get_download_dir() logging.debug( "Copying downloadable assets file definitions from %s " "into %s", base_download_dir, download_dir) download_file_list = glob.glob(os.path.join(base_download_dir, "*.ini")) logging.debug("Donwload file list: %s", download_file_list) for src_file in download_file_list: dst_file = os.path.join(download_dir, os.path.basename(src_file)) if not os.path.isfile(dst_file): shutil.copyfile(src_file, dst_file) else: diff_cmd = "diff -Naur %s %s" % (dst_file, src_file) diff_result = process.run(diff_cmd, ignore_status=True, verbose=False) if diff_result.exit_status != 0: logging.info("%s result:\n %s", diff_result.command, diff_result.stdout) answer = genio.ask('Download file "%s" differs from "%s". ' 'Overwrite?' % (dst_file, src_file), auto=not interactive) if answer == "y": logging.debug("Restoring download file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: logging.debug("Preserving existing %s file", dst_file) else: logging.debug('Download file %s exists, not touching', dst_file)
def sync_download_dir(interactive): base_download_dir = data_dir.get_base_download_dir() download_dir = data_dir.get_download_dir() logging.debug("Copying downloadable assets file definitions from %s " "into %s", base_download_dir, download_dir) download_file_list = glob.glob(os.path.join(base_download_dir, "*.ini")) for src_file in download_file_list: dst_file = os.path.join(download_dir, os.path.basename(src_file)) if not os.path.isfile(dst_file): shutil.copyfile(src_file, dst_file) else: diff_cmd = "diff -Naur %s %s" % (dst_file, src_file) diff_result = process.run( diff_cmd, ignore_status=True, verbose=False) if diff_result.exit_status != 0: logging.info("%s result:\n %s", diff_result.command, diff_result.stdout) answer = genio.ask('Download file "%s" differs from "%s". ' 'Overwrite?' % (dst_file, src_file), auto=not interactive) if answer == "y": logging.debug("Restoring download file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: logging.debug("Preserving existing %s file", dst_file) else: logging.debug('Download file %s exists, not touching', dst_file)
def create_config_files(test_dir, shared_dir, interactive, step=None, force_update=False): def is_file_tracked(fl): tracked_result = process.run("git ls-files %s --error-unmatch" % fl, ignore_status=True, verbose=False) return tracked_result.exit_status == 0 if step is None: step = 0 logging.info("") step += 1 logging.info("%d - Generating config set", step) config_file_list = data_dir.SubdirGlobList(os.path.join(test_dir, "cfg"), "*.cfg", config_filter) config_file_list = [cf for cf in config_file_list if is_file_tracked(cf)] config_file_list_shared = glob.glob(os.path.join(shared_dir, "cfg", "*.cfg")) # Handle overrides of cfg files. Let's say a test provides its own # subtest.cfg.sample, this file takes precedence over the shared # subtest.cfg.sample. So, yank this file from the cfg file list. config_file_list_shared_keep = [] for cf in config_file_list_shared: basename = os.path.basename(cf) target = os.path.join(test_dir, "cfg", basename) if target not in config_file_list: config_file_list_shared_keep.append(cf) config_file_list += config_file_list_shared_keep for config_file in config_file_list: src_file = config_file dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file)) if not os.path.isfile(dst_file): logging.debug("Creating config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: diff_cmd = "diff -Naur %s %s" % (dst_file, src_file) diff_result = process.run( diff_cmd, ignore_status=True, verbose=False) if diff_result.exit_status != 0: logging.info("%s result:\n %s", diff_result.command, diff_result.stdout) answer = genio.ask("Config file %s differs from %s." "Overwrite?" % (dst_file, src_file), auto=force_update or not interactive) if answer == "y": logging.debug("Restoring config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: logging.debug("Preserving existing %s file", dst_file) else: logging.debug("Config file %s exists, not touching", dst_file) return step
def download_asset(asset, interactive=True, restore_image=False): """ Download an asset defined on an asset file. Asset files are located under /shared/downloads, are .ini files with the following keys defined: title Title string to display in the download progress bar. url URL of the resource sha1_url URL with SHA1 information for the resource, in the form sha1sum file_basename destination Location of your file relative to the data directory (TEST_SUITE_ROOT/shared/data) destination Location of the uncompressed file relative to the data directory (TEST_SUITE_ROOT/shared/data) uncompress_cmd Command that needs to be executed with the compressed file as a parameter :param asset: String describing an asset file. :param interactive: Whether to ask the user before downloading the file. :param restore_image: If the asset is a compressed image, we can uncompress in order to restore the image. """ asset_info = get_asset_info(asset) destination = asset_info['destination'] if (interactive and not os.path.isfile(destination)): answer = genio.ask("File %s not present. Do you want to download it?" % asset_info['title']) else: answer = "y" if answer == "y": download_file(asset_info=asset_info, interactive=interactive, force=restore_image)
def download_file(asset_info, interactive=False, force=False): """ Verifies if file that can be find on url is on destination with right hash. This function will verify the SHA1 hash of the file. If the file appears to be missing or corrupted, let the user know. :param asset_info: Dictionary returned by get_asset_info """ file_ok = False problems_ignored = False had_to_download = False sha1 = None url = asset_info['url'] sha1_url = asset_info['sha1_url'] destination = asset_info['destination'] title = asset_info['title'] if sha1_url is not None: try: logging.info("Verifying expected SHA1 sum from %s", sha1_url) sha1_file = urllib.request.urlopen(sha1_url) sha1_contents = decode_to_text(sha1_file.read()) sha1 = sha1_contents.split(" ")[0] logging.info("Expected SHA1 sum: %s", sha1) except Exception as e: logging.error("Failed to get SHA1 from file: %s", e) else: sha1 = None destination_dir = os.path.dirname(destination) if not os.path.isdir(destination_dir): os.makedirs(destination_dir) if not os.path.isfile(destination): logging.warning("File %s not found", destination) if interactive: answer = genio.ask("Would you like to download it from %s?" % url) else: answer = 'y' if answer == 'y': try: download.url_download_interactive(url, destination, "Downloading %s" % title) had_to_download = True except Exception as download_failure: logging.error("Check your internet connection: %s", download_failure) else: logging.warning("Missing file %s", destination) else: logging.info("Found %s", destination) if sha1 is None: answer = 'n' else: answer = 'y' if answer == 'y': actual_sha1 = crypto.hash_file(destination, algorithm='sha1') if actual_sha1 != sha1: logging.info("Actual SHA1 sum: %s", actual_sha1) if interactive: answer = genio.ask("The file seems corrupted or outdated. " "Would you like to download it?") else: logging.info("The file seems corrupted or outdated") answer = 'y' if answer == 'y': logging.info("Updating image to the latest available...") while not file_ok: try: download.url_download_interactive( url, destination, title) except Exception as download_failure: logging.error("Check your internet connection: %s", download_failure) sha1_post_download = crypto.hash_file(destination, algorithm='sha1') had_to_download = True if sha1_post_download != sha1: logging.error("Actual SHA1 sum: %s", actual_sha1) if interactive: answer = genio.ask("The file downloaded %s is " "corrupted. Would you like " "to try again?" % destination) else: answer = 'n' if answer == 'n': problems_ignored = True logging.error("File %s is corrupted" % destination) file_ok = True else: file_ok = False else: file_ok = True else: file_ok = True logging.info("SHA1 sum check OK") else: problems_ignored = True logging.info("File %s present, but did not verify integrity", destination) if file_ok: if not problems_ignored: logging.info("%s present, with proper checksum", destination) uncompress_asset(asset_info=asset_info, force=force or had_to_download)
sha1_contents = sha1_file.read() sha1 = sha1_contents.split(" ")[0] logging.info("Expected SHA1 sum: %s", sha1) except Exception, e: logging.error("Failed to get SHA1 from file: %s", e) else: sha1 = None destination_dir = os.path.dirname(destination) if not os.path.isdir(destination_dir): os.makedirs(destination_dir) if not os.path.isfile(destination): logging.warning("File %s not found", destination) if interactive: answer = genio.ask("Would you like to download it from %s?" % url) else: answer = 'y' if answer == 'y': download.url_download_interactive(url, destination, "Downloading %s" % title) had_to_download = True else: logging.warning("Missing file %s", destination) else: logging.info("Found %s", destination) if sha1 is None: answer = 'n' else: answer = 'y'
def verify_selinux(datadir, imagesdir, isosdir, tmpdir, interactive, selinux=False): """ Verify/Set/Warn about SELinux and default file contexts for testing. :param datadir: Abs. path to data-directory symlink :param imagesdir: Abs. path to data/images directory :param isosdir: Abs. path to data/isos directory :param tmpdir: Abs. path to avocado-vt tmp dir :param interactive: True if running from console :param selinux: Whether setup SELinux contexts for shared/data """ # datadir can be a symlink, but these must not have any imagesdir = os.path.realpath(imagesdir) isosdir = os.path.realpath(isosdir) tmpdir = os.path.realpath(tmpdir) needs_relabel = None try: # Raise SeCmdError if selinux not installed if utils_selinux.get_status() == 'enforcing': # Check if default contexts are set if not haz_defcon(datadir, imagesdir, isosdir, tmpdir): if selinux: answer = "y" else: answer = genio.ask( "Setup all undefined default SE" "Linux contexts for shared/data/?", auto=not interactive) else: answer = "n" if answer.lower() == "y": # Assume relabeling is needed if changes made needs_relabel = set_defcon(datadir, imagesdir, isosdir, tmpdir) # Only relabel if files/dirs don't match default labels_ok = utils_selinux.verify_defcon(datadir, False) labels_ok &= utils_selinux.verify_defcon(imagesdir, True) labels_ok &= utils_selinux.verify_defcon(isosdir, True) labels_ok &= utils_selinux.verify_defcon(tmpdir, True) if labels_ok: needs_relabel = False else: logging.warning("On-disk SELinux labels do not match defaults") needs_relabel = True # Disabled or Permissive mode is same result as not installed else: logging.info("SELinux in permissive or disabled, testing" "in enforcing mode is highly encourraged.") except utils_selinux.SemanageError: logging.info("Could not set default SELinux contexts. Please") logging.info("consider installing the semanage program then ") logging.info("verifying and/or running running:") # Paths must be transmogrified (changed) into regular expressions logging.info("semanage fcontext --add -t virt_var_lib_t '%s'", utils_selinux.transmogrify_usr_local(datadir)) logging.info( "semanage fcontext --add -t virt_image_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(imagesdir))) logging.info( "semanage fcontext --add -t virt_content_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(isosdir))) logging.info( "semanage fcontext --add -t user_tmp_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(tmpdir))) needs_relabel = None # Next run will catch if relabeling needed except utils_selinux.SelinuxError: # Catchall SELinux related logging.info("SELinux not available, or error in command/setup.") logging.info("Please manually verify default file contexts before") logging.info("testing with SELinux enabled and enforcing.") if needs_relabel: if selinux: answer = "y" else: answer = genio.ask("Relabel from default contexts?", auto=not interactive) if answer.lower() == 'y': changes = utils_selinux.apply_defcon(datadir, False) changes += utils_selinux.apply_defcon(imagesdir, True) changes += utils_selinux.apply_defcon(isosdir, True) changes += utils_selinux.apply_defcon(tmpdir, True) logging.info("Corrected contexts on %d files/dirs", len(changes))
def create_config_files(test_dir, shared_dir, interactive, t_type, step=None, force_update=False): def is_file_tracked(fl): tracked_result = process.run("git ls-files %s --error-unmatch" % fl, ignore_status=True, verbose=False) return tracked_result.exit_status == 0 if step is None: step = 0 logging.info("") step += 1 logging.info("%d - Generating config set", step) config_file_list = data_dir.SubdirGlobList(os.path.join(test_dir, "cfg"), "*.cfg", config_filter) config_file_list = [cf for cf in config_file_list if is_file_tracked(cf)] config_file_list_shared = glob.glob( os.path.join(shared_dir, "cfg", "*.cfg")) provider_info_specific = [] provider_names_specific = asset.get_test_provider_names(t_type) for specific_provider in provider_names_specific: provider_info_specific.append( asset.get_test_provider_info(specific_provider)) specific_subdirs = asset.get_test_provider_subdirs(t_type) for subdir in specific_subdirs: for p in provider_info_specific: if 'cartesian_configs' in p['backends'][t_type]: for c in p['backends'][t_type]['cartesian_configs']: cfg = os.path.join(subdir, "cfg", c) config_file_list.append(cfg) # Handle overrides of cfg files. Let's say a test provides its own # subtest.cfg.sample, this file takes precedence over the shared # subtest.cfg.sample. So, yank this file from the cfg file list. config_file_list_shared_keep = [] for cf in config_file_list_shared: basename = os.path.basename(cf) target = os.path.join(test_dir, "cfg", basename) if target not in config_file_list: config_file_list_shared_keep.append(cf) config_file_list += config_file_list_shared_keep for config_file in config_file_list: src_file = config_file dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file)) if not os.path.isfile(dst_file): logging.debug("Creating config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: diff_cmd = "diff -Naur %s %s" % (dst_file, src_file) diff_result = process.run(diff_cmd, ignore_status=True, verbose=False) if diff_result.exit_status != 0: logging.info("%s result:\n %s", diff_result.command, diff_result.stdout) answer = genio.ask("Config file %s differs from %s." "Overwrite?" % (dst_file, src_file), auto=force_update or not interactive) if answer == "y": logging.debug("Restoring config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: logging.debug("Preserving existing %s file", dst_file) else: if force_update: update_msg = 'Config file %s exists, equal to sample' else: update_msg = 'Config file %s exists, not touching' logging.debug(update_msg, dst_file) return step
def verify_selinux(datadir, imagesdir, isosdir, tmpdir, interactive, selinux=False): """ Verify/Set/Warn about SELinux and default file contexts for testing. :param datadir: Abs. path to data-directory symlink :param imagesdir: Abs. path to data/images directory :param isosdir: Abs. path to data/isos directory :param tmpdir: Abs. path to avocado-vt tmp dir :param interactive: True if running from console :param selinux: Whether setup SELinux contexts for shared/data """ # datadir can be a symlink, but these must not have any imagesdir = os.path.realpath(imagesdir) isosdir = os.path.realpath(isosdir) tmpdir = os.path.realpath(tmpdir) needs_relabel = None try: # Raise SeCmdError if selinux not installed if utils_selinux.get_status() == 'enforcing': # Check if default contexts are set if not haz_defcon(datadir, imagesdir, isosdir, tmpdir): if selinux: answer = "y" else: answer = genio.ask("Setup all undefined default SE" "Linux contexts for shared/data/?", auto=not interactive) else: answer = "n" if answer.lower() == "y": # Assume relabeling is needed if changes made needs_relabel = set_defcon(datadir, imagesdir, isosdir, tmpdir) # Only relabel if files/dirs don't match default labels_ok = utils_selinux.verify_defcon(datadir, False) labels_ok &= utils_selinux.verify_defcon(imagesdir, True) labels_ok &= utils_selinux.verify_defcon(isosdir, True) labels_ok &= utils_selinux.verify_defcon(tmpdir, True) if labels_ok: needs_relabel = False else: logging.warning("On-disk SELinux labels do not match defaults") needs_relabel = True # Disabled or Permissive mode is same result as not installed else: logging.info("SELinux in permissive or disabled, testing" "in enforcing mode is highly encourraged.") except utils_selinux.SemanageError: logging.info("Could not set default SELinux contexts. Please") logging.info("consider installing the semanage program then ") logging.info("verifying and/or running running:") # Paths must be transmogrified (changed) into regular expressions logging.info("semanage fcontext --add -t virt_var_lib_t '%s'", utils_selinux.transmogrify_usr_local(datadir)) logging.info("semanage fcontext --add -t virt_image_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(imagesdir))) logging.info("semanage fcontext --add -t virt_content_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(isosdir))) logging.info("semanage fcontext --add -t user_tmp_t '%s'", utils_selinux.transmogrify_usr_local( utils_selinux.transmogrify_sub_dirs(tmpdir))) needs_relabel = None # Next run will catch if relabeling needed except utils_selinux.SelinuxError: # Catchall SELinux related logging.info("SELinux not available, or error in command/setup.") logging.info("Please manually verify default file contexts before") logging.info("testing with SELinux enabled and enforcing.") if needs_relabel: if selinux: answer = "y" else: answer = genio.ask("Relabel from default contexts?", auto=not interactive) if answer.lower() == 'y': changes = utils_selinux.apply_defcon(datadir, False) changes += utils_selinux.apply_defcon(imagesdir, True) changes += utils_selinux.apply_defcon(isosdir, True) changes += utils_selinux.apply_defcon(tmpdir, True) logging.info("Corrected contexts on %d files/dirs", len(changes))
def create_config_files(test_dir, shared_dir, interactive, t_type, step=None, force_update=False): def is_file_tracked(fl): tracked_result = process.run("git ls-files %s --error-unmatch" % fl, ignore_status=True, verbose=False) return tracked_result.exit_status == 0 if step is None: step = 0 LOG.info("") step += 1 LOG.info("%d - Generating config set", step) config_file_list = data_dir.SubdirGlobList(os.path.join(test_dir, "cfg"), "*.cfg", get_config_filter()) config_file_list = [cf for cf in config_file_list if is_file_tracked(cf)] config_file_list_shared = glob.glob(os.path.join(shared_dir, "cfg", "*.cfg")) provider_info_specific = [] provider_names_specific = asset.get_test_provider_names(t_type) for specific_provider in provider_names_specific: provider_info_specific.append( asset.get_test_provider_info(specific_provider)) specific_subdirs = asset.get_test_provider_subdirs(t_type) for subdir in specific_subdirs: for p in provider_info_specific: if 'cartesian_configs' in p['backends'][t_type]: for c in p['backends'][t_type]['cartesian_configs']: cfg = os.path.join(subdir, "cfg", c) config_file_list.append(cfg) # Handle overrides of cfg files. Let's say a test provides its own # subtest.cfg.sample, this file takes precedence over the shared # subtest.cfg.sample. So, yank this file from the cfg file list. config_file_list_shared_keep = [] for cf in config_file_list_shared: basename = os.path.basename(cf) target = os.path.join(test_dir, "cfg", basename) if target not in config_file_list: config_file_list_shared_keep.append(cf) config_file_list += config_file_list_shared_keep for config_file in config_file_list: src_file = config_file dst_file = os.path.join(test_dir, "cfg", os.path.basename(config_file)) if not os.path.isfile(dst_file): LOG.debug("Creating config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: diff_cmd = "diff -Naur %s %s" % (dst_file, src_file) diff_result = process.run( diff_cmd, ignore_status=True, verbose=False) if diff_result.exit_status != 0: LOG.info("%s result:\n %s", diff_result.command, diff_result.stdout) answer = genio.ask("Config file %s differs from %s." "Overwrite?" % (dst_file, src_file), auto=force_update or not interactive) if answer == "y": LOG.debug("Restoring config file %s from sample", dst_file) shutil.copyfile(src_file, dst_file) else: LOG.debug("Preserving existing %s file", dst_file) else: if force_update: update_msg = 'Config file %s exists, equal to sample' else: update_msg = 'Config file %s exists, not touching' LOG.debug(update_msg, dst_file) return step
def download_file(asset_info, interactive=False, force=False): """ Verifies if file that can be find on url is on destination with right hash. This function will verify the SHA1 hash of the file. If the file appears to be missing or corrupted, let the user know. :param asset_info: Dictionary returned by get_asset_info """ file_ok = False problems_ignored = False had_to_download = False sha1 = None url = asset_info['url'] sha1_url = asset_info['sha1_url'] destination = asset_info['destination'] title = asset_info['title'] if sha1_url is not None: try: logging.info("Verifying expected SHA1 sum from %s", sha1_url) sha1_file = urllib.request.urlopen(sha1_url) sha1_contents = decode_to_text(sha1_file.read()) sha1 = sha1_contents.split(" ")[0] logging.info("Expected SHA1 sum: %s", sha1) except Exception as e: logging.error("Failed to get SHA1 from file: %s", e) else: sha1 = None destination_dir = os.path.dirname(destination) if not os.path.isdir(destination_dir): os.makedirs(destination_dir) if not os.path.isfile(destination): logging.warning("File %s not found", destination) if interactive: answer = genio.ask("Would you like to download it from %s?" % url) else: answer = 'y' if answer == 'y': try: download.url_download_interactive(url, destination, "Downloading %s" % title) had_to_download = True except Exception as download_failure: logging.error("Check your internet connection: %s", download_failure) else: logging.warning("Missing file %s", destination) else: logging.info("Found %s", destination) if sha1 is None: answer = 'n' else: answer = 'y' if answer == 'y': actual_sha1 = crypto.hash_file(destination, algorithm='sha1') if actual_sha1 != sha1: logging.info("Actual SHA1 sum: %s", actual_sha1) if interactive: answer = genio.ask("The file seems corrupted or outdated. " "Would you like to download it?") else: logging.info("The file seems corrupted or outdated") answer = 'y' if answer == 'y': logging.info("Updating image to the latest available...") while not file_ok: try: download.url_download_interactive(url, destination, title) except Exception as download_failure: logging.error("Check your internet connection: %s", download_failure) sha1_post_download = crypto.hash_file(destination, algorithm='sha1') had_to_download = True if sha1_post_download != sha1: logging.error("Actual SHA1 sum: %s", actual_sha1) if interactive: answer = genio.ask("The file downloaded %s is " "corrupted. Would you like " "to try again?" % destination) else: answer = 'n' if answer == 'n': problems_ignored = True logging.error("File %s is corrupted" % destination) file_ok = True else: file_ok = False else: file_ok = True else: file_ok = True logging.info("SHA1 sum check OK") else: problems_ignored = True logging.info("File %s present, but did not verify integrity", destination) if file_ok: if not problems_ignored: logging.info("%s present, with proper checksum", destination) uncompress_asset(asset_info=asset_info, force=force or had_to_download)