def run(self, args): LOG_UI.info('Config files read (in order):') for cfg_path in settings.config_paths: LOG_UI.debug(' %s' % cfg_path) if settings.config_paths_failed: LOG_UI.error('\nConfig files that failed to read:') for cfg_path in settings.config_paths_failed: LOG_UI.error(' %s' % cfg_path) LOG_UI.debug("") if not args.datadir: blength = 0 for section in settings.config.sections(): for value in settings.config.items(section): clength = len('%s.%s' % (section, value[0])) if clength > blength: blength = clength format_str = " %-" + str(blength) + "s %s" LOG_UI.debug(format_str, 'Section.Key', 'Value') for section in settings.config.sections(): for value in settings.config.items(section): config_key = ".".join((section, value[0])) LOG_UI.debug(format_str, config_key, value[1]) else: LOG_UI.debug("Avocado replaces config dirs that can't be accessed") LOG_UI.debug("with sensible defaults. Please edit your local config") LOG_UI.debug("file to customize values") LOG_UI.debug('') LOG_UI.info('Avocado Data Directories:') LOG_UI.debug(' base ' + data_dir.get_base_dir()) LOG_UI.debug(' tests ' + data_dir.get_test_dir()) LOG_UI.debug(' data ' + data_dir.get_data_dir()) LOG_UI.debug(' logs ' + data_dir.get_logs_dir()) LOG_UI.debug(' cache ' + ", ".join(data_dir.get_cache_dirs()))
def fetch_assets(test_file, klass=None, method=None, logger=None): """ Fetches the assets based on keywords listed on FetchAssetHandler.calls. :param test_file: File name of instrumented test to be evaluated :type test_file: str :returns: list of names that were successfully fetched and list of fails. """ cache_dirs = data_dir.get_cache_dirs() success = [] fail = [] handler = FetchAssetHandler(test_file, klass, method) for call in handler.calls: expire = call.pop('expire', None) if expire is not None: expire = data_structures.time_to_seconds(str(expire)) # make dictionary unpacking compatible with python 3.4 as it does # not support constructions like: # Asset(**call, cache_dirs=cache_dirs, expire=expire) call['cache_dirs'] = cache_dirs call['expire'] = expire try: asset_obj = Asset(**call) if logger is not None: logger.info('Fetching asset from %s:%s.%s', test_file, klass, method) asset_obj.fetch() success.append(call['name']) except (OSError, ValueError) as failed: fail.append(failed) return success, fail
def download_image(distro, version=None, arch=None): """ Downloads the vmimage to the cache directory if doesn't already exist :param distro: Name of image distribution :type distro: str :param version: Version of image :type version: str :param arch: Architecture of image :type arch: str :raise AttributeError: When image can't be downloaded :return: Information about downloaded image :rtype: dict """ cache_dir = data_dir.get_cache_dirs()[0] image_info = vmimage.get(name=distro, version=version, arch=arch, cache_dir=cache_dir) file_path = image_info.base_image image = { 'name': distro, 'version': image_info.version, 'arch': image_info.arch, 'file': file_path } return image
def handle_default(): LOG_UI.info("Config files read (in order, '*' means the file exists " "and had been read):") # Getting from settings because is already sorted config = settings.as_dict() for cfg_path in settings.all_config_paths: if cfg_path in settings.config_paths: LOG_UI.debug(' * %s', cfg_path) else: LOG_UI.debug(' %s', cfg_path) LOG_UI.debug("") if not config.get('config.datadir'): blength = 0 for namespace, value in config.items(): clength = len(namespace) if clength > blength: blength = clength format_str = " %-" + str(blength) + "s %s" LOG_UI.debug(format_str, 'Section.Key', 'Value') for namespace, value in config.items(): LOG_UI.debug(format_str, namespace, value) else: LOG_UI.debug("Avocado replaces config dirs that can't be accessed") LOG_UI.debug("with sensible defaults. Please edit your local config") LOG_UI.debug("file to customize values") LOG_UI.debug('') LOG_UI.info('Avocado Data Directories:') LOG_UI.debug(' base %s', data_dir.get_base_dir()) LOG_UI.debug(' tests %s', data_dir.get_test_dir()) LOG_UI.debug(' data %s', data_dir.get_data_dir()) LOG_UI.debug(' logs %s', data_dir.get_logs_dir()) LOG_UI.debug(' cache %s', ", ".join(data_dir.get_cache_dirs()))
def run(self, config): LOG_UI.info("Config files read (in order, '*' means the file exists " "and had been read):") for cfg_path in settings.all_config_paths: if cfg_path in settings.config_paths: LOG_UI.debug(' * %s', cfg_path) else: LOG_UI.debug(' %s', cfg_path) LOG_UI.debug("") if not config.get("datadir"): blength = 0 for section in settings.config.sections(): for value in settings.config.items(section): clength = len('%s.%s' % (section, value[0])) if clength > blength: blength = clength format_str = " %-" + str(blength) + "s %s" LOG_UI.debug(format_str, 'Section.Key', 'Value') for section in settings.config.sections(): for value in settings.config.items(section): config_key = ".".join((section, value[0])) LOG_UI.debug(format_str, config_key, value[1]) else: LOG_UI.debug("Avocado replaces config dirs that can't be accessed") LOG_UI.debug("with sensible defaults. Please edit your local config") LOG_UI.debug("file to customize values") LOG_UI.debug('') LOG_UI.info('Avocado Data Directories:') LOG_UI.debug(' base %s', data_dir.get_base_dir()) LOG_UI.debug(' tests %s', data_dir.get_test_dir()) LOG_UI.debug(' data %s', data_dir.get_data_dir()) LOG_UI.debug(' logs %s', data_dir.get_logs_dir()) LOG_UI.debug(' cache %s', ", ".join(data_dir.get_cache_dirs()))
def _create_test_files(self, urlopen_mock): with unittest.mock.patch('avocado.core.data_dir.settings.settings', self.stg): expected_images = [{'name': 'Fedora', 'file': 'Fedora-Cloud-Base-{version}-{build}.{arch}.qcow2', 'url': FEDORA_PAGE}, {'name': 'JeOS', 'file': 'jeos-{version}-{arch}.qcow2.xz', 'url': JEOS_PAGE}, {'name': 'CirrOS', 'file': 'cirros-{version}-{arch}-disk.img', 'url': CIRROS_PAGE} ] cache_dir = data_dir.get_cache_dirs()[0] providers = [provider() for provider in vmimage_util.list_providers()] for provider in providers: for image in expected_images: if image['name'] == provider.name: urlread_mocked = unittest.mock.Mock(return_value=image["url"]) urlopen_mock.return_value = unittest.mock.Mock(read=urlread_mocked) image['type'] = "vmimage" image['version'] = provider.version image['arch'] = provider.arch image['build'] = "1234" image['file'] = os.path.join(cache_dir, image['file'].format( version=image['version'], build=image['build'], arch=image['arch'])) open(image["file"], "w").close() create_metadata_file(image['file'], image) return sorted(expected_images, key=lambda i: i['name'])
def handle_purge(self, config): days = config.get('assets.purge.days') size_filter = config.get('assets.purge.size_filter') overall_limit = config.get('assets.purge.overall_limit') if self._count_filter_args(config) != 1: msg = ("You should choose --by-size-filter or --by-days. " "For help, run: avocado assets purge --help") LOG_UI.error(msg) return exit_codes.AVOCADO_FAIL cache_dirs = data_dir.get_cache_dirs() try: if days is not None: Asset.remove_assets_by_unused_for_days(days, cache_dirs) elif size_filter is not None: Asset.remove_assets_by_size(size_filter, cache_dirs) elif overall_limit is not None: try: size = DataSize(overall_limit).b Asset.remove_assets_by_overall_limit(size, cache_dirs) except InvalidDataSize: error_msg = "You are using an invalid suffix. " error_msg += "Use one of the following values: " error_msg += ",".join(DataSize.MULTIPLIERS.keys()) LOG_UI.error(error_msg) return exit_codes.AVOCADO_FAIL except (FileNotFoundError, OSError) as e: LOG_UI.error("Could not remove asset: %s", e) return exit_codes.AVOCADO_FAIL return exit_codes.AVOCADO_ALL_OK
def run(self, args): LOG_UI.info("Config files read (in order, '*' means the file exists " "and had been read):") for cfg_path in settings.all_config_paths: if cfg_path in settings.config_paths: LOG_UI.debug(' * %s', cfg_path) else: LOG_UI.debug(' %s', cfg_path) LOG_UI.debug("") if not args.datadir: blength = 0 for section in settings.config.sections(): for value in settings.config.items(section): clength = len('%s.%s' % (section, value[0])) if clength > blength: blength = clength format_str = " %-" + str(blength) + "s %s" LOG_UI.debug(format_str, 'Section.Key', 'Value') for section in settings.config.sections(): for value in settings.config.items(section): config_key = ".".join((section, value[0])) LOG_UI.debug(format_str, config_key, value[1]) else: LOG_UI.debug("Avocado replaces config dirs that can't be accessed") LOG_UI.debug("with sensible defaults. Please edit your local config") LOG_UI.debug("file to customize values") LOG_UI.debug('') LOG_UI.info('Avocado Data Directories:') LOG_UI.debug(' base %s', data_dir.get_base_dir()) LOG_UI.debug(' tests %s', data_dir.get_test_dir()) LOG_UI.debug(' data %s', data_dir.get_data_dir()) LOG_UI.debug(' logs %s', data_dir.get_logs_dir()) LOG_UI.debug(' cache %s', ", ".join(data_dir.get_cache_dirs()))
def main(): for image in KNOWN_IMAGES: name, version, arch, checksum, algorithm = image print("%s version %s (%s): " % (name, version, arch), end='') download = vmimage.get(name=name, version=version, arch=arch, checksum=checksum, algorithm=algorithm, cache_dir=data_dir.get_cache_dirs()[0]) print(download.base_image)
def fetch_assets(test_file, klass=None, method=None, logger=None): """ Fetches the assets based on keywords listed on FetchAssetHandler.calls. :param test_file: File name of instrumented test to be evaluated :type test_file: str :returns: list of names that were successfully fetched and list of fails. """ def validate_parameters(call): """ Validate the parameters to make sure we have a supported case. :param call: List of parameter to the Asset object. :type call: dict :returns: True or False """ name = call.get('name', None) locations = call.get('locations', None) # probably, parameter name was defined as a class attribute if ((name is None) or # probably, parameter locations was defined as a class attribute (not urllib.parse.urlparse(name).scheme and locations is None)): return False return True cache_dirs = data_dir.get_cache_dirs() success = [] fail = [] handler = FetchAssetHandler(test_file, klass, method) for call in handler.calls: # validate the parameters if not validate_parameters(call): continue expire = call.pop('expire', None) if expire is not None: expire = data_structures.time_to_seconds(str(expire)) try: # make dictionary unpacking compatible with python 3.4 as it does # not support constructions like: # Asset(**call, cache_dirs=cache_dirs, expire=expire) call['cache_dirs'] = cache_dirs call['expire'] = expire asset_obj = Asset(**call) if logger is not None: logger.info('Fetching asset from %s:%s.%s', test_file, klass, method) asset_obj.fetch() success.append(call['name']) except (OSError, ValueError) as failed: fail.append(failed) return success, fail
def handle_purge(self, config): days = config.get('assets.purge.days') size_filter = config.get('assets.purge.size_filter') if (days is None and size_filter is None) \ or (days is not None and size_filter is not None): msg = ("You should choose --by-size-filter or --by-days. " "For help, run: avocado assets purge --help") LOG_UI.error(msg) return cache_dirs = data_dir.get_cache_dirs() try: if days is not None: Asset.remove_assets_by_unused_for_days(days, cache_dirs) elif size_filter is not None: Asset.remove_assets_by_size(size_filter, cache_dirs) except (FileNotFoundError, OSError) as e: LOG_UI.error("Could not remove asset: %s", e)
def handle_register(self, config): cache_dirs = data_dir.get_cache_dirs() name = config.get('assets.register.name') asset_hash = config.get('assets.register.sha1_hash') location = config.get('assets.register.url') # Adding a twice the location is a small hack due the current logic to # return "by_name". This needs to be improved soon. asset = Asset(name=name, asset_hash=asset_hash, locations=[location, location], cache_dirs=cache_dirs) try: asset.find_asset_file() LOG_UI.error("Asset with name %s already registered.", name) except OSError: try: asset.fetch() LOG_UI.info("Done. Now you can reference it by name %s", name) except OSError as e: LOG_UI.error(e)
def handle_list(self, config): days = config.get('assets.list.days') size_filter = config.get('assets.list.size_filter') if (days is not None and size_filter is not None): msg = ("You should choose --by-size-filter or --by-days. " "For help, run: avocado assets list --help") LOG_UI.error(msg) return # IMO, this should get data from config instead. See #4443 cache_dirs = data_dir.get_cache_dirs() try: assets = None if days is not None: assets = Asset.get_assets_unused_for_days(days, cache_dirs) elif size_filter is not None: assets = Asset.get_assets_by_size(size_filter, cache_dirs) elif assets is None: assets = Asset.get_all_assets(cache_dirs) except (FileNotFoundError, OSError) as e: LOG_UI.error("Could get assets: %s", e) return matrix = [] for asset in assets: stat = os.stat(asset) basename = os.path.basename(asset) hash_path = "{}-CHECKSUM".format(asset) atime = datetime.fromtimestamp(stat.st_atime) _, checksum = Asset.read_hash_from_file(hash_path) matrix.append((basename, str(checksum or "unknown")[:10], atime.strftime('%Y-%m-%d %H:%M:%S'), display_data_size(stat.st_size))) header = ("asset", "checksum", "atime", "size") output = list(iter_tabular_output(matrix, header)) if len(output) == 1: LOG_UI.info("No asset found in your cache system.") else: for line in output: LOG_UI.info(line)
def list_downloaded_images(): """ List the available Image inside avocado cache :return: list with image's parameters :rtype: list of dicts """ images = [] for cache_dir in data_dir.get_cache_dirs(): for root, _, files in os.walk(cache_dir): if files: metadata_files = [ pos_json for pos_json in files if pos_json.endswith('_metadata.json') ] files = list(set(files) - set(metadata_files)) for metadata_file in metadata_files: with open(os.path.join(root, metadata_file), 'r') as data: metadata = json.loads(data.read()) if isinstance(metadata, dict): if metadata.get("type", None) == "vmimage": provider = None for p in vmimage.IMAGE_PROVIDERS: if p.name == metadata["name"]: provider = p(metadata["version"], metadata["build"], metadata["arch"]) break if provider is not None: for image in files: if re.match(provider.file_name, image): data = { "name": provider.name, "version": provider.version, "arch": provider.arch, "file": os.path.join(root, image) } images.append(data) break return images
def run(self, args): LOG_UI.info('Config files read (in order):') for cfg_path in settings.config_paths: LOG_UI.debug(' %s' % cfg_path) if settings.config_paths_failed: LOG_UI.error('\nConfig files that failed to read:') for cfg_path in settings.config_paths_failed: LOG_UI.error(' %s' % cfg_path) LOG_UI.debug("") if not args.datadir: blength = 0 for section in settings.config.sections(): for value in settings.config.items(section): clength = len('%s.%s' % (section, value[0])) if clength > blength: blength = clength format_str = " %-" + str(blength) + "s %s" LOG_UI.debug(format_str, 'Section.Key', 'Value') for section in settings.config.sections(): for value in settings.config.items(section): config_key = ".".join((section, value[0])) LOG_UI.debug(format_str, config_key, value[1]) else: LOG_UI.debug("Avocado replaces config dirs that can't be accessed") LOG_UI.debug( "with sensible defaults. Please edit your local config") LOG_UI.debug("file to customize values") LOG_UI.debug('') LOG_UI.info('Avocado Data Directories:') LOG_UI.debug(' base ' + data_dir.get_base_dir()) LOG_UI.debug(' tests ' + data_dir.get_test_dir()) LOG_UI.debug(' data ' + data_dir.get_data_dir()) LOG_UI.debug(' logs ' + data_dir.get_logs_dir()) LOG_UI.debug(' cache ' + ", ".join(data_dir.get_cache_dirs()))