def run(self, args):
LOG_UI.info('Config files read (in order):')
for cfg_path in settings.config_paths:
LOG_UI.debug(' %s' % cfg_path)
if settings.config_paths_failed:
LOG_UI.error('\nConfig files that failed to read:')
for cfg_path in settings.config_paths_failed:
LOG_UI.error(' %s' % cfg_path)
LOG_UI.debug("")
if not args.datadir:
blength = 0
for section in settings.config.sections():
for value in settings.config.items(section):
clength = len('%s.%s' % (section, value[0]))
if clength > blength:
blength = clength
format_str = " %-" + str(blength) + "s %s"
LOG_UI.debug(format_str, 'Section.Key', 'Value')
for section in settings.config.sections():
for value in settings.config.items(section):
config_key = ".".join((section, value[0]))
LOG_UI.debug(format_str, config_key, value[1])
else:
LOG_UI.debug("Avocado replaces config dirs that can't be accessed")
LOG_UI.debug("with sensible defaults. Please edit your local config")
LOG_UI.debug("file to customize values")
LOG_UI.debug('')
LOG_UI.info('Avocado Data Directories:')
LOG_UI.debug(' base ' + data_dir.get_base_dir())
LOG_UI.debug(' tests ' + data_dir.get_test_dir())
LOG_UI.debug(' data ' + data_dir.get_data_dir())
LOG_UI.debug(' logs ' + data_dir.get_logs_dir())
LOG_UI.debug(' cache ' + ", ".join(data_dir.get_cache_dirs()))
def fetch_assets(test_file, klass=None, method=None, logger=None):
"""
Fetches the assets based on keywords listed on FetchAssetHandler.calls.
:param test_file: File name of instrumented test to be evaluated
:type test_file: str
:returns: list of names that were successfully fetched and list of
fails.
"""
cache_dirs = data_dir.get_cache_dirs()
success = []
fail = []
handler = FetchAssetHandler(test_file, klass, method)
for call in handler.calls:
expire = call.pop('expire', None)
if expire is not None:
expire = data_structures.time_to_seconds(str(expire))
# make dictionary unpacking compatible with python 3.4 as it does
# not support constructions like:
# Asset(**call, cache_dirs=cache_dirs, expire=expire)
call['cache_dirs'] = cache_dirs
call['expire'] = expire
try:
asset_obj = Asset(**call)
if logger is not None:
logger.info('Fetching asset from %s:%s.%s', test_file, klass,
method)
asset_obj.fetch()
success.append(call['name'])
except (OSError, ValueError) as failed:
fail.append(failed)
return success, fail
def download_image(distro, version=None, arch=None):
"""
Downloads the vmimage to the cache directory if doesn't already exist
:param distro: Name of image distribution
:type distro: str
:param version: Version of image
:type version: str
:param arch: Architecture of image
:type arch: str
:raise AttributeError: When image can't be downloaded
:return: Information about downloaded image
:rtype: dict
"""
cache_dir = data_dir.get_cache_dirs()[0]
image_info = vmimage.get(name=distro,
version=version,
arch=arch,
cache_dir=cache_dir)
file_path = image_info.base_image
image = {
'name': distro,
'version': image_info.version,
'arch': image_info.arch,
'file': file_path
}
return image
def handle_default():
LOG_UI.info("Config files read (in order, '*' means the file exists "
"and had been read):")
# Getting from settings because is already sorted
config = settings.as_dict()
for cfg_path in settings.all_config_paths:
if cfg_path in settings.config_paths:
LOG_UI.debug(' * %s', cfg_path)
else:
LOG_UI.debug(' %s', cfg_path)
LOG_UI.debug("")
if not config.get('config.datadir'):
blength = 0
for namespace, value in config.items():
clength = len(namespace)
if clength > blength:
blength = clength
format_str = " %-" + str(blength) + "s %s"
LOG_UI.debug(format_str, 'Section.Key', 'Value')
for namespace, value in config.items():
LOG_UI.debug(format_str, namespace, value)
else:
LOG_UI.debug("Avocado replaces config dirs that can't be accessed")
LOG_UI.debug("with sensible defaults. Please edit your local config")
LOG_UI.debug("file to customize values")
LOG_UI.debug('')
LOG_UI.info('Avocado Data Directories:')
LOG_UI.debug(' base %s', data_dir.get_base_dir())
LOG_UI.debug(' tests %s', data_dir.get_test_dir())
LOG_UI.debug(' data %s', data_dir.get_data_dir())
LOG_UI.debug(' logs %s', data_dir.get_logs_dir())
LOG_UI.debug(' cache %s', ", ".join(data_dir.get_cache_dirs()))
def run(self, config):
LOG_UI.info("Config files read (in order, '*' means the file exists "
"and had been read):")
for cfg_path in settings.all_config_paths:
if cfg_path in settings.config_paths:
LOG_UI.debug(' * %s', cfg_path)
else:
LOG_UI.debug(' %s', cfg_path)
LOG_UI.debug("")
if not config.get("datadir"):
blength = 0
for section in settings.config.sections():
for value in settings.config.items(section):
clength = len('%s.%s' % (section, value[0]))
if clength > blength:
blength = clength
format_str = " %-" + str(blength) + "s %s"
LOG_UI.debug(format_str, 'Section.Key', 'Value')
for section in settings.config.sections():
for value in settings.config.items(section):
config_key = ".".join((section, value[0]))
LOG_UI.debug(format_str, config_key, value[1])
else:
LOG_UI.debug("Avocado replaces config dirs that can't be accessed")
LOG_UI.debug("with sensible defaults. Please edit your local config")
LOG_UI.debug("file to customize values")
LOG_UI.debug('')
LOG_UI.info('Avocado Data Directories:')
LOG_UI.debug(' base %s', data_dir.get_base_dir())
LOG_UI.debug(' tests %s', data_dir.get_test_dir())
LOG_UI.debug(' data %s', data_dir.get_data_dir())
LOG_UI.debug(' logs %s', data_dir.get_logs_dir())
LOG_UI.debug(' cache %s', ", ".join(data_dir.get_cache_dirs()))
def _create_test_files(self, urlopen_mock):
with unittest.mock.patch('avocado.core.data_dir.settings.settings', self.stg):
expected_images = [{'name': 'Fedora', 'file': 'Fedora-Cloud-Base-{version}-{build}.{arch}.qcow2',
'url': FEDORA_PAGE},
{'name': 'JeOS', 'file': 'jeos-{version}-{arch}.qcow2.xz', 'url': JEOS_PAGE},
{'name': 'CirrOS', 'file': 'cirros-{version}-{arch}-disk.img', 'url': CIRROS_PAGE}
]
cache_dir = data_dir.get_cache_dirs()[0]
providers = [provider() for provider in vmimage_util.list_providers()]
for provider in providers:
for image in expected_images:
if image['name'] == provider.name:
urlread_mocked = unittest.mock.Mock(return_value=image["url"])
urlopen_mock.return_value = unittest.mock.Mock(read=urlread_mocked)
image['type'] = "vmimage"
image['version'] = provider.version
image['arch'] = provider.arch
image['build'] = "1234"
image['file'] = os.path.join(cache_dir, image['file'].format(
version=image['version'],
build=image['build'],
arch=image['arch']))
open(image["file"], "w").close()
create_metadata_file(image['file'], image)
return sorted(expected_images, key=lambda i: i['name'])
def handle_purge(self, config):
days = config.get('assets.purge.days')
size_filter = config.get('assets.purge.size_filter')
overall_limit = config.get('assets.purge.overall_limit')
if self._count_filter_args(config) != 1:
msg = ("You should choose --by-size-filter or --by-days. "
"For help, run: avocado assets purge --help")
LOG_UI.error(msg)
return exit_codes.AVOCADO_FAIL
cache_dirs = data_dir.get_cache_dirs()
try:
if days is not None:
Asset.remove_assets_by_unused_for_days(days, cache_dirs)
elif size_filter is not None:
Asset.remove_assets_by_size(size_filter, cache_dirs)
elif overall_limit is not None:
try:
size = DataSize(overall_limit).b
Asset.remove_assets_by_overall_limit(size, cache_dirs)
except InvalidDataSize:
error_msg = "You are using an invalid suffix. "
error_msg += "Use one of the following values: "
error_msg += ",".join(DataSize.MULTIPLIERS.keys())
LOG_UI.error(error_msg)
return exit_codes.AVOCADO_FAIL
except (FileNotFoundError, OSError) as e:
LOG_UI.error("Could not remove asset: %s", e)
return exit_codes.AVOCADO_FAIL
return exit_codes.AVOCADO_ALL_OK
def run(self, args):
LOG_UI.info("Config files read (in order, '*' means the file exists "
"and had been read):")
for cfg_path in settings.all_config_paths:
if cfg_path in settings.config_paths:
LOG_UI.debug(' * %s', cfg_path)
else:
LOG_UI.debug(' %s', cfg_path)
LOG_UI.debug("")
if not args.datadir:
blength = 0
for section in settings.config.sections():
for value in settings.config.items(section):
clength = len('%s.%s' % (section, value[0]))
if clength > blength:
blength = clength
format_str = " %-" + str(blength) + "s %s"
LOG_UI.debug(format_str, 'Section.Key', 'Value')
for section in settings.config.sections():
for value in settings.config.items(section):
config_key = ".".join((section, value[0]))
LOG_UI.debug(format_str, config_key, value[1])
else:
LOG_UI.debug("Avocado replaces config dirs that can't be accessed")
LOG_UI.debug("with sensible defaults. Please edit your local config")
LOG_UI.debug("file to customize values")
LOG_UI.debug('')
LOG_UI.info('Avocado Data Directories:')
LOG_UI.debug(' base %s', data_dir.get_base_dir())
LOG_UI.debug(' tests %s', data_dir.get_test_dir())
LOG_UI.debug(' data %s', data_dir.get_data_dir())
LOG_UI.debug(' logs %s', data_dir.get_logs_dir())
LOG_UI.debug(' cache %s', ", ".join(data_dir.get_cache_dirs()))
def main():
for image in KNOWN_IMAGES:
name, version, arch, checksum, algorithm = image
print("%s version %s (%s): " % (name, version, arch), end='')
download = vmimage.get(name=name, version=version, arch=arch,
checksum=checksum, algorithm=algorithm,
cache_dir=data_dir.get_cache_dirs()[0])
print(download.base_image)
def fetch_assets(test_file, klass=None, method=None, logger=None):
"""
Fetches the assets based on keywords listed on FetchAssetHandler.calls.
:param test_file: File name of instrumented test to be evaluated
:type test_file: str
:returns: list of names that were successfully fetched and list of
fails.
"""
def validate_parameters(call):
"""
Validate the parameters to make sure we have a supported case.
:param call: List of parameter to the Asset object.
:type call: dict
:returns: True or False
"""
name = call.get('name', None)
locations = call.get('locations', None)
# probably, parameter name was defined as a class attribute
if ((name is None) or
# probably, parameter locations was defined as a class attribute
(not urllib.parse.urlparse(name).scheme and
locations is None)):
return False
return True
cache_dirs = data_dir.get_cache_dirs()
success = []
fail = []
handler = FetchAssetHandler(test_file, klass, method)
for call in handler.calls:
# validate the parameters
if not validate_parameters(call):
continue
expire = call.pop('expire', None)
if expire is not None:
expire = data_structures.time_to_seconds(str(expire))
try:
# make dictionary unpacking compatible with python 3.4 as it does
# not support constructions like:
# Asset(**call, cache_dirs=cache_dirs, expire=expire)
call['cache_dirs'] = cache_dirs
call['expire'] = expire
asset_obj = Asset(**call)
if logger is not None:
logger.info('Fetching asset from %s:%s.%s',
test_file, klass, method)
asset_obj.fetch()
success.append(call['name'])
except (OSError, ValueError) as failed:
fail.append(failed)
return success, fail
def handle_purge(self, config):
days = config.get('assets.purge.days')
size_filter = config.get('assets.purge.size_filter')
if (days is None and size_filter is None) \
or (days is not None and size_filter is not None):
msg = ("You should choose --by-size-filter or --by-days. "
"For help, run: avocado assets purge --help")
LOG_UI.error(msg)
return
cache_dirs = data_dir.get_cache_dirs()
try:
if days is not None:
Asset.remove_assets_by_unused_for_days(days, cache_dirs)
elif size_filter is not None:
Asset.remove_assets_by_size(size_filter, cache_dirs)
except (FileNotFoundError, OSError) as e:
LOG_UI.error("Could not remove asset: %s", e)
def handle_register(self, config):
cache_dirs = data_dir.get_cache_dirs()
name = config.get('assets.register.name')
asset_hash = config.get('assets.register.sha1_hash')
location = config.get('assets.register.url')
# Adding a twice the location is a small hack due the current logic to
# return "by_name". This needs to be improved soon.
asset = Asset(name=name,
asset_hash=asset_hash,
locations=[location, location],
cache_dirs=cache_dirs)
try:
asset.find_asset_file()
LOG_UI.error("Asset with name %s already registered.", name)
except OSError:
try:
asset.fetch()
LOG_UI.info("Done. Now you can reference it by name %s", name)
except OSError as e:
LOG_UI.error(e)
def handle_list(self, config):
days = config.get('assets.list.days')
size_filter = config.get('assets.list.size_filter')
if (days is not None and size_filter is not None):
msg = ("You should choose --by-size-filter or --by-days. "
"For help, run: avocado assets list --help")
LOG_UI.error(msg)
return
# IMO, this should get data from config instead. See #4443
cache_dirs = data_dir.get_cache_dirs()
try:
assets = None
if days is not None:
assets = Asset.get_assets_unused_for_days(days, cache_dirs)
elif size_filter is not None:
assets = Asset.get_assets_by_size(size_filter, cache_dirs)
elif assets is None:
assets = Asset.get_all_assets(cache_dirs)
except (FileNotFoundError, OSError) as e:
LOG_UI.error("Could get assets: %s", e)
return
matrix = []
for asset in assets:
stat = os.stat(asset)
basename = os.path.basename(asset)
hash_path = "{}-CHECKSUM".format(asset)
atime = datetime.fromtimestamp(stat.st_atime)
_, checksum = Asset.read_hash_from_file(hash_path)
matrix.append((basename, str(checksum or "unknown")[:10],
atime.strftime('%Y-%m-%d %H:%M:%S'),
display_data_size(stat.st_size)))
header = ("asset", "checksum", "atime", "size")
output = list(iter_tabular_output(matrix, header))
if len(output) == 1:
LOG_UI.info("No asset found in your cache system.")
else:
for line in output:
LOG_UI.info(line)
def list_downloaded_images():
"""
List the available Image inside avocado cache
:return: list with image's parameters
:rtype: list of dicts
"""
images = []
for cache_dir in data_dir.get_cache_dirs():
for root, _, files in os.walk(cache_dir):
if files:
metadata_files = [
pos_json for pos_json in files
if pos_json.endswith('_metadata.json')
]
files = list(set(files) - set(metadata_files))
for metadata_file in metadata_files:
with open(os.path.join(root, metadata_file), 'r') as data:
metadata = json.loads(data.read())
if isinstance(metadata, dict):
if metadata.get("type", None) == "vmimage":
provider = None
for p in vmimage.IMAGE_PROVIDERS:
if p.name == metadata["name"]:
provider = p(metadata["version"],
metadata["build"],
metadata["arch"])
break
if provider is not None:
for image in files:
if re.match(provider.file_name, image):
data = {
"name": provider.name,
"version": provider.version,
"arch": provider.arch,
"file": os.path.join(root, image)
}
images.append(data)
break
return images
def run(self, args):
LOG_UI.info('Config files read (in order):')
for cfg_path in settings.config_paths:
LOG_UI.debug(' %s' % cfg_path)
if settings.config_paths_failed:
LOG_UI.error('\nConfig files that failed to read:')
for cfg_path in settings.config_paths_failed:
LOG_UI.error(' %s' % cfg_path)
LOG_UI.debug("")
if not args.datadir:
blength = 0
for section in settings.config.sections():
for value in settings.config.items(section):
clength = len('%s.%s' % (section, value[0]))
if clength > blength:
blength = clength
format_str = " %-" + str(blength) + "s %s"
LOG_UI.debug(format_str, 'Section.Key', 'Value')
for section in settings.config.sections():
for value in settings.config.items(section):
config_key = ".".join((section, value[0]))
LOG_UI.debug(format_str, config_key, value[1])
else:
LOG_UI.debug("Avocado replaces config dirs that can't be accessed")
LOG_UI.debug(
"with sensible defaults. Please edit your local config")
LOG_UI.debug("file to customize values")
LOG_UI.debug('')
LOG_UI.info('Avocado Data Directories:')
LOG_UI.debug(' base ' + data_dir.get_base_dir())
LOG_UI.debug(' tests ' + data_dir.get_test_dir())
LOG_UI.debug(' data ' + data_dir.get_data_dir())
LOG_UI.debug(' logs ' + data_dir.get_logs_dir())
LOG_UI.debug(' cache ' +
", ".join(data_dir.get_cache_dirs()))
