def fetch_assets(test_file, klass=None, method=None, logger=None):
"""Fetches the assets based on keywords listed on FetchAssetHandler.calls.
:param test_file: File name of instrumented test to be evaluated
:type test_file: str
:returns: list of names that were successfully fetched and list of
fails.
"""
cache_dirs = settings.as_dict().get('datadir.paths.cache_dirs')
timeout = settings.as_dict().get('assets.fetch.timeout')
success = []
fail = []
handler = FetchAssetHandler(test_file, klass, method)
for call in handler.calls:
expire = call.pop('expire', None)
if expire is not None:
expire = data_structures.time_to_seconds(str(expire))
try:
asset_obj = Asset(**call, cache_dirs=cache_dirs, expire=expire)
if logger is not None:
logger.info('Fetching asset from %s:%s.%s', test_file, klass,
method)
asset_obj.fetch(timeout)
success.append(call['name'])
except (OSError, ValueError) as failed:
fail.append(failed)
return success, fail
def main():
configure_logging_settings()
for version in ['3.6', '3.7', '3.8', '3.9', '3.10']:
url = get_egg_url(python_version=version)
asset = Asset(url, cache_dirs=CACHE_DIRS)
asset.fetch()
return True
def fetch_assets(test_file, klass=None, method=None, logger=None):
"""
Fetches the assets based on keywords listed on FetchAssetHandler.calls.
:param test_file: File name of instrumented test to be evaluated
:type test_file: str
:returns: list of names that were successfully fetched and list of
fails.
"""
cache_dirs = data_dir.get_cache_dirs()
success = []
fail = []
handler = FetchAssetHandler(test_file, klass, method)
for call in handler.calls:
expire = call.pop('expire', None)
if expire is not None:
expire = data_structures.time_to_seconds(str(expire))
# make dictionary unpacking compatible with python 3.4 as it does
# not support constructions like:
# Asset(**call, cache_dirs=cache_dirs, expire=expire)
call['cache_dirs'] = cache_dirs
call['expire'] = expire
try:
asset_obj = Asset(**call)
if logger is not None:
logger.info('Fetching asset from %s:%s.%s', test_file, klass,
method)
asset_obj.fetch()
success.append(call['name'])
except (OSError, ValueError) as failed:
fail.append(failed)
return success, fail
def fetch_assets(test_file, klass=None, method=None, logger=None):
"""
Fetches the assets based on keywords listed on FetchAssetHandler.calls.
:param test_file: File name of instrumented test to be evaluated
:type test_file: str
:returns: list of names that were successfully fetched and list of
fails.
"""
def validate_parameters(call):
"""
Validate the parameters to make sure we have a supported case.
:param call: List of parameter to the Asset object.
:type call: dict
:returns: True or False
"""
name = call.get('name', None)
locations = call.get('locations', None)
# probably, parameter name was defined as a class attribute
if ((name is None) or
# probably, parameter locations was defined as a class attribute
(not urllib.parse.urlparse(name).scheme and
locations is None)):
return False
return True
cache_dirs = data_dir.get_cache_dirs()
success = []
fail = []
handler = FetchAssetHandler(test_file, klass, method)
for call in handler.calls:
# validate the parameters
if not validate_parameters(call):
continue
expire = call.pop('expire', None)
if expire is not None:
expire = data_structures.time_to_seconds(str(expire))
try:
# make dictionary unpacking compatible with python 3.4 as it does
# not support constructions like:
# Asset(**call, cache_dirs=cache_dirs, expire=expire)
call['cache_dirs'] = cache_dirs
call['expire'] = expire
asset_obj = Asset(**call)
if logger is not None:
logger.info('Fetching asset from %s:%s.%s',
test_file, klass, method)
asset_obj.fetch()
success.append(call['name'])
except (OSError, ValueError) as failed:
fail.append(failed)
return success, fail
def test_asset_purge_by_overall_cache_size(self):
"""Make sure that we can set cache limits."""
# creates a single byte asset
asset_file = tempfile.NamedTemporaryFile(dir=self.base_dir.name, delete=False)
asset_file.write(b"\xff")
asset_file.close()
config = self.config_file.name
url = asset_file.name
name = "should-be-removed"
cmd_line = f"{AVOCADO} --config {config} assets register {name} {url}"
result = process.run(cmd_line)
self.assertIn(f"Now you can reference it by name {name}", result.stdout_text)
cmd_line = f"{AVOCADO} --config {config} assets purge " f"--by-overall-limit 2"
process.run(cmd_line)
cmd_line = f"{AVOCADO} --config {config} assets list"
result = process.run(cmd_line)
self.assertIn(name, result.stdout_text)
# Double check that the sum of assets is not bigger than 2bytes
size_sum = 0
for asset in Asset.get_all_assets([self.mapping["cache_dir"]]):
size_sum += os.stat(asset).st_size
self.assertLessEqual(size_sum, 2)
def test_asset_purge_by_overall_cache_size(self):
"""Make sure that we can set cache limits."""
# creates a single byte asset
asset_file = tempfile.NamedTemporaryFile(dir=self.base_dir.name, delete=False)
asset_file.write(b'\xff')
asset_file.close()
config = self.config_file.name
url = asset_file.name
name = "should-be-removed"
cmd_line = "%s --config %s assets register %s %s" % (AVOCADO,
config,
name,
url)
result = process.run(cmd_line)
self.assertIn("Now you can reference it by name {}".format(name),
result.stdout_text)
cmd_line = "%s --config %s assets purge --by-overall-limit 2" % (AVOCADO,
config)
process.run(cmd_line)
cmd_line = "%s --config %s assets list" % (AVOCADO, config)
result = process.run(cmd_line)
self.assertIn(name, result.stdout_text)
# Double check that the sum of assets is not bigger than 2bytes
size_sum = 0
for asset in Asset.get_all_assets([self.mapping['cache_dir']]):
size_sum += os.stat(asset).st_size
self.assertLessEqual(size_sum, 2)
def handle_purge(self, config):
days = config.get('assets.purge.days')
size_filter = config.get('assets.purge.size_filter')
if (days is None and size_filter is None) \
or (days is not None and size_filter is not None):
msg = ("You should choose --by-size-filter or --by-days. "
"For help, run: avocado assets purge --help")
LOG_UI.error(msg)
return
cache_dirs = data_dir.get_cache_dirs()
try:
if days is not None:
Asset.remove_assets_by_unused_for_days(days, cache_dirs)
elif size_filter is not None:
Asset.remove_assets_by_size(size_filter, cache_dirs)
except (FileNotFoundError, OSError) as e:
LOG_UI.error("Could not remove asset: %s", e)
def _fetch_asset(name, asset_hash, algorithm, locations, cache_dirs,
expire, queue):
asset_manager = Asset(name, asset_hash, algorithm, locations,
cache_dirs, expire)
result = 'pass'
stdout = ''
stderr = ''
try:
asset_file = asset_manager.fetch()
stdout = 'File fetched at %s' % asset_file
except OSError as exc:
result = 'error'
stderr = str(exc)
output = {'result': result, 'stdout': stdout, 'stderr': stderr}
queue.put(output)
def _fetch_asset(name, asset_hash, algorithm, locations, cache_dirs,
expire, queue):
asset_manager = Asset(name, asset_hash, algorithm, locations,
cache_dirs, expire)
result = "pass"
stdout = ""
stderr = ""
try:
asset_file = asset_manager.fetch()
stdout = f"File fetched at {asset_file}"
except OSError as exc:
result = "error"
stderr = str(exc)
output = {"result": result, "stdout": stdout, "stderr": stderr}
queue.put(output)
def handle_list(self, config):
days = config.get("assets.list.days")
size_filter = config.get("assets.list.size_filter")
if self._count_filter_args(config) == 2:
msg = ("You should choose --by-size-filter or --by-days. "
"For help, run: avocado assets list --help")
LOG_UI.error(msg)
return exit_codes.AVOCADO_FAIL
cache_dirs = config.get("datadir.paths.cache_dirs")
try:
assets = None
if days is not None:
assets = Asset.get_assets_unused_for_days(days, cache_dirs)
elif size_filter is not None:
assets = Asset.get_assets_by_size(size_filter, cache_dirs)
elif assets is None:
assets = Asset.get_all_assets(cache_dirs)
except (FileNotFoundError, OSError) as e:
LOG_UI.error("Could get assets: %s", e)
return exit_codes.AVOCADO_FAIL
matrix = []
for asset in assets:
stat = os.stat(asset)
basename = os.path.basename(asset)
hash_path = f"{asset}-CHECKSUM"
atime = datetime.fromtimestamp(stat.st_atime)
_, checksum = Asset.read_hash_from_file(hash_path)
matrix.append((
basename,
str(checksum or "unknown")[:10],
atime.strftime("%Y-%m-%d %H:%M:%S"),
display_data_size(stat.st_size),
))
header = ("asset", "checksum", "atime", "size")
output = list(iter_tabular_output(matrix, header))
if len(output) == 1:
LOG_UI.info("No asset found in your cache system.")
else:
for line in output:
LOG_UI.info(line)
def handle_purge(self, config):
days = config.get('assets.purge.days')
size_filter = config.get('assets.purge.size_filter')
overall_limit = config.get('assets.purge.overall_limit')
if self._count_filter_args(config) != 1:
msg = ("You should choose --by-size-filter or --by-days. "
"For help, run: avocado assets purge --help")
LOG_UI.error(msg)
return exit_codes.AVOCADO_FAIL
cache_dirs = config.get('datadir.paths.cache_dirs')
try:
if days is not None:
Asset.remove_assets_by_unused_for_days(days, cache_dirs)
elif size_filter is not None:
Asset.remove_assets_by_size(size_filter, cache_dirs)
elif overall_limit is not None:
try:
size = DataSize(overall_limit).b
Asset.remove_assets_by_overall_limit(size, cache_dirs)
except InvalidDataSize:
error_msg = "You are using an invalid suffix. "
error_msg += "Use one of the following values: "
error_msg += ",".join(DataSize.MULTIPLIERS.keys())
LOG_UI.error(error_msg)
return exit_codes.AVOCADO_FAIL
except (FileNotFoundError, OSError) as e:
LOG_UI.error("Could not remove asset: %s", e)
return exit_codes.AVOCADO_FAIL
return exit_codes.AVOCADO_ALL_OK
def handle_list(self, config):
days = config.get('assets.list.days')
size_filter = config.get('assets.list.size_filter')
if (days is not None and size_filter is not None):
msg = ("You should choose --by-size-filter or --by-days. "
"For help, run: avocado assets list --help")
LOG_UI.error(msg)
return
# IMO, this should get data from config instead. See #4443
cache_dirs = data_dir.get_cache_dirs()
try:
assets = None
if days is not None:
assets = Asset.get_assets_unused_for_days(days, cache_dirs)
elif size_filter is not None:
assets = Asset.get_assets_by_size(size_filter, cache_dirs)
elif assets is None:
assets = Asset.get_all_assets(cache_dirs)
except (FileNotFoundError, OSError) as e:
LOG_UI.error("Could get assets: %s", e)
return
matrix = []
for asset in assets:
stat = os.stat(asset)
basename = os.path.basename(asset)
hash_path = "{}-CHECKSUM".format(asset)
atime = datetime.fromtimestamp(stat.st_atime)
_, checksum = Asset.read_hash_from_file(hash_path)
matrix.append((basename, str(checksum or "unknown")[:10],
atime.strftime('%Y-%m-%d %H:%M:%S'),
display_data_size(stat.st_size)))
header = ("asset", "checksum", "atime", "size")
output = list(iter_tabular_output(matrix, header))
if len(output) == 1:
LOG_UI.info("No asset found in your cache system.")
else:
for line in output:
LOG_UI.info(line)
def handle_register(config):
cache_dirs = config.get("datadir.paths.cache_dirs")
name = config.get("assets.register.name")
asset_hash = config.get("assets.register.sha1_hash")
location = config.get("assets.register.url")
# Adding a twice the location is a small hack due the current logic to
# return "by_name". This needs to be improved soon.
asset = Asset(
name=name,
asset_hash=asset_hash,
locations=[location, location],
cache_dirs=cache_dirs,
)
try:
asset.find_asset_file()
LOG_UI.error("Asset with name %s already registered.", name)
return exit_codes.AVOCADO_WARNING
except OSError:
try:
asset.fetch()
LOG_UI.info("Done. Now you can reference it by name %s", name)
return exit_codes.AVOCADO_ALL_OK
except OSError as e:
LOG_UI.error(e)
return exit_codes.AVOCADO_FAIL
def handle_register(self, config):
cache_dirs = data_dir.get_cache_dirs()
name = config.get('assets.register.name')
asset_hash = config.get('assets.register.sha1_hash')
location = config.get('assets.register.url')
# Adding a twice the location is a small hack due the current logic to
# return "by_name". This needs to be improved soon.
asset = Asset(name=name,
asset_hash=asset_hash,
locations=[location, location],
cache_dirs=cache_dirs)
try:
asset.find_asset_file()
LOG_UI.error("Asset with name %s already registered.", name)
except OSError:
try:
asset.fetch()
LOG_UI.info("Done. Now you can reference it by name %s", name)
except OSError as e:
LOG_UI.error(e)
def _fetch_asset(self, url):
cachedirs = self.config.get('datadir.paths.cache_dirs')
asset = Asset(url, cache_dirs=cachedirs)
return asset.fetch()
