def fetch_assets(test_file, klass=None, method=None, logger=None): """Fetches the assets based on keywords listed on FetchAssetHandler.calls. :param test_file: File name of instrumented test to be evaluated :type test_file: str :returns: list of names that were successfully fetched and list of fails. """ cache_dirs = settings.as_dict().get('datadir.paths.cache_dirs') timeout = settings.as_dict().get('assets.fetch.timeout') success = [] fail = [] handler = FetchAssetHandler(test_file, klass, method) for call in handler.calls: expire = call.pop('expire', None) if expire is not None: expire = data_structures.time_to_seconds(str(expire)) try: asset_obj = Asset(**call, cache_dirs=cache_dirs, expire=expire) if logger is not None: logger.info('Fetching asset from %s:%s.%s', test_file, klass, method) asset_obj.fetch(timeout) success.append(call['name']) except (OSError, ValueError) as failed: fail.append(failed) return success, fail
def main(): configure_logging_settings() for version in ['3.6', '3.7', '3.8', '3.9', '3.10']: url = get_egg_url(python_version=version) asset = Asset(url, cache_dirs=CACHE_DIRS) asset.fetch() return True
def fetch_assets(test_file, klass=None, method=None, logger=None): """ Fetches the assets based on keywords listed on FetchAssetHandler.calls. :param test_file: File name of instrumented test to be evaluated :type test_file: str :returns: list of names that were successfully fetched and list of fails. """ cache_dirs = data_dir.get_cache_dirs() success = [] fail = [] handler = FetchAssetHandler(test_file, klass, method) for call in handler.calls: expire = call.pop('expire', None) if expire is not None: expire = data_structures.time_to_seconds(str(expire)) # make dictionary unpacking compatible with python 3.4 as it does # not support constructions like: # Asset(**call, cache_dirs=cache_dirs, expire=expire) call['cache_dirs'] = cache_dirs call['expire'] = expire try: asset_obj = Asset(**call) if logger is not None: logger.info('Fetching asset from %s:%s.%s', test_file, klass, method) asset_obj.fetch() success.append(call['name']) except (OSError, ValueError) as failed: fail.append(failed) return success, fail
def fetch_assets(test_file, klass=None, method=None, logger=None): """ Fetches the assets based on keywords listed on FetchAssetHandler.calls. :param test_file: File name of instrumented test to be evaluated :type test_file: str :returns: list of names that were successfully fetched and list of fails. """ def validate_parameters(call): """ Validate the parameters to make sure we have a supported case. :param call: List of parameter to the Asset object. :type call: dict :returns: True or False """ name = call.get('name', None) locations = call.get('locations', None) # probably, parameter name was defined as a class attribute if ((name is None) or # probably, parameter locations was defined as a class attribute (not urllib.parse.urlparse(name).scheme and locations is None)): return False return True cache_dirs = data_dir.get_cache_dirs() success = [] fail = [] handler = FetchAssetHandler(test_file, klass, method) for call in handler.calls: # validate the parameters if not validate_parameters(call): continue expire = call.pop('expire', None) if expire is not None: expire = data_structures.time_to_seconds(str(expire)) try: # make dictionary unpacking compatible with python 3.4 as it does # not support constructions like: # Asset(**call, cache_dirs=cache_dirs, expire=expire) call['cache_dirs'] = cache_dirs call['expire'] = expire asset_obj = Asset(**call) if logger is not None: logger.info('Fetching asset from %s:%s.%s', test_file, klass, method) asset_obj.fetch() success.append(call['name']) except (OSError, ValueError) as failed: fail.append(failed) return success, fail
def test_asset_purge_by_overall_cache_size(self): """Make sure that we can set cache limits.""" # creates a single byte asset asset_file = tempfile.NamedTemporaryFile(dir=self.base_dir.name, delete=False) asset_file.write(b"\xff") asset_file.close() config = self.config_file.name url = asset_file.name name = "should-be-removed" cmd_line = f"{AVOCADO} --config {config} assets register {name} {url}" result = process.run(cmd_line) self.assertIn(f"Now you can reference it by name {name}", result.stdout_text) cmd_line = f"{AVOCADO} --config {config} assets purge " f"--by-overall-limit 2" process.run(cmd_line) cmd_line = f"{AVOCADO} --config {config} assets list" result = process.run(cmd_line) self.assertIn(name, result.stdout_text) # Double check that the sum of assets is not bigger than 2bytes size_sum = 0 for asset in Asset.get_all_assets([self.mapping["cache_dir"]]): size_sum += os.stat(asset).st_size self.assertLessEqual(size_sum, 2)
def test_asset_purge_by_overall_cache_size(self): """Make sure that we can set cache limits.""" # creates a single byte asset asset_file = tempfile.NamedTemporaryFile(dir=self.base_dir.name, delete=False) asset_file.write(b'\xff') asset_file.close() config = self.config_file.name url = asset_file.name name = "should-be-removed" cmd_line = "%s --config %s assets register %s %s" % (AVOCADO, config, name, url) result = process.run(cmd_line) self.assertIn("Now you can reference it by name {}".format(name), result.stdout_text) cmd_line = "%s --config %s assets purge --by-overall-limit 2" % (AVOCADO, config) process.run(cmd_line) cmd_line = "%s --config %s assets list" % (AVOCADO, config) result = process.run(cmd_line) self.assertIn(name, result.stdout_text) # Double check that the sum of assets is not bigger than 2bytes size_sum = 0 for asset in Asset.get_all_assets([self.mapping['cache_dir']]): size_sum += os.stat(asset).st_size self.assertLessEqual(size_sum, 2)
def handle_purge(self, config): days = config.get('assets.purge.days') size_filter = config.get('assets.purge.size_filter') if (days is None and size_filter is None) \ or (days is not None and size_filter is not None): msg = ("You should choose --by-size-filter or --by-days. " "For help, run: avocado assets purge --help") LOG_UI.error(msg) return cache_dirs = data_dir.get_cache_dirs() try: if days is not None: Asset.remove_assets_by_unused_for_days(days, cache_dirs) elif size_filter is not None: Asset.remove_assets_by_size(size_filter, cache_dirs) except (FileNotFoundError, OSError) as e: LOG_UI.error("Could not remove asset: %s", e)
def _fetch_asset(name, asset_hash, algorithm, locations, cache_dirs, expire, queue): asset_manager = Asset(name, asset_hash, algorithm, locations, cache_dirs, expire) result = 'pass' stdout = '' stderr = '' try: asset_file = asset_manager.fetch() stdout = 'File fetched at %s' % asset_file except OSError as exc: result = 'error' stderr = str(exc) output = {'result': result, 'stdout': stdout, 'stderr': stderr} queue.put(output)
def _fetch_asset(name, asset_hash, algorithm, locations, cache_dirs, expire, queue): asset_manager = Asset(name, asset_hash, algorithm, locations, cache_dirs, expire) result = "pass" stdout = "" stderr = "" try: asset_file = asset_manager.fetch() stdout = f"File fetched at {asset_file}" except OSError as exc: result = "error" stderr = str(exc) output = {"result": result, "stdout": stdout, "stderr": stderr} queue.put(output)
def handle_list(self, config): days = config.get("assets.list.days") size_filter = config.get("assets.list.size_filter") if self._count_filter_args(config) == 2: msg = ("You should choose --by-size-filter or --by-days. " "For help, run: avocado assets list --help") LOG_UI.error(msg) return exit_codes.AVOCADO_FAIL cache_dirs = config.get("datadir.paths.cache_dirs") try: assets = None if days is not None: assets = Asset.get_assets_unused_for_days(days, cache_dirs) elif size_filter is not None: assets = Asset.get_assets_by_size(size_filter, cache_dirs) elif assets is None: assets = Asset.get_all_assets(cache_dirs) except (FileNotFoundError, OSError) as e: LOG_UI.error("Could get assets: %s", e) return exit_codes.AVOCADO_FAIL matrix = [] for asset in assets: stat = os.stat(asset) basename = os.path.basename(asset) hash_path = f"{asset}-CHECKSUM" atime = datetime.fromtimestamp(stat.st_atime) _, checksum = Asset.read_hash_from_file(hash_path) matrix.append(( basename, str(checksum or "unknown")[:10], atime.strftime("%Y-%m-%d %H:%M:%S"), display_data_size(stat.st_size), )) header = ("asset", "checksum", "atime", "size") output = list(iter_tabular_output(matrix, header)) if len(output) == 1: LOG_UI.info("No asset found in your cache system.") else: for line in output: LOG_UI.info(line)
def handle_purge(self, config): days = config.get('assets.purge.days') size_filter = config.get('assets.purge.size_filter') overall_limit = config.get('assets.purge.overall_limit') if self._count_filter_args(config) != 1: msg = ("You should choose --by-size-filter or --by-days. " "For help, run: avocado assets purge --help") LOG_UI.error(msg) return exit_codes.AVOCADO_FAIL cache_dirs = config.get('datadir.paths.cache_dirs') try: if days is not None: Asset.remove_assets_by_unused_for_days(days, cache_dirs) elif size_filter is not None: Asset.remove_assets_by_size(size_filter, cache_dirs) elif overall_limit is not None: try: size = DataSize(overall_limit).b Asset.remove_assets_by_overall_limit(size, cache_dirs) except InvalidDataSize: error_msg = "You are using an invalid suffix. " error_msg += "Use one of the following values: " error_msg += ",".join(DataSize.MULTIPLIERS.keys()) LOG_UI.error(error_msg) return exit_codes.AVOCADO_FAIL except (FileNotFoundError, OSError) as e: LOG_UI.error("Could not remove asset: %s", e) return exit_codes.AVOCADO_FAIL return exit_codes.AVOCADO_ALL_OK
def handle_list(self, config): days = config.get('assets.list.days') size_filter = config.get('assets.list.size_filter') if (days is not None and size_filter is not None): msg = ("You should choose --by-size-filter or --by-days. " "For help, run: avocado assets list --help") LOG_UI.error(msg) return # IMO, this should get data from config instead. See #4443 cache_dirs = data_dir.get_cache_dirs() try: assets = None if days is not None: assets = Asset.get_assets_unused_for_days(days, cache_dirs) elif size_filter is not None: assets = Asset.get_assets_by_size(size_filter, cache_dirs) elif assets is None: assets = Asset.get_all_assets(cache_dirs) except (FileNotFoundError, OSError) as e: LOG_UI.error("Could get assets: %s", e) return matrix = [] for asset in assets: stat = os.stat(asset) basename = os.path.basename(asset) hash_path = "{}-CHECKSUM".format(asset) atime = datetime.fromtimestamp(stat.st_atime) _, checksum = Asset.read_hash_from_file(hash_path) matrix.append((basename, str(checksum or "unknown")[:10], atime.strftime('%Y-%m-%d %H:%M:%S'), display_data_size(stat.st_size))) header = ("asset", "checksum", "atime", "size") output = list(iter_tabular_output(matrix, header)) if len(output) == 1: LOG_UI.info("No asset found in your cache system.") else: for line in output: LOG_UI.info(line)
def handle_register(config): cache_dirs = config.get("datadir.paths.cache_dirs") name = config.get("assets.register.name") asset_hash = config.get("assets.register.sha1_hash") location = config.get("assets.register.url") # Adding a twice the location is a small hack due the current logic to # return "by_name". This needs to be improved soon. asset = Asset( name=name, asset_hash=asset_hash, locations=[location, location], cache_dirs=cache_dirs, ) try: asset.find_asset_file() LOG_UI.error("Asset with name %s already registered.", name) return exit_codes.AVOCADO_WARNING except OSError: try: asset.fetch() LOG_UI.info("Done. Now you can reference it by name %s", name) return exit_codes.AVOCADO_ALL_OK except OSError as e: LOG_UI.error(e) return exit_codes.AVOCADO_FAIL
def handle_register(self, config): cache_dirs = data_dir.get_cache_dirs() name = config.get('assets.register.name') asset_hash = config.get('assets.register.sha1_hash') location = config.get('assets.register.url') # Adding a twice the location is a small hack due the current logic to # return "by_name". This needs to be improved soon. asset = Asset(name=name, asset_hash=asset_hash, locations=[location, location], cache_dirs=cache_dirs) try: asset.find_asset_file() LOG_UI.error("Asset with name %s already registered.", name) except OSError: try: asset.fetch() LOG_UI.info("Done. Now you can reference it by name %s", name) except OSError as e: LOG_UI.error(e)
def _fetch_asset(self, url): cachedirs = self.config.get('datadir.paths.cache_dirs') asset = Asset(url, cache_dirs=cachedirs) return asset.fetch()