def remove_caches(cls, options): """Remove all or selected caches.""" force = getattr(options, 'force_cache', False) if force: try: shutil.rmtree(CACHE_DIR) except FileNotFoundError: pass except IOError as e: raise UserException(f'failed removing cache dir: {e}') else: try: for cache_type, paths in cls.existing().items(): if cache_type in options.cache_types: for path in paths: if options.dry_run: print(f'Would remove {path}') else: path.unlink() # remove empty cache dirs try: while str(path) != CACHE_DIR: path.parent.rmdir() path = path.parent except OSError as e: if e.errno == errno.ENOTEMPTY: continue raise except IOError as e: raise UserException( f'failed removing {cache_type} cache: {path!r}: {e}') return 0
def _replay(options, out, err): # assume JSON encoded file, fallback to pickle format processed = 0 exc = None with options.reporter(out) as reporter: try: for result in reporters.JsonStream.from_file(options.results): reporter.report(result) processed += 1 except reporters.DeserializationError as e: if not processed: options.results.seek(0) try: for result in reporters.PickleStream.from_file( options.results): reporter.report(result) processed += 1 except reporters.DeserializationError as e: exc = e else: exc = e if exc: if not processed: raise UserException('invalid or unsupported replay file') raise UserException( f'corrupted results file {options.results.name!r}: {exc}') return 0
def _path_restrict(path, namespace): """Generate custom package restriction from a given path. This drops the repo restriction (initial entry in path restrictions) since runs can only be made against single repo targets so the extra restriction is redundant and breaks several custom sources involving raw pkgs (lacking a repo attr) or faked repos. """ repo = namespace.target_repo restrictions = [] path = os.path.realpath(path) try: restrictions = repo.path_restrict(path)[1:] except ValueError as e: raise UserException(str(e)) restrict = packages.AndRestriction( *restrictions) if restrictions else packages.AlwaysTrue # allow location specific scopes to override the path restrict scope for scope in (x for x in base.scopes.values() if x.level == 0): scope_path = pjoin(namespace.target_repo.location, scope.desc) if path.startswith(scope_path): break else: scope = _restrict_to_scope(restrict) return scope, restrict
def _kill_pipe(self, *args, error=None): """Handle terminating the mangling process group.""" if self._runner.is_alive(): os.killpg(self._runner.pid, signal.SIGKILL) if error is not None: # propagate exception raised during parallelized mangling raise UserException(error) raise KeyboardInterrupt
def parse_config(self, configs): """Parse given config files.""" self.config = configparser.ConfigParser() try: for f in configs: self.config.read(f) except configparser.ParsingError as e: raise UserException(f'parsing config file failed: {e}') return self.config
def config(self): """Config file object related to a given parser.""" config = configparser.ConfigParser() try: for f in self.configs: config.read(f) except configparser.ParsingError as e: raise UserException(f'parsing config file failed: {e}') return config
def __init__(self, *args): super().__init__(*args) try: from .net import Session self.session = Session( concurrent=self.options.tasks, timeout=self.options.timeout) except ImportError as e: if e.name == 'requests': raise UserException('network checks require requests to be installed') raise
def run(*args, **kwargs): """Wrapper for running git via subprocess.run().""" kwargs.setdefault('check', True) kwargs.setdefault('text', True) cmd = ['git'] + list(args) # output git command that would be run to stderr if '--dry-run' in args: git_cmd = ' '.join(x for x in cmd if x != '--dry-run') sys.stderr.write(f'{git_cmd}\n') try: return subprocess.run(cmd, **kwargs) except FileNotFoundError as e: raise UserException(str(e)) except subprocess.CalledProcessError as e: raise GitError(e.returncode)
def _path_restrict(path, namespace): """Generate custom package restriction from a given path. This drops the repo restriction (initial entry in path restrictions) since runs can only be made against single repo targets so the extra restriction is redundant and breaks several custom sources involving raw pkgs (lacking a repo attr) or faked repos. """ repo = namespace.target_repo restrictions = [] try: restrictions = repo.path_restrict(path)[1:] except ValueError as e: raise UserException(str(e)) if restrictions: return packages.AndRestriction(*restrictions) return packages.AlwaysTrue
def __init__(self, *args, arches_addon=None): super().__init__(*args) target_repo = self.options.target_repo self.official_arches = target_repo.known_arches self.desired_arches = getattr(self.options, 'arches', None) if self.desired_arches is None or self.options.selected_arches is None: # copy it to be safe self.desired_arches = set(self.official_arches) self.global_insoluble = set() profile_filters = defaultdict(list) chunked_data_cache = {} cached_profiles = defaultdict(dict) if self.options.cache['profiles']: for repo in target_repo.trees: cache_file = self.cache_file(repo) # add profiles-base -> repo mapping to ease storage procedure cached_profiles[repo.config.profiles_base]['repo'] = repo try: with open(cache_file, 'rb') as f: cache = pickle.load(f) if cache.version == self.cache.version: cached_profiles[repo.config.profiles_base].update( cache) else: logger.debug( 'forcing %s profile cache regen ' 'due to outdated version', repo.repo_id) os.remove(cache_file) except FileNotFoundError as e: pass except (AttributeError, EOFError, ImportError, IndexError) as e: logger.debug('forcing %s profile cache regen: %s', repo.repo_id, e) os.remove(cache_file) for k in sorted(self.desired_arches): if k.lstrip("~") not in self.desired_arches: continue stable_key = k.lstrip("~") unstable_key = "~" + stable_key stable_r = packages.PackageRestriction( "keywords", values.ContainmentMatch2((stable_key, ))) unstable_r = packages.PackageRestriction( "keywords", values.ContainmentMatch2(( stable_key, unstable_key, ))) default_masked_use = tuple( set(x for x in self.official_arches if x != stable_key)) for profile_obj, profile in self.options.arch_profiles.get(k, []): files = self.profile_data.get(profile, None) try: cached_profile = cached_profiles[profile.base][ profile.path] if files != cached_profile['files']: # force refresh of outdated cache entry raise KeyError masks = cached_profile['masks'] unmasks = cached_profile['unmasks'] immutable_flags = cached_profile['immutable_flags'] stable_immutable_flags = cached_profile[ 'stable_immutable_flags'] enabled_flags = cached_profile['enabled_flags'] stable_enabled_flags = cached_profile[ 'stable_enabled_flags'] pkg_use = cached_profile['pkg_use'] iuse_effective = cached_profile['iuse_effective'] use = cached_profile['use'] provides_repo = cached_profile['provides_repo'] except KeyError: logger.debug('profile regen: %s', profile.path) try: masks = profile_obj.masks unmasks = profile_obj.unmasks immutable_flags = profile_obj.masked_use.clone( unfreeze=True) immutable_flags.add_bare_global((), default_masked_use) immutable_flags.optimize(cache=chunked_data_cache) immutable_flags.freeze() stable_immutable_flags = profile_obj.stable_masked_use.clone( unfreeze=True) stable_immutable_flags.add_bare_global( (), default_masked_use) stable_immutable_flags.optimize( cache=chunked_data_cache) stable_immutable_flags.freeze() enabled_flags = profile_obj.forced_use.clone( unfreeze=True) enabled_flags.add_bare_global((), (stable_key, )) enabled_flags.optimize(cache=chunked_data_cache) enabled_flags.freeze() stable_enabled_flags = profile_obj.stable_forced_use.clone( unfreeze=True) stable_enabled_flags.add_bare_global((), (stable_key, )) stable_enabled_flags.optimize(cache=chunked_data_cache) stable_enabled_flags.freeze() pkg_use = profile_obj.pkg_use iuse_effective = profile_obj.iuse_effective provides_repo = profile_obj.provides_repo # finalize enabled USE flags use = set() misc.incremental_expansion(use, profile_obj.use, 'while expanding USE') use = frozenset(use) except profiles_mod.ProfileError: # unsupported EAPI or other issue, profile checks will catch this continue if self.options.cache['profiles']: cached_profiles[profile.base]['update'] = True cached_profiles[profile.base][profile.path] = { 'files': files, 'masks': masks, 'unmasks': unmasks, 'immutable_flags': immutable_flags, 'stable_immutable_flags': stable_immutable_flags, 'enabled_flags': enabled_flags, 'stable_enabled_flags': stable_enabled_flags, 'pkg_use': pkg_use, 'iuse_effective': iuse_effective, 'use': use, 'provides_repo': provides_repo, } # used to interlink stable/unstable lookups so that if # unstable says it's not visible, stable doesn't try # if stable says something is visible, unstable doesn't try. stable_cache = set() unstable_insoluble = ProtectedSet(self.global_insoluble) # few notes. for filter, ensure keywords is last, on the # offchance a non-metadata based restrict foregos having to # access the metadata. # note that the cache/insoluble are inversly paired; # stable cache is usable for unstable, but not vice versa. # unstable insoluble is usable for stable, but not vice versa vfilter = domain.generate_filter(target_repo.pkg_masks | masks, unmasks) profile_filters[stable_key].append( ProfileData(profile.path, stable_key, provides_repo, packages.AndRestriction(vfilter, stable_r), iuse_effective, use, pkg_use, stable_immutable_flags, stable_enabled_flags, stable_cache, ProtectedSet(unstable_insoluble), profile.status, profile.deprecated)) profile_filters[unstable_key].append( ProfileData(profile.path, unstable_key, provides_repo, packages.AndRestriction(vfilter, unstable_r), iuse_effective, use, pkg_use, immutable_flags, enabled_flags, ProtectedSet(stable_cache), unstable_insoluble, profile.status, profile.deprecated)) # dump updated profile filters for k, v in cached_profiles.items(): if v.pop('update', False): repo = v.pop('repo') cache_file = self.cache_file(repo) try: os.makedirs(os.path.dirname(cache_file), exist_ok=True) with open(cache_file, 'wb+') as f: pickle.dump( _ProfilesCache( cached_profiles[repo.config.profiles_base]), f) except IOError as e: msg = (f'failed dumping {repo.repo_id} profiles cache: ' f'{cache_file!r}: {e.strerror}') raise UserException(msg) profile_evaluate_dict = {} for key, profile_list in profile_filters.items(): similar = profile_evaluate_dict[key] = [] for profile in profile_list: for existing in similar: if (existing[0].masked_use == profile.masked_use and existing[0].forced_use == profile.forced_use): existing.append(profile) break else: similar.append([profile]) self.profile_evaluate_dict = profile_evaluate_dict self.profile_filters = profile_filters
def update_cache(self, output_lock, force=False): """Update related cache and push updates to disk.""" try: # running from scan subcommand repos = self.options.target_repo.trees except AttributeError: # running from cache subcommand repos = self.options.domain.ebuild_repos if self.options.cache['git']: for repo in repos: try: commit = self.get_commit_hash(repo.location) except ValueError as e: continue # initialize cache file location cache_file = self.cache_file(repo) git_repo = None cache_repo = True if not force: # try loading cached, historical repo data try: with open(cache_file, 'rb') as f: git_repo = pickle.load(f) if git_repo.version != self.cache.version: logger.debug( 'forcing git repo cache regen due to outdated version' ) os.remove(cache_file) git_repo = None except FileNotFoundError as e: pass except (AttributeError, EOFError, ImportError, IndexError) as e: logger.debug('forcing git repo cache regen: %s', e) os.remove(cache_file) git_repo = None if (git_repo is not None and repo.location == getattr( git_repo, 'location', None)): if commit != git_repo.commit: with output_lock: old, new = git_repo.commit[:13], commit[:13] print( f'updating {repo} git repo cache: {old} -> {new}', file=sys.stderr, ) git_repo.update(commit, debug=self.options.debug) else: cache_repo = False else: with output_lock: print( f'creating {repo} git repo cache: {commit[:13]}', file=sys.stderr, ) git_repo = ParsedGitRepo(repo, commit, debug=self.options.debug) if git_repo: self._cached_repos[repo.location] = git_repo # push repo to disk if it was created or updated if cache_repo: try: os.makedirs(os.path.dirname(cache_file), exist_ok=True) with open(cache_file, 'wb+') as f: pickle.dump(git_repo, f) except IOError as e: msg = f'failed dumping git pkg repo: {cache_file!r}: {e.strerror}' raise UserException(msg)
def check_args(parser, namespace): profiles_dir = getattr(namespace, "profiles_dir", None) if profiles_dir is not None: profiles_dir = abspath(profiles_dir) if not os.path.isdir(profiles_dir): parser.error(f"invalid profiles base: {profiles_dir!r}") selected_profiles = namespace.profiles if selected_profiles is None: selected_profiles = ((), ()) if profiles_dir: profiles_obj = repo_objs.BundledProfiles(profiles_dir) else: profiles_obj = namespace.target_repo.config.profiles def norm_name(s): """Expand status keywords and format paths.""" if s in ('dev', 'exp', 'stable', 'deprecated'): for x in profiles_obj.paths(s): yield x else: yield '/'.join([_f for _f in s.split('/') if _f]) disabled, enabled = selected_profiles disabled = set(disabled) enabled = set(enabled) # remove profiles that are both enabled and disabled toggled = enabled.intersection(disabled) enabled = enabled.difference(toggled) disabled = disabled.difference(toggled) ignore_deprecated = 'deprecated' not in enabled # expand status keywords, e.g. 'stable' -> set of stable profiles disabled = set(chain.from_iterable(map(norm_name, disabled))) enabled = set(chain.from_iterable(map(norm_name, enabled))) # If no profiles are enabled, then all that are defined in # profiles.desc are scanned except ones that are explicitly disabled. if not enabled: enabled = { profile for profile, status in chain.from_iterable( profiles_obj.arch_profiles.values()) } profile_paths = enabled.difference(disabled) # only default to using cache when run without target args within a repo if namespace.cache is None and namespace.default_target is None: namespace.cache = False # initialize cache dir cache_dir = pjoin(const.USER_CACHE_PATH, 'pkgcheck') namespace.cache_file = pjoin(cache_dir, 'profiles.pickle') if ((namespace.cache is None or namespace.cache) and not os.path.exists(cache_dir)): try: os.makedirs(cache_dir) except IOError as e: raise UserException( f'failed creating profiles cache: {cache_dir!r}: {e.strerror}' ) namespace.forced_cache = bool(namespace.cache) # We hold onto the profiles as we're going, due to the fact that # profile nodes are weakly cached; hold onto all for this loop, avoids # a lot of reparsing at the expense of slightly more memory usage # temporarily. cached_profiles = [] arch_profiles = defaultdict(list) for profile_path in profile_paths: try: p = profiles_obj.create_profile(profile_path) except profiles.ProfileError as e: # Only throw errors if the profile was selected by the user, bad # repo profiles will be caught during repo metadata scans. if namespace.profiles is not None: parser.error(f'invalid profile: {e.path!r}: {e.error}') continue if ignore_deprecated and p.deprecated: continue cached_profiles.append(p) if p.arch is None: parser.error( f"profile {p.path!r} lacks arch settings, unable to use it" ) arch_profiles[p.arch].append((profile_path, p)) namespace.arch_profiles = arch_profiles