def __init__(self, *args, stable_arches_addon=None): super().__init__(*args) arches = {x.strip().lstrip("~") for x in self.options.stable_arches} # stable, then unstable, then file self.arch_restricts = {} for arch in arches: self.arch_restricts[arch] = [ packages.PackageRestriction( "keywords", values.ContainmentMatch2((arch,))), packages.PackageRestriction( "keywords", values.ContainmentMatch2((f"~{arch}",))) ]
def generate_intersects_from_pkg_node(self, pkg_node, tag=None): arch = pkg_node.get("arch") if arch is not None: arch = tuple(str(arch.strip()).split()) if not arch or "*" in arch: arch = None vuln = list(pkg_node.findall("vulnerable")) if not vuln: return None elif len(vuln) > 1: vuln_list = [self.generate_restrict_from_range(x) for x in vuln] vuln = packages.OrRestriction(*vuln_list) else: vuln_list = [self.generate_restrict_from_range(vuln[0])] vuln = vuln_list[0] if arch is not None: vuln = packages.AndRestriction( vuln, packages.PackageRestriction( "keywords", values.ContainmentMatch2(arch, match_all=False))) invuln = (pkg_node.findall("unaffected")) if not invuln: # wrap it. return packages.KeyedAndRestriction(vuln, tag=tag) invuln_list = [ self.generate_restrict_from_range(x, negate=True) for x in invuln ] invuln = [x for x in invuln_list if x not in vuln_list] if not invuln: if tag is None: return packages.KeyedAndRestriction(vuln, tag=tag) return packages.KeyedAndRestriction(vuln, tag=tag) return packages.KeyedAndRestriction(vuln, tag=tag, *invuln)
def _make_keywords_filter(self, default_keys, accept_keywords, incremental=False): """Generates a restrict that matches iff the keywords are allowed.""" if not accept_keywords and not self.profile.keywords: return packages.PackageRestriction( "keywords", values.ContainmentMatch2(frozenset(default_keys))) if self.unstable_arch not in default_keys: # stable; thus empty entries == ~arch def f(r, v): if not v: return r, self.unstable_arch return r, v data = collapsed_restrict_to_data( ((packages.AlwaysTrue, default_keys),), (f(*i) for i in accept_keywords)) else: if incremental: f = collapsed_restrict_to_data else: f = non_incremental_collapsed_restrict_to_data data = f(((packages.AlwaysTrue, default_keys),), accept_keywords) if incremental: raise NotImplementedError(self._incremental_apply_keywords_filter) #f = self._incremental_apply_keywords_filter else: f = self._apply_keywords_filter return delegate(partial(f, data))
def __init__(self, false_use, true_use): v = [] if false_use: v.append(values.ContainmentMatch2(false_use, negate=True, match_all=True)) if true_use: v.append(values.ContainmentMatch2(true_use, match_all=True)) l = len(v) if l == 2: v = values.AndRestriction(*v) elif l == 1: v = v[0] else: v = values.AlwaysTrue super().__init__('use', v)
class PerlCheck(Check): """Perl ebuild related checks.""" _restricted_source = (sources.RestrictionRepoSource, (packages.PackageRestriction( 'inherited', values.ContainmentMatch2('perl-module')), )) _source = (sources.EbuildFileRepoSource, (), (('source', _restricted_source), )) known_results = frozenset([MismatchedPerlVersion]) def __init__(self, *args): super().__init__(*args) self.dist_version_re = re.compile( 'DIST_VERSION=(?P<dist_version>\d+(\.\d+)*)\s*\n') # Initialize connection with perl script. This is done during # __init__() since only one running version of the script is shared # between however many scanning processes will be run. Also, it makes # it easier to disable this check if required perl deps are missing. self.perl = _PerlConnection(self.options) def feed(self, pkg): match = self.dist_version_re.search(''.join(pkg.lines)) if match is not None: dist_version = match.group('dist_version') normalized = self.perl.normalize(dist_version) if normalized != pkg.version: yield MismatchedPerlVersion(dist_version, normalized, pkg=pkg)
def __init__(self, options, stable_arches=None): super().__init__(options) arches = frozenset(arch.strip().lstrip("~") for arch in options.stable_arches) self.target_arches = frozenset( "~%s" % arch.strip().lstrip("~") for arch in arches) source_arches = options.source_arches if source_arches is None: source_arches = options.stable_arches self.source_arches = frozenset( arch.lstrip("~") for arch in source_arches) self.source_filter = packages.PackageRestriction( "keywords", values.ContainmentMatch2(self.source_arches))
def __init__(self, *args, stable_arches_addon=None): super().__init__(*args) self.all_arches = frozenset(self.options.arches) self.stable_arches = frozenset(arch.strip().lstrip("~") for arch in self.options.stable_arches) self.target_arches = frozenset(f'~{arch}' for arch in self.stable_arches) source_arches = self.options.source_arches if source_arches is None: source_arches = self.options.stable_arches self.source_arches = frozenset( arch.lstrip("~") for arch in source_arches) self.source_filter = packages.PackageRestriction( "keywords", values.ContainmentMatch2(self.source_arches))
def _parse(value): return packages.PackageRestriction( attr, values.ContainmentMatch2( values_kls( token_kls(piece.strip()) for piece in value.split(','))))
def _mk_required_use_node(data): if data[0] == '!': return values.ContainmentMatch2(data[1:], negate=True) return values.ContainmentMatch2(data)
def __init__(self, *args, arches_addon=None): super().__init__(*args) target_repo = self.options.target_repo self.official_arches = target_repo.known_arches self.desired_arches = getattr(self.options, 'arches', None) if self.desired_arches is None or self.options.selected_arches is None: # copy it to be safe self.desired_arches = set(self.official_arches) self.global_insoluble = set() profile_filters = defaultdict(list) chunked_data_cache = {} cached_profiles = defaultdict(dict) if self.options.cache['profiles']: for repo in target_repo.trees: cache_file = self.cache_file(repo) # add profiles-base -> repo mapping to ease storage procedure cached_profiles[repo.config.profiles_base]['repo'] = repo try: with open(cache_file, 'rb') as f: cache = pickle.load(f) if cache.version == self.cache.version: cached_profiles[repo.config.profiles_base].update( cache) else: logger.debug( 'forcing %s profile cache regen ' 'due to outdated version', repo.repo_id) os.remove(cache_file) except FileNotFoundError as e: pass except (AttributeError, EOFError, ImportError, IndexError) as e: logger.debug('forcing %s profile cache regen: %s', repo.repo_id, e) os.remove(cache_file) for k in sorted(self.desired_arches): if k.lstrip("~") not in self.desired_arches: continue stable_key = k.lstrip("~") unstable_key = "~" + stable_key stable_r = packages.PackageRestriction( "keywords", values.ContainmentMatch2((stable_key, ))) unstable_r = packages.PackageRestriction( "keywords", values.ContainmentMatch2(( stable_key, unstable_key, ))) default_masked_use = tuple( set(x for x in self.official_arches if x != stable_key)) for profile_obj, profile in self.options.arch_profiles.get(k, []): files = self.profile_data.get(profile, None) try: cached_profile = cached_profiles[profile.base][ profile.path] if files != cached_profile['files']: # force refresh of outdated cache entry raise KeyError masks = cached_profile['masks'] unmasks = cached_profile['unmasks'] immutable_flags = cached_profile['immutable_flags'] stable_immutable_flags = cached_profile[ 'stable_immutable_flags'] enabled_flags = cached_profile['enabled_flags'] stable_enabled_flags = cached_profile[ 'stable_enabled_flags'] pkg_use = cached_profile['pkg_use'] iuse_effective = cached_profile['iuse_effective'] use = cached_profile['use'] provides_repo = cached_profile['provides_repo'] except KeyError: logger.debug('profile regen: %s', profile.path) try: masks = profile_obj.masks unmasks = profile_obj.unmasks immutable_flags = profile_obj.masked_use.clone( unfreeze=True) immutable_flags.add_bare_global((), default_masked_use) immutable_flags.optimize(cache=chunked_data_cache) immutable_flags.freeze() stable_immutable_flags = profile_obj.stable_masked_use.clone( unfreeze=True) stable_immutable_flags.add_bare_global( (), default_masked_use) stable_immutable_flags.optimize( cache=chunked_data_cache) stable_immutable_flags.freeze() enabled_flags = profile_obj.forced_use.clone( unfreeze=True) enabled_flags.add_bare_global((), (stable_key, )) enabled_flags.optimize(cache=chunked_data_cache) enabled_flags.freeze() stable_enabled_flags = profile_obj.stable_forced_use.clone( unfreeze=True) stable_enabled_flags.add_bare_global((), (stable_key, )) stable_enabled_flags.optimize(cache=chunked_data_cache) stable_enabled_flags.freeze() pkg_use = profile_obj.pkg_use iuse_effective = profile_obj.iuse_effective provides_repo = profile_obj.provides_repo # finalize enabled USE flags use = set() misc.incremental_expansion(use, profile_obj.use, 'while expanding USE') use = frozenset(use) except profiles_mod.ProfileError: # unsupported EAPI or other issue, profile checks will catch this continue if self.options.cache['profiles']: cached_profiles[profile.base]['update'] = True cached_profiles[profile.base][profile.path] = { 'files': files, 'masks': masks, 'unmasks': unmasks, 'immutable_flags': immutable_flags, 'stable_immutable_flags': stable_immutable_flags, 'enabled_flags': enabled_flags, 'stable_enabled_flags': stable_enabled_flags, 'pkg_use': pkg_use, 'iuse_effective': iuse_effective, 'use': use, 'provides_repo': provides_repo, } # used to interlink stable/unstable lookups so that if # unstable says it's not visible, stable doesn't try # if stable says something is visible, unstable doesn't try. stable_cache = set() unstable_insoluble = ProtectedSet(self.global_insoluble) # few notes. for filter, ensure keywords is last, on the # offchance a non-metadata based restrict foregos having to # access the metadata. # note that the cache/insoluble are inversly paired; # stable cache is usable for unstable, but not vice versa. # unstable insoluble is usable for stable, but not vice versa vfilter = domain.generate_filter(target_repo.pkg_masks | masks, unmasks) profile_filters[stable_key].append( ProfileData(profile.path, stable_key, provides_repo, packages.AndRestriction(vfilter, stable_r), iuse_effective, use, pkg_use, stable_immutable_flags, stable_enabled_flags, stable_cache, ProtectedSet(unstable_insoluble), profile.status, profile.deprecated)) profile_filters[unstable_key].append( ProfileData(profile.path, unstable_key, provides_repo, packages.AndRestriction(vfilter, unstable_r), iuse_effective, use, pkg_use, immutable_flags, enabled_flags, ProtectedSet(stable_cache), unstable_insoluble, profile.status, profile.deprecated)) # dump updated profile filters for k, v in cached_profiles.items(): if v.pop('update', False): repo = v.pop('repo') cache_file = self.cache_file(repo) try: os.makedirs(os.path.dirname(cache_file), exist_ok=True) with open(cache_file, 'wb+') as f: pickle.dump( _ProfilesCache( cached_profiles[repo.config.profiles_base]), f) except IOError as e: msg = (f'failed dumping {repo.repo_id} profiles cache: ' f'{cache_file!r}: {e.strerror}') raise UserException(msg) profile_evaluate_dict = {} for key, profile_list in profile_filters.items(): similar = profile_evaluate_dict[key] = [] for profile in profile_list: for existing in similar: if (existing[0].masked_use == profile.masked_use and existing[0].forced_use == profile.forced_use): existing.append(profile) break else: similar.append([profile]) self.profile_evaluate_dict = profile_evaluate_dict self.profile_filters = profile_filters
def _fast_identify_candidates(self, restrict, sorter): pkg_restrict = set() cat_restrict = set() cat_exact = set() pkg_exact = set() for x in collect_package_restrictions(restrict, ( "category", "package", )): if x.attr == "category": cat_restrict.add(x.restriction) elif x.attr == "package": pkg_restrict.add(x.restriction) for e, s in ((pkg_exact, pkg_restrict), (cat_exact, cat_restrict)): l = [ x for x in s if isinstance(x, values.StrExactMatch) and not x.negate ] s.difference_update(l) e.update(x.exact for x in l) del l if restrict.negate: cat_exact = pkg_exact = () if cat_exact: if not cat_restrict and len(cat_exact) == 1: # Cannot use pop here, cat_exact is reused below. c = next(iter(cat_exact)) if not pkg_restrict and len(pkg_exact) == 1: cp = (c, pkg_exact.pop()) if cp in self.versions: return [cp] return [] cats_iter = [c] else: cat_restrict.add(values.ContainmentMatch2( frozenset(cat_exact))) cats_iter = sorter(self._cat_filter(cat_restrict)) elif cat_restrict: cats_iter = self._cat_filter(cat_restrict, negate=restrict.negate) else: cats_iter = sorter(self.categories) if pkg_exact: if not pkg_restrict: if sorter is iter: pkg_exact = tuple(pkg_exact) else: pkg_exact = sorter(pkg_exact) return ((c, p) for c in cats_iter for p in pkg_exact) else: pkg_restrict.add(values.ContainmentMatch2( frozenset(pkg_exact))) if pkg_restrict: return self._package_filter(cats_iter, pkg_restrict, negate=restrict.negate) elif not cat_restrict: if sorter is iter and not cat_exact: return self.versions else: return ((c, p) for c in cats_iter for p in sorter(self.packages.get(c, ()))) return ((c, p) for c in cats_iter for p in sorter(self.packages.get(c, ())))
def parse(cls, dep_str, element_class, operators=None, attr=None, element_func=None, transitive_use_atoms=False, allow_src_uri_file_renames=False): """ :param dep_str: string abiding by DepSet syntax :param operators: mapping of node -> callable for special operators in DepSet syntax :param element_func: if None, element_class is used for generating elements, else it's used to generate elements. Mainly useful for when you need to curry a few args for instance generation, since element_class _must_ be a class :param element_class: class of generated elements :param attr: name of the DepSet attribute being parsed """ if element_func is None: element_func = element_class if cls.parse_depset is not None and not (allow_src_uri_file_renames): restrictions = None if operators is None: has_conditionals, restrictions = cls.parse_depset( dep_str, element_func, boolean.AndRestriction, boolean.OrRestriction) else: for x in operators: if x not in ("", "||"): break else: has_conditionals, restrictions = cls.parse_depset( dep_str, element_func, operators.get(""), operators.get("||")) if restrictions is not None: if not has_conditionals and transitive_use_atoms: has_conditionals = cls._has_transitive_use_atoms(restrictions) return cls(restrictions, element_class, has_conditionals) restrictions = [] if operators is None: operators = {"||": boolean.OrRestriction, "": boolean.AndRestriction} raw_conditionals = [] depsets = [restrictions] node_conds = False words = iter(dep_str.split()) # we specifically do it this way since expandable_chain has a bit of nasty # overhead to the tune of 33% slower if allow_src_uri_file_renames: words = expandable_chain(words) k = None try: for k in words: if ")" == k: # no elements == error. if closures don't map up, # indexerror would be chucked from trying to pop # the frame so that is addressed. if not depsets[-1] or not raw_conditionals: raise DepsetParseError(dep_str, attr=attr) elif raw_conditionals[-1] in operators: if len(depsets[-1]) == 1: depsets[-2].append(depsets[-1][0]) else: depsets[-2].append( operators[raw_conditionals[-1]](*depsets[-1])) else: node_conds = True c = raw_conditionals[-1] if c[0] == "!": c = values.ContainmentMatch2(c[1:-1], negate=True) else: c = values.ContainmentMatch2(c[:-1]) depsets[-2].append( packages.Conditional("use", c, tuple(depsets[-1]))) raw_conditionals.pop() depsets.pop() elif "(" == k: k = '' # push another frame on depsets.append([]) raw_conditionals.append(k) elif k[-1] == '?' or k in operators: # use conditional or custom op. # no tokens left == bad dep_str. k2 = next(words) if k2 != "(": raise DepsetParseError(dep_str, k2, attr=attr) # push another frame on depsets.append([]) raw_conditionals.append(k) elif "|" in k: raise DepsetParseError(dep_str, k, attr=attr) elif allow_src_uri_file_renames: try: k2 = next(words) except StopIteration: depsets[-1].append(element_func(k)) else: if k2 != '->': depsets[-1].append(element_func(k)) words.appendleft((k2,)) else: k3 = next(words) # file rename depsets[-1].append(element_func(k, k3)) else: # node/element depsets[-1].append(element_func(k)) except IGNORED_EXCEPTIONS: raise except DepsetParseError: # [][-1] for a frame access, which means it was a parse error. raise except StopIteration: if k is None: raise raise DepsetParseError(dep_str, k, attr=attr) except Exception as e: raise DepsetParseError(dep_str, e, attr=attr) from e # check if any closures required if len(depsets) != 1: raise DepsetParseError(dep_str, attr=attr) if transitive_use_atoms and not node_conds: # localize to this scope for speed. element_class = transitive_use_atom # we can't rely on iter(self) here since it doesn't # descend through boolean restricts. node_conds = cls._has_transitive_use_atoms(restrictions) return cls(tuple(restrictions), element_class, node_conds)