def load(self): debug = self._debug if debug: writemsg_level("\nPackagesSystemSet: profiles: %s\n" % (self._profiles, ), level=logging.DEBUG, noiselevel=-1) mylist = [ grabfile_package(os.path.join(x.location, "packages"), verify_eapi=True, eapi=x.eapi, eapi_default=None, allow_build_id=x.allow_build_id, allow_repo=allow_profile_repo_deps(x)) for x in self._profiles ] if debug: writemsg_level("\nPackagesSystemSet: raw packages: %s\n" % \ (mylist,), level=logging.DEBUG, noiselevel=-1) mylist = stack_lists(mylist, incremental=1) if debug: writemsg_level("\nPackagesSystemSet: stacked packages: %s\n" % \ (mylist,), level=logging.DEBUG, noiselevel=-1) self._setAtoms([x[1:] for x in mylist if x[0] == "*"])
def load(self): self._setAtoms(x for x in stack_lists([ grabfile_package(os.path.join(y.location, "packages"), verify_eapi=True, eapi=y.eapi, eapi_default=None, allow_build_id=y.allow_build_id, allow_repo=allow_profile_repo_deps(y)) for y in self._profiles if "profile-set" in y.profile_formats ], incremental=1) if x[:1] != "*")
def __init__(self, profiles, debug=False): super(ProfilePackageSet, self).__init__(allow_repo=any( allow_profile_repo_deps(y) for y in profiles)) self._profiles = profiles if profiles: desc_profile = profiles[-1] if desc_profile.user_config and len(profiles) > 1: desc_profile = profiles[-2] description = desc_profile.location else: description = None self.description = "Profile packages for profile %s" % description
def _parse_repository_files_to_dict_of_dicts(self, file_name, repositories, eapi_filter=None): ret = {} for repo in repositories.repos_with_profiles(): ret[repo.name] = self._parse_file_to_dict( os.path.join(repo.location, "profiles", file_name), eapi_filter=eapi_filter, eapi_default=repo.eapi, allow_repo=allow_profile_repo_deps(repo), allow_build_id=("build-id" in repo.profile_formats)) return ret
def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, juststrings=False, eapi_filter=None): return tuple( self._parse_file_to_dict(os.path.join(profile.location, file_name), juststrings, recursive=profile.portage1_directories, eapi_filter=eapi_filter, user_config=profile.user_config, eapi=profile.eapi, eapi_default=None, allow_build_id=profile.allow_build_id, allow_repo=allow_profile_repo_deps( profile)) for profile in locations)
def grab_pmask(loc, repo_config): if loc not in pmask_cache: path = os.path.join(loc, 'profiles', 'package.mask') pmask_cache[loc] = grabfile_package( path, recursive=repo_config.portage1_profiles, remember_source_file=True, verify_eapi=True, eapi_default=repo_config.eapi, allow_repo=allow_profile_repo_deps(repo_config), allow_build_id=("build-id" in repo_config.profile_formats)) if repo_config.portage1_profiles_compat and os.path.isdir( path): warnings.warn( _("Repository '%(repo_name)s' is implicitly using " "'portage-1' profile format in its profiles/package.mask, but " "the repository profiles are not marked as that format. This will break " "in the future. Please either convert the following paths " "to files, or add\nprofile-formats = portage-1\nto the " "repository's layout.conf.\n") % dict(repo_name=repo_config.name)) return pmask_cache[loc]
def repo_metadata(portdb, repoman_settings): # get lists of valid keywords, licenses, and use kwlist = set() liclist = set() uselist = set() profile_list = [] global_pmasklines = [] for repo in portdb.repositories: path = repo.location try: liclist.update(os.listdir(os.path.join(path, "licenses"))) except OSError: pass kwlist.update( portage.grabfile(os.path.join(path, "profiles", "arch.list"))) use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc')) for x in use_desc: x = x.split() if x: uselist.add(x[0]) expand_desc_dir = os.path.join(path, 'profiles', 'desc') try: expand_list = os.listdir(expand_desc_dir) except OSError: pass else: for fn in expand_list: if not fn[-5:] == '.desc': continue use_prefix = fn[:-5].lower() + '_' for x in portage.grabfile(os.path.join(expand_desc_dir, fn)): x = x.split() if x: uselist.add(use_prefix + x[0]) global_pmasklines.append( portage.util.grabfile_package( os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True, allow_repo=allow_profile_repo_deps(repo))) desc_path = os.path.join(path, 'profiles', 'profiles.desc') try: desc_file = io.open(_unicode_encode(desc_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') except EnvironmentError: pass else: for i, x in enumerate(desc_file): if x[0] == "#": continue arch = x.split() if len(arch) == 0: continue if len(arch) != 3: err("wrong format: \"%s\" in %s line %d" % ( bad(x.strip()), desc_path, i + 1, )) elif arch[0] not in kwlist: err("invalid arch: \"%s\" in %s line %d" % ( bad(arch[0]), desc_path, i + 1, )) elif arch[2] not in valid_profile_types: err("invalid profile type: \"%s\" in %s line %d" % ( bad(arch[2]), desc_path, i + 1, )) profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path) if not os.path.isdir(profile_desc.abs_path): logging.error( "Invalid %s profile (%s) for arch %s in %s line %d", arch[2], arch[1], arch[0], desc_path, i + 1) continue if os.path.exists( os.path.join(profile_desc.abs_path, 'deprecated')): continue profile_list.append(profile_desc) desc_file.close() global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1) global_pmaskdict = {} for x in global_pmasklines: global_pmaskdict.setdefault(x.cp, []).append(x) del global_pmasklines return (kwlist, liclist, uselist, profile_list, global_pmaskdict, list_checks(kwlist, liclist, uselist, repoman_settings))
def __init__( self, repositories, profiles, abs_user_config, user_config=True, strict_umatched_removal=False, ): self._punmaskdict = ExtendedAtomDict(list) self._pmaskdict = ExtendedAtomDict(list) # Preserves atoms that are eliminated by negative # incrementals in user_pkgmasklines. self._pmaskdict_raw = ExtendedAtomDict(list) # Read profile/package.mask from every repo. # Repositories inherit masks from their parent profiles and # are able to remove mask from them with -atoms. # Such a removal affects only the current repo, but not the parent. # Add ::repo specs to every atom to make sure atoms only affect # packages from the current repo. # Cache the repository-wide package.mask files as a particular # repo may be often referenced by others as the master. pmask_cache = {} def grab_pmask(loc, repo_config): if loc not in pmask_cache: path = os.path.join(loc, "profiles", "package.mask") pmask_cache[loc] = grabfile_package( path, recursive=repo_config.portage1_profiles, remember_source_file=True, verify_eapi=True, eapi_default=repo_config.eapi, allow_repo=allow_profile_repo_deps(repo_config), allow_build_id=("build-id" in repo_config.profile_formats), ) if repo_config.portage1_profiles_compat and os.path.isdir( path): warnings.warn( _("Repository '%(repo_name)s' is implicitly using " "'portage-1' profile format in its profiles/package.mask, but " "the repository profiles are not marked as that format. This will break " "in the future. Please either convert the following paths " "to files, or add\nprofile-formats = portage-1\nto the " "repository's layout.conf.\n") % dict(repo_name=repo_config.name)) return pmask_cache[loc] repo_pkgmasklines = [] for repo in repositories.repos_with_profiles(): lines = [] repo_lines = grab_pmask(repo.location, repo) removals = frozenset(line[0][1:] for line in repo_lines if line[0][:1] == "-") matched_removals = set() for master in repo.masters: master_lines = grab_pmask(master.location, master) for line in master_lines: if line[0] in removals: matched_removals.add(line[0]) # Since we don't stack masters recursively, there aren't any # atoms earlier in the stack to be matched by negative atoms in # master_lines. Also, repo_lines may contain negative atoms # that are intended to negate atoms from a different master # than the one with which we are currently stacking. Therefore, # we disable warn_for_unmatched_removal here (see bug #386569). lines.append( stack_lists( [master_lines, repo_lines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=False, )) # It's safe to warn for unmatched removal if masters have not # been overridden by the user, which is guaranteed when # user_config is false (when called by repoman). if repo.masters: unmatched_removals = removals.difference(matched_removals) if unmatched_removals and not user_config: source_file = os.path.join(repo.location, "profiles", "package.mask") unmatched_removals = list(unmatched_removals) if len(unmatched_removals) > 3: writemsg( _("--- Unmatched removal atoms in %s: %s and %s more\n" ) % ( source_file, ", ".join("-" + x for x in unmatched_removals[:3]), len(unmatched_removals) - 3, ), noiselevel=-1, ) else: writemsg( _("--- Unmatched removal atom(s) in %s: %s\n") % ( source_file, ", ".join("-" + x for x in unmatched_removals), ), noiselevel=-1, ) else: lines.append( stack_lists( [repo_lines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=not user_config, strict_warn_for_unmatched_removal= strict_umatched_removal, )) repo_pkgmasklines.extend( append_repo(stack_lists(lines), repo.name, remember_source_file=True)) repo_pkgunmasklines = [] for repo in repositories.repos_with_profiles(): if not repo.portage1_profiles: continue repo_lines = grabfile_package( os.path.join(repo.location, "profiles", "package.unmask"), recursive=1, remember_source_file=True, verify_eapi=True, eapi_default=repo.eapi, allow_repo=allow_profile_repo_deps(repo), allow_build_id=("build-id" in repo.profile_formats), ) lines = stack_lists( [repo_lines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal, ) repo_pkgunmasklines.extend( append_repo(lines, repo.name, remember_source_file=True)) # Read package.mask from the user's profile. Stack them in the end # to allow profiles to override masks from their parent profiles. profile_pkgmasklines = [] profile_pkgunmasklines = [] for x in profiles: profile_pkgmasklines.append( grabfile_package( os.path.join(x.location, "package.mask"), recursive=x.portage1_directories, remember_source_file=True, verify_eapi=True, eapi=x.eapi, eapi_default=None, allow_repo=allow_profile_repo_deps(x), allow_build_id=x.allow_build_id, )) if x.portage1_directories: profile_pkgunmasklines.append( grabfile_package( os.path.join(x.location, "package.unmask"), recursive=x.portage1_directories, remember_source_file=True, verify_eapi=True, eapi=x.eapi, eapi_default=None, allow_repo=allow_profile_repo_deps(x), allow_build_id=x.allow_build_id, )) profile_pkgmasklines = stack_lists( profile_pkgmasklines, incremental=1, remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal, ) profile_pkgunmasklines = stack_lists( profile_pkgunmasklines, incremental=1, remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal, ) # Read /etc/portage/package.mask. Don't stack it to allow the user to # remove mask atoms from everywhere with -atoms. user_pkgmasklines = [] user_pkgunmasklines = [] if user_config: user_pkgmasklines = grabfile_package( os.path.join(abs_user_config, "package.mask"), recursive=1, allow_wildcard=True, allow_repo=True, remember_source_file=True, verify_eapi=False, allow_build_id=True, ) user_pkgunmasklines = grabfile_package( os.path.join(abs_user_config, "package.unmask"), recursive=1, allow_wildcard=True, allow_repo=True, remember_source_file=True, verify_eapi=False, allow_build_id=True, ) # Stack everything together. At this point, only user_pkgmasklines may contain -atoms. # Don't warn for unmatched -atoms here, since we don't do it for any other user config file. raw_pkgmasklines = stack_lists( [repo_pkgmasklines, profile_pkgmasklines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=False, ignore_repo=True, ) pkgmasklines = stack_lists( [repo_pkgmasklines, profile_pkgmasklines, user_pkgmasklines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=False, ignore_repo=True, ) pkgunmasklines = stack_lists( [repo_pkgunmasklines, profile_pkgunmasklines, user_pkgunmasklines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=False, ignore_repo=True, ) for x, source_file in raw_pkgmasklines: self._pmaskdict_raw.setdefault(x.cp, []).append(x) for x, source_file in pkgmasklines: self._pmaskdict.setdefault(x.cp, []).append(x) for x, source_file in pkgunmasklines: self._punmaskdict.setdefault(x.cp, []).append(x) for d in (self._pmaskdict_raw, self._pmaskdict, self._punmaskdict): for k, v in d.items(): d[k] = tuple(v)
def __init__(self, profiles, abs_user_config, user_config=True, global_accept_keywords=""): self._pkeywords_list = [] rawpkeywords = [ grabdict_package(os.path.join(x.location, "package.keywords"), recursive=x.portage1_directories, verify_eapi=True, eapi=x.eapi, eapi_default=None, allow_repo=allow_profile_repo_deps(x), allow_build_id=x.allow_build_id) for x in profiles ] for pkeyworddict in rawpkeywords: if not pkeyworddict: # Omit non-existent files from the stack. continue cpdict = {} for k, v in pkeyworddict.items(): cpdict.setdefault(k.cp, {})[k] = v self._pkeywords_list.append(cpdict) self._pkeywords_list = tuple(self._pkeywords_list) self._p_accept_keywords = [] raw_p_accept_keywords = [ grabdict_package(os.path.join(x.location, "package.accept_keywords"), recursive=x.portage1_directories, verify_eapi=True, eapi=x.eapi, eapi_default=None, allow_repo=allow_profile_repo_deps(x)) for x in profiles ] for d in raw_p_accept_keywords: if not d: # Omit non-existent files from the stack. continue cpdict = {} for k, v in d.items(): cpdict.setdefault(k.cp, {})[k] = tuple(v) self._p_accept_keywords.append(cpdict) self._p_accept_keywords = tuple(self._p_accept_keywords) self.pkeywordsdict = ExtendedAtomDict(dict) if user_config: user_accept_kwrds_path = os.path.join(abs_user_config, "package.accept_keywords") user_kwrds_path = os.path.join(abs_user_config, "package.keywords") pkgdict = grabdict_package(user_kwrds_path, recursive=1, allow_wildcard=True, allow_repo=True, verify_eapi=False, allow_build_id=True) if pkgdict and portage._internal_caller: warnings.warn( _("%s is deprecated, use %s instead") % (user_kwrds_path, user_accept_kwrds_path), UserWarning) for k, v in grabdict_package(user_accept_kwrds_path, recursive=1, allow_wildcard=True, allow_repo=True, verify_eapi=False, allow_build_id=True).items(): pkgdict.setdefault(k, []).extend(v) accept_keywords_defaults = global_accept_keywords.split() accept_keywords_defaults = tuple('~' + keyword for keyword in \ accept_keywords_defaults if keyword[:1] not in "~-") for k, v in pkgdict.items(): # default to ~arch if no specific keyword is given if not v: v = accept_keywords_defaults else: v = tuple(v) self.pkeywordsdict.setdefault(k.cp, {})[k] = v