class base(object): """ base restriction matching object. all derivatives *should* be __slot__ based (lot of instances may wind up in memory). """ __metaclass__ = caching.WeakInstMeta __inst_caching__ = True # __weakref__ here's is implicit via the metaclass __slots__ = () package_matching = False klass.inject_immutable_instance(locals()) def match(self, *arg, **kwargs): raise NotImplementedError def force_False(self, *arg, **kwargs): return not self.match(*arg, **kwargs) def force_True(self, *arg, **kwargs): return self.match(*arg, **kwargs) def __len__(self): return 1
class BundledProfiles(object): klass.inject_immutable_instance(locals()) def __init__(self, profile_base, format='pms'): object.__setattr__(self, 'profile_base', profile_base) object.__setattr__(self, 'format', format) @klass.jit_attr def arch_profiles(self): """Return the mapping of arches to profiles for a repo.""" d = mappings.defaultdict(list) fp = pjoin(self.profile_base, 'profiles.desc') try: for line in iter_read_bash(fp): l = line.split() try: key, profile, status = l except ValueError: logger.error( f"{fp}: line doesn't follow 'key profile status' form: {line}") continue # Normalize the profile name on the offchance someone slipped an extra / # into it. d[key].append(_KnownProfile( '/'.join(filter(None, profile.split('/'))), status)) except FileNotFoundError: logger.debug(f"No profile descriptions found at {fp!r}") return mappings.ImmutableDict( (k, tuple(sorted(v))) for k, v in d.items()) def arches(self, status=None): """All arches with profiles defined in the repo.""" arches = [] for arch, profiles in self.arch_profiles.items(): for _profile_path, profile_status in profiles: if status is None or profile_status == status: arches.append(arch) return frozenset(arches) def paths(self, status=None): """Yield profile paths optionally matching a given status.""" if status == 'deprecated': for root, dirs, files in os.walk(self.profile_base): if os.path.exists(pjoin(root, 'deprecated')): yield root[len(self.profile_base) + 1:] else: for profile_path, profile_status in chain.from_iterable(self.arch_profiles.values()): if status is None or status == profile_status: yield profile_path def create_profile(self, node): """Return profile object for a given path.""" return profiles.OnDiskProfile(self.profile_base, node)
class BundledProfiles(object): klass.inject_immutable_instance(locals()) def __init__(self, profile_base, format='pms'): object.__setattr__(self, 'profile_base', profile_base) object.__setattr__(self, 'format', format) @klass.jit_attr def arch_profiles(self): d = mappings.defaultdict(list) fp = pjoin(self.profile_base, 'profiles.desc') try: for line in iter_read_bash(fp): l = line.split() try: key, profile, status = l except ValueError: logger.error( "%s: line doesn't follow 'key profile status' form: %s", fp, line) continue # Normalize the profile name on the offchance someone slipped an extra / # into it. d[key].append( _KnownProfile('/'.join(filter(None, profile.split('/'))), status)) except EnvironmentError as e: if e.errno != errno.ENOENT: raise logger.debug("No profile descriptions found at %r", fp) return mappings.ImmutableDict( (k, tuple(sorted(v))) for k, v in d.iteritems()) def status_profiles(self, status): """Yield profiles matching a given status.""" for profile, s in chain.from_iterable(self.arch_profiles.itervalues()): if status == s: yield profile def create_profile(self, node): return profiles.OnDiskProfile(self.profile_base, node)
def f(scope): klass.inject_immutable_instance(scope)
class RepoConfig(syncable.tree): layout_offset = "metadata/layout.conf" default_hashes = ('size', 'sha256', 'sha512', 'whirlpool') supported_profile_formats = ('pms', 'portage-1', 'portage-2') supported_cache_formats = ('pms', 'md5-dict') klass.inject_immutable_instance(locals()) __metaclass__ = WeakInstMeta __inst_caching__ = True pkgcore_config_type = ConfigHint(typename='repo_config', types={ 'config_name': 'str', 'syncer': 'lazy_ref:syncer', }) def __init__(self, location, config_name=None, syncer=None, profiles_base='profiles'): object.__setattr__(self, 'config_name', config_name) object.__setattr__(self, 'location', location) object.__setattr__(self, 'profiles_base', pjoin(self.location, profiles_base)) syncable.tree.__init__(self, syncer) self._parse_config() def _parse_config(self): """Load data from the repo's metadata/layout.conf file.""" path = pjoin(self.location, self.layout_offset) data = read_dict(iter_read_bash(readlines_ascii(path, True, True)), source_isiter=True, strip=True, filename=path) sf = object.__setattr__ hashes = data.get('manifest-hashes', '').lower().split() if hashes: hashes = ['size'] + hashes hashes = tuple(iter_stable_unique(hashes)) else: hashes = self.default_hashes manifest_policy = data.get('use-manifests', 'strict').lower() d = { 'disabled': (manifest_policy == 'false'), 'strict': (manifest_policy == 'strict'), 'thin': (data.get('thin-manifests', '').lower() == 'true'), 'signed': (data.get('sign-manifests', 'true').lower() == 'true'), 'hashes': hashes, } # complain if profiles/repo_name is missing repo_name = readfile(pjoin(self.profiles_base, 'repo_name'), True) if repo_name is None: if not self.is_empty: logger.warning("repo lacks a defined name: %r", self.location) repo_name = '<unlabeled repo %s>' % self.location # repo-name setting from metadata/layout.conf overrides profiles/repo_name if it exists sf(self, 'repo_name', data.get('repo-name', repo_name.strip())) sf(self, 'manifests', _immutable_attr_dict(d)) masters = data.get('masters') if masters is None: if not self.is_empty: logger.warning( "repo at %r, named %r, doesn't specify masters in metadata/layout.conf. " "Please explicitly set masters (use \"masters =\" if the repo " "is standalone).", self.location, self.repo_id) masters = () else: masters = tuple(iter_stable_unique(masters.split())) sf(self, 'masters', masters) aliases = data.get('aliases', '').split() + [self.repo_id, self.location] sf(self, 'aliases', tuple(iter_stable_unique(aliases))) sf(self, 'eapis_deprecated', tuple(iter_stable_unique(data.get('eapis-deprecated', '').split()))) v = set(data.get('cache-formats', 'pms').lower().split()) if not v: v = [None] elif not v.intersection(self.supported_cache_formats): v = ['pms'] sf(self, 'cache_format', list(v)[0]) profile_formats = set( data.get('profile-formats', 'pms').lower().split()) if not profile_formats: logger.warning( "repo at %r has unset profile-formats, defaulting to pms") profile_formats = set(['pms']) unknown = profile_formats.difference(self.supported_profile_formats) if unknown: logger.warning("repo at %r has unsupported profile format%s: %s", self.location, pluralism(unknown), ', '.join(sorted(unknown))) profile_formats.difference_update(unknown) profile_formats.add('pms') sf(self, 'profile_formats', profile_formats) @klass.jit_attr def raw_known_arches(self): """All valid KEYWORDS for the repo.""" try: return frozenset( iter_read_bash(pjoin(self.profiles_base, 'arch.list'))) except EnvironmentError as e: if e.errno != errno.ENOENT: raise return frozenset() @klass.jit_attr def raw_use_desc(self): """Global USE flags for the repo.""" # todo: convert this to using a common exception base, with # conversion of ValueErrors... def converter(key): return (packages.AlwaysTrue, key) return tuple(self._split_use_desc_file('use.desc', converter)) @klass.jit_attr def raw_use_local_desc(self): """Local USE flags for the repo.""" def converter(key): # todo: convert this to using a common exception base, with # conversion of ValueErrors/atom exceptions... chunks = key.split(':', 1) return (atom.atom(chunks[0]), chunks[1]) return tuple(self._split_use_desc_file('use.local.desc', converter)) @klass.jit_attr def raw_use_expand_desc(self): """USE_EXPAND settings for the repo.""" base = pjoin(self.profiles_base, 'desc') try: targets = sorted(listdir_files(base)) except EnvironmentError as e: if e.errno != errno.ENOENT: raise return () def f(): for use_group in targets: group = use_group.split('.', 1)[0] + "_" def converter(key): return (packages.AlwaysTrue, group + key) for x in self._split_use_desc_file('desc/%s' % use_group, converter): yield x return tuple(f()) def _split_use_desc_file(self, name, converter): line = None fp = pjoin(self.profiles_base, name) try: for line in iter_read_bash(fp): key, val = line.split(None, 1) key = converter(key) yield key[0], (key[1], val.split('-', 1)[1].strip()) except EnvironmentError as e: if e.errno != errno.ENOENT: raise except ValueError: if line is None: raise compatibility.raise_from( ValueError("Failed parsing %r: line was %r" % (fp, line))) known_arches = klass.alias_attr('raw_known_arches') use_desc = klass.alias_attr('raw_use_desc') use_local_desc = klass.alias_attr('raw_use_local_desc') use_expand_desc = klass.alias_attr('raw_use_expand_desc') @klass.jit_attr def is_empty(self): """Return boolean related to if the repo has files in it.""" result = True try: # any files existing means it's not empty result = not listdir(self.location) except EnvironmentError as e: if e.errno != errno.ENOENT: raise if result: logger.debug("repo is empty: %r", self.location) return result @klass.jit_attr def repo_id(self): """Main identifier for the repo. The name set in repos.conf for a repo overrides any repo-name settings in the repo. """ if self.config_name is not None: return self.config_name return self.repo_name arch_profiles = klass.alias_attr('profiles.arch_profiles') @klass.jit_attr def profiles(self): return BundledProfiles(self.profiles_base)
class RepoConfig(syncable.tree): layout_offset = "metadata/layout.conf" default_hashes = ('size', 'sha256', 'sha512', 'whirlpool') klass.inject_immutable_instance(locals()) __metaclass__ = WeakInstMeta __inst_caching__ = True pkgcore_config_type = ConfigHint(typename='raw_repo', types={'syncer': 'lazy_ref:syncer'}) def __init__(self, location, syncer=None, profiles_base='profiles'): object.__setattr__(self, 'location', location) object.__setattr__(self, 'profiles_base', pjoin(self.location, profiles_base)) syncable.tree.__init__(self, syncer) self.parse_config() def load_config(self): path = pjoin(self.location, self.layout_offset) return read_dict(iter_read_bash(readlines_ascii(path, True, True)), source_isiter=True, strip=True, filename=path) def parse_config(self): data = self.load_config() sf = object.__setattr__ hashes = data.get('manifest-hashes', '').lower().split() if hashes: hashes = ['size'] + hashes hashes = tuple(iter_stable_unique(hashes)) else: hashes = self.default_hashes manifest_policy = data.get('use-manifests', 'strict').lower() d = { 'disabled': (manifest_policy == 'false'), 'strict': (manifest_policy == 'strict'), 'thin': (data.get('thin-manifests', '').lower() == 'true'), 'signed': (data.get('sign-manifests', 'true').lower() == 'true'), 'hashes': hashes, } sf(self, 'manifests', _immutable_attr_dict(d)) masters = data.get('masters') if masters is None: if self.repo_id != 'gentoo' and not self.is_empty: logger.warning( "repository at %r, named %r, doesn't specify masters in metadata/layout.conf. " "Defaulting to whatever repository is defined as 'default' (gentoo usually). " "Please explicitly set the masters, or set masters = '' if the repository " "is standalone.", self.location, self.repo_id) else: masters = tuple(iter_stable_unique(masters.split())) sf(self, 'masters', masters) sf(self, 'aliases', tuple(iter_stable_unique(data.get('aliases', '').split()))) sf(self, 'eapis_deprecated', tuple(iter_stable_unique(data.get('eapis-deprecated', '').split()))) v = set(data.get('cache-formats', 'pms').lower().split()) if not v.intersection(['pms', 'md5-dict']): v = 'pms' sf(self, 'cache_format', list(v)[0]) v = set(data.get('profile-formats', 'pms').lower().split()) if not v: # dumb ass overlay devs, treat it as missing. v = set(['pms']) unknown = v.difference(['pms', 'portage-1', 'portage-2']) if unknown: logger.warning( "repository at %r has an unsupported profile format: %s" % (self.location, ', '.join(repr(x) for x in sorted(v)))) v = 'pms' sf(self, 'profile_format', list(v)[0]) @klass.jit_attr def raw_known_arches(self): try: return frozenset( iter_read_bash(pjoin(self.profiles_base, 'arch.list'))) except EnvironmentError as e: if e.errno != errno.ENOENT: raise return frozenset() @klass.jit_attr def raw_use_desc(self): # todo: convert this to using a common exception base, with # conversion of ValueErrors... def converter(key): return (packages.AlwaysTrue, key) return tuple(self._split_use_desc_file('use.desc', converter)) @klass.jit_attr def raw_use_local_desc(self): def converter(key): # todo: convert this to using a common exception base, with # conversion of ValueErrors/atom exceptoins... chunks = key.split(':', 1) return (atom.atom(chunks[0]), chunks[1]) return tuple(self._split_use_desc_file('use.local.desc', converter)) @klass.jit_attr def raw_use_expand_desc(self): base = pjoin(self.profiles_base, 'desc') try: targets = sorted(listdir_files(base)) except EnvironmentError as e: if e.errno != errno.ENOENT: raise return () def f(): for use_group in targets: group = use_group.split('.', 1)[0] + "_" def converter(key): return (packages.AlwaysTrue, group + key) for blah in self._split_use_desc_file('desc/%s' % use_group, converter): yield blah return tuple(f()) def _split_use_desc_file(self, name, converter): line = None fp = pjoin(self.profiles_base, name) try: for line in iter_read_bash(fp): key, val = line.split(None, 1) key = converter(key) yield key[0], (key[1], val.split('-', 1)[1].strip()) except EnvironmentError as e: if e.errno != errno.ENOENT: raise except ValueError as v: if line is None: raise compatibility.raise_from( ValueError("Failed parsing %r: line was %r" % (fp, line))) known_arches = klass.alias_attr('raw_known_arches') use_desc = klass.alias_attr('raw_use_desc') use_local_desc = klass.alias_attr('raw_use_local_desc') use_expand_desc = klass.alias_attr('raw_use_expand_desc') @klass.jit_attr def is_empty(self): result = True try: # any files existing means it's not empty result = not listdir(self.location) except EnvironmentError as e: if e.errno != errno.ENOENT: raise if result: logger.debug("repository at %r is empty" % (self.location, )) return result @klass.jit_attr def repo_id(self): val = readfile(pjoin(self.profiles_base, 'repo_name'), True) if val is None: if not self.is_empty: logger.warning( "repository at location %r lacks a defined repo_name", self.location) val = '<unlabeled repository %s>' % self.location return val.strip() arch_profiles = klass.alias_attr('profiles.arch_profiles') @klass.jit_attr def profiles(self): return BundledProfiles(self.profiles_base)
class fsBase: """base class, all extensions must derive from this class""" __slots__ = ("location", "mtime", "mode", "uid", "gid") __attrs__ = __slots__ __default_attrs__ = {} locals().update( (x.replace("is", "is_"), False) for x in __all__ if x.startswith("is") and x.islower() and not x.endswith("fs_obj")) klass.inject_richcmp_methods_from_cmp(locals()) klass.inject_immutable_instance(locals()) def __init__(self, location, strict=True, **d): d["location"] = normpath(location) s = object.__setattr__ if strict: for k in self.__attrs__: s(self, k, d[k]) else: for k, v in d.items(): s(self, k, v) gen_doc_additions(__init__, __attrs__) def change_attributes(self, **kwds): d = {x: getattr(self, x) for x in self.__attrs__ if hasattr(self, x)} d.update(kwds) # split location out location = d.pop("location") if not location.startswith(path_seperator): location = abspath(location) d["strict"] = False return self.__class__(location, **d) def __getattr__(self, attr): # we would only get called if it doesn't exist. if attr not in self.__attrs__: raise AttributeError(self, attr) obj = self.__default_attrs__.get(attr) if not callable(obj): return obj return obj(self) def __hash__(self): return hash(self.location) def __eq__(self, other): if not isinstance(other, self.__class__): return False return self.location == other.location def __ne__(self, other): return not self == other def realpath(self, cache=None): """calculate the abspath/canonicalized path for this entry, returning a new instance if the path differs. :keyword cache: Either None (no cache), or a data object of path-> resolved. Currently unused, but left in for forwards compatibility """ new_path = realpath(self.location) if new_path == self.location: return self return self.change_attributes(location=new_path) @property def basename(self): return basename(self.location) @property def dirname(self): return dirname(self.location) def fnmatch(self, pattern): return fnmatch.fnmatch(self.location, pattern) def __cmp__(self, other): return cmp(self.location, other.location) def __str__(self): return self.location
class RepoConfig(syncable.tree, metaclass=WeakInstMeta): """Configuration data for an ebuild repository.""" layout_offset = "metadata/layout.conf" default_hashes = ('size', 'blake2b', 'sha512') default_required_hashes = ('size', 'blake2b') supported_profile_formats = ('pms', 'portage-1', 'portage-2', 'profile-set') supported_cache_formats = ('md5-dict', 'pms') klass.inject_immutable_instance(locals()) __inst_caching__ = True pkgcore_config_type = ConfigHint( typename='repo_config', types={ 'config_name': 'str', 'syncer': 'lazy_ref:syncer', }) def __init__(self, location, config_name=None, syncer=None, profiles_base='profiles'): object.__setattr__(self, 'config_name', config_name) object.__setattr__(self, 'location', location) object.__setattr__(self, 'profiles_base', pjoin(self.location, profiles_base)) if not self.eapi.is_supported: raise repo_errors.UnsupportedRepo(self) super().__init__(syncer) self._parse_config() def _parse_config(self): """Load data from the repo's metadata/layout.conf file.""" path = pjoin(self.location, self.layout_offset) data = read_dict( iter_read_bash( readlines_ascii(path, strip_whitespace=True, swallow_missing=True)), source_isiter=True, strip=True, filename=path, ignore_errors=True) sf = object.__setattr__ sf(self, 'repo_name', data.get('repo-name', None)) hashes = data.get('manifest-hashes', '').lower().split() if hashes: hashes = ['size'] + hashes hashes = tuple(iter_stable_unique(hashes)) else: hashes = self.default_hashes required_hashes = data.get('manifest-required-hashes', '').lower().split() if required_hashes: required_hashes = ['size'] + required_hashes required_hashes = tuple(iter_stable_unique(required_hashes)) else: required_hashes = self.default_required_hashes manifest_policy = data.get('use-manifests', 'strict').lower() d = { 'disabled': (manifest_policy == 'false'), 'strict': (manifest_policy == 'strict'), 'thin': (data.get('thin-manifests', '').lower() == 'true'), 'signed': (data.get('sign-manifests', 'true').lower() == 'true'), 'hashes': hashes, 'required_hashes': required_hashes, } sf(self, 'manifests', _immutable_attr_dict(d)) masters = data.get('masters') if masters is None: if not self.is_empty: logger.warning( f"repo at {self.location!r}, named {self.repo_id!r}, doesn't " "specify masters in metadata/layout.conf. Please explicitly " "set masters (use \"masters =\" if the repo is standalone).") masters = () else: masters = tuple(iter_stable_unique(masters.split())) sf(self, 'masters', masters) aliases = data.get('aliases', '').split() + [self.repo_id, self.location] sf(self, 'aliases', tuple(iter_stable_unique(aliases))) sf(self, 'eapis_deprecated', tuple(iter_stable_unique(data.get('eapis-deprecated', '').split()))) sf(self, 'eapis_banned', tuple(iter_stable_unique(data.get('eapis-banned', '').split()))) v = set(data.get('cache-formats', 'pms').lower().split()) if not v: v = [None] else: # sort into favored order v = [f for f in self.supported_cache_formats if f in v] if not v: logger.warning(f'unknown cache format: falling back to pms format') v = ['pms'] sf(self, 'cache_format', list(v)[0]) profile_formats = set(data.get('profile-formats', 'pms').lower().split()) if not profile_formats: logger.warning( f"{self.repo_id!r} repo at {self.location!r} has explicitly " "unset profile-formats, defaulting to pms") profile_formats = {'pms'} unknown = profile_formats.difference(self.supported_profile_formats) if unknown: logger.warning( "%r repo at %r has unsupported profile format%s: %s", self.repo_id, self.location, pluralism(unknown), ', '.join(sorted(unknown))) profile_formats.difference_update(unknown) profile_formats.add('pms') sf(self, 'profile_formats', profile_formats) @klass.jit_attr def raw_known_arches(self): """All valid KEYWORDS for the repo.""" try: return frozenset(iter_read_bash( pjoin(self.profiles_base, 'arch.list'))) except FileNotFoundError: return frozenset() @klass.jit_attr def raw_use_desc(self): """Global USE flags for the repo.""" # todo: convert this to using a common exception base, with # conversion of ValueErrors... def converter(key): return (packages.AlwaysTrue, key) return tuple(self._split_use_desc_file('use.desc', converter)) @klass.jit_attr def raw_use_local_desc(self): """Local USE flags for the repo.""" def converter(key): # todo: convert this to using a common exception base, with # conversion of ValueErrors/atom exceptions... chunks = key.split(':', 1) return (atom.atom(chunks[0]), chunks[1]) return tuple(self._split_use_desc_file('use.local.desc', converter)) @klass.jit_attr def raw_use_expand_desc(self): """USE_EXPAND settings for the repo.""" base = pjoin(self.profiles_base, 'desc') try: targets = sorted(listdir_files(base)) except FileNotFoundError: return () def f(): for use_group in targets: group = use_group.split('.', 1)[0] + "_" def converter(key): return (packages.AlwaysTrue, group + key) for x in self._split_use_desc_file(f'desc/{use_group}', converter): yield x return tuple(f()) def _split_use_desc_file(self, name, converter): line = None fp = pjoin(self.profiles_base, name) try: for line in iter_read_bash(fp): key, val = line.split(None, 1) key = converter(key) yield key[0], (key[1], val.split('-', 1)[1].strip()) except FileNotFoundError: pass except ValueError as e: if line is None: raise raise ValueError(f"Failed parsing {fp!r}: line was {line!r}") from e known_arches = klass.alias_attr('raw_known_arches') use_desc = klass.alias_attr('raw_use_desc') use_local_desc = klass.alias_attr('raw_use_local_desc') use_expand_desc = klass.alias_attr('raw_use_expand_desc') @klass.jit_attr def is_empty(self): """Return boolean related to if the repo has files in it.""" result = True try: # any files existing means it's not empty result = not listdir(self.location) if result: logger.debug(f"repo is empty: {self.location!r}") except FileNotFoundError: pass return result @klass.jit_attr def pms_repo_name(self): """Repository name from profiles/repo_name (as defined by PMS). We're more lenient than the spec and don't verify it conforms to the specified format. """ name = readfile(pjoin(self.profiles_base, 'repo_name'), none_on_missing=True) if name is not None: name = name.split('\n', 1)[0].strip() return name @klass.jit_attr def repo_id(self): """Main identifier for the repo. The precedence order is as follows: repos.conf name, repo-name from metadata/layout.conf, profiles/repo_name, and finally a fallback to the repo's location for unlabeled repos. """ if self.config_name: return self.config_name if self.repo_name: return self.repo_name if self.pms_repo_name: return self.pms_repo_name if not self.is_empty: logger.warning(f"repo lacks a defined name: {self.location!r}") return f'<unlabeled repo: {self.location!r}>' @klass.jit_attr def updates(self): """Package updates for the repo defined in profiles/updates/*.""" updates_dir = pjoin(self.profiles_base, 'updates') d = pkg_updates.read_updates(updates_dir) return mappings.ImmutableDict(d) @klass.jit_attr def profiles(self): return BundledProfiles(self.profiles_base) arch_profiles = klass.alias_attr('profiles.arch_profiles') @klass.jit_attr def eapi(self): try: path = pjoin(self.profiles_base, 'eapi') data = (x.strip() for x in iter_read_bash(path)) data = [_f for _f in data if _f] if len(data) != 1: logger.warning(f"multiple EAPI lines detected: {path!r}") return get_eapi(data[0]) except (FileNotFoundError, IndexError): return get_eapi('0')