def test_parents(self): path = pjoin(self.dir, self.profile) os.mkdir(pjoin(path, "child")) self.write_file("parent", "..", profile="%s/child" % self.profile) p = self.klass(pjoin(path, "child")) self.assertEqual(1, len(p.parents)) self.assertEqual(p.parents[0].path, path)
def test_packages(self): p = self.klass(pjoin(self.dir, self.profile)) self.assertEqual(p.system, empty) self.assertEqual(p.visibility, empty) self.parsing_checks("packages", "system") self.write_file("packages", "#foo\n") p = self.klass(pjoin(self.dir, self.profile)) self.assertEqual(p.visibility, empty) self.assertEqual(p.system, empty) self.write_file("packages", "#foo\ndev-util/diffball\n") p = self.klass(pjoin(self.dir, self.profile)) self.assertEqual(p.system, empty) self.assertEqual(list(p.visibility), [(), (atom("dev-util/diffball", negate_vers=True),)]) self.write_file("packages", "-dev-util/diffball\ndev-foo/bar\n*dev-sys/atom\n" "-*dev-sys/atom2\nlock-foo/dar") p = self.klass(pjoin(self.dir, self.profile)) self.assertEqual(p.system, ((atom("dev-sys/atom2"),), (atom("dev-sys/atom"),))) self.assertEqual( [set(x) for x in p.visibility], [ set([atom("dev-util/diffball", negate_vers=True)]), set([atom("dev-foo/bar", negate_vers=True), atom("lock-foo/dar", negate_vers=True)]), ], ) self.simple_eapi_awareness_check("packages", "system")
def parsing_checks(self, filename, attr, data="", line_negation=True): path = pjoin(self.dir, self.profile) self.write_file(filename, data) getattr(self.klass(path), attr) self.write_file(filename, "-") self.assertRaises(profiles.ProfileError, getattr, self.klass(path), attr) self.wipe_path(pjoin(path, filename))
def fetch_one(self, fetchable, observer, retry=False): if fetchable.filename in self._basenames: return True # fetching files without uri won't fly # XXX hack atm, could use better logic but works for now try: fp = self.fetcher(fetchable) except fetch_errors.ChksumFailure as e: # checksum failed, rename file and try refetching path = pjoin(self.fetcher.distdir, fetchable.filename) failed_filename = f'{fetchable.filename}._failed_chksum_' failed_path = pjoin(self.fetcher.distdir, failed_filename) os.rename(path, failed_path) if retry: raise observer.error(str(e)) observer.error(f'renaming to {failed_filename!r} and refetching from upstream') observer.flush() # refetch directly from upstream return self.fetch_one(fetchable.upstream, observer, retry=True) except fetch_errors.FetchFailed as e: fp = None if fp is None: return False self.verified_files[fp] = fetchable self._basenames.add(fetchable.filename) return True
def test_from_abspath(self): self.mk_profiles({"name": "profiles"}, {"name": "profiles/1"}) base = pjoin(self.dir, "profiles") p = self.kls.from_abspath(pjoin(base, "1")) self.assertNotEqual(p, None) self.assertEqual(normpath(p.basepath), normpath(base)) self.assertEqual(normpath(p.profile), normpath(pjoin(base, "1")))
def run_check(*args): # create a fresh tree for the profile work everytime. # do this, so that it's always a unique pathway- this sidesteps # any potential issues of ProfileNode instance caching. path = pjoin(self.dir, 'foo', str(counter.next())) shutil.copytree(pjoin(self.dir, 'foo'), path, symlinks=True) return self.process_check(path, list(args))
def config_from_make_conf(location="/etc/"): """ generate a config from a file location :param location: location the portage configuration is based in, defaults to /etc """ # this actually differs from portage parsing- we allow # make.globals to provide vars used in make.conf, portage keeps # them seperate (kind of annoying) config_root = os.environ.get("PORTAGE_CONFIGROOT", "/") base_path = pjoin(config_root, location.strip("/")) portage_base = pjoin(base_path, "portage") # this isn't preserving incremental behaviour for features/use # unfortunately conf_dict = {} try: load_make_config(conf_dict, pjoin(base_path, 'make.globals')) except errors.ParsingError, e: if not getattr(getattr(e, 'exc', None), 'errno', None) == errno.ENOENT: raise try: load_make_config(conf_dict, pjoin(config_root, 'usr/share/portage/config/make.globals')) except compatibility.IGNORED_EXCEPTIONS: raise except: compatibility.raise_from(errors.ParsingError( "failed to find a usable make.globals"))
def add_sets(config, root, portage_base_dir): config["world"] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.filelist.WorldFile", "location": pjoin(root, const.WORLD_FILE)}) config["system"] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.system.SystemSet", "profile": "profile"}) config["installed"] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.installed.Installed", "vdb": "vdb"}) config["versioned-installed"] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.installed.VersionedInstalled", "vdb": "vdb"}) set_fp = pjoin(portage_base_dir, "sets") try: for setname in listdir_files(set_fp): # Potential for name clashes here, those will just make # the set not show up in config. if setname in ("system", "world"): logger.warning( "user defined set %s is disallowed; ignoring" % pjoin(set_fp, setname)) continue config[setname] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.filelist.FileList", "location": pjoin(set_fp, setname)}) except OSError as e: if e.errno != errno.ENOENT: raise
def regen(self, binary, basepath): ignores = ("dir", "dir.old") try: files = listdir_files(basepath) except OSError as oe: if oe.errno == errno.ENOENT: return raise if self.should_skip_directory(basepath, files): return # wipe old indexes. for x in set(ignores).intersection(files): os.remove(pjoin(basepath, x)) index = pjoin(basepath, 'dir') for x in files: if x in ignores or x.startswith("."): continue ret, data = spawn.spawn_get_output( [binary, '--quiet', pjoin(basepath, x), '--dir-file', index], collect_fds=(1,2), split_lines=False) if not data or "already exists" in data or \ "warning: no info dir entry" in data: continue yield pjoin(basepath, x)
def _install_targets(self, targets): dirs = set() for x in targets: basename = os.path.basename(x) suffix = os.path.splitext(basename)[1] if self.eapi.archive_suffixes_re.match(suffix): # TODO: uncompress/warn? suffix = os.path.splitext(basename.rsplit('.', 1)[0])[1] name = basename mandir = f'man{suffix[1:]}' if self.language_override and self.opts.i18n: mandir = pjoin(self.opts.i18n, mandir) elif self.language_detect: match = self.detect_lang_re.match(basename) if match: name = f'{match.group(0)}.{match.group(3)}' mandir = pjoin(match.group(1), mandir) if self.valid_mandir_re.match(mandir): if mandir not in dirs: self.install_dirs([mandir]) dirs.add(mandir) self.install([(x, pjoin(mandir, name))]) else: raise IpcCommandError(f'invalid man page: {x}')
def setUp(self): TempDirMixin.setUp(self) self.dir_orig = self.dir self.dir_master = pjoin(self.dir, 'master') self.dir_slave = pjoin(self.dir, 'slave') ensure_dirs(self.dir_master) ensure_dirs(self.dir_slave) self.dir = self.dir_slave self.master_pdir = pjoin(self.dir_master, 'profiles') self.pdir = self.slave_pdir = pjoin(self.dir_slave, 'profiles') ensure_dirs(self.master_pdir) ensure_dirs(self.slave_pdir) with open(pjoin(self.master_pdir, 'repo_name'), 'w') as f: f.write('master\n') with open(pjoin(self.slave_pdir, 'repo_name'), 'w') as f: f.write('slave\n') ensure_dirs(pjoin(self.dir_master, 'metadata')) ensure_dirs(pjoin(self.dir_slave, 'metadata')) with open(pjoin(self.dir_master, 'metadata', 'layout.conf'), 'w') as f: f.write('masters =\n') with open(pjoin(self.dir_slave, 'metadata', 'layout.conf'), 'w') as f: f.write('masters = master\n')
def _add_sets(self): self["world"] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.filelist.WorldFile", "location": pjoin(self.root, econst.WORLD_FILE.lstrip('/'))}) self["system"] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.system.SystemSet", "profile": "profile"}) self["installed"] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.installed.Installed", "vdb": "vdb"}) self["versioned-installed"] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.installed.VersionedInstalled", "vdb": "vdb"}) set_fp = pjoin(self.dir, "sets") try: for setname in listdir_files(set_fp): # Potential for name clashes here, those will just make # the set not show up in config. if setname in ("system", "world"): logger.warning( "user defined set %r is disallowed; ignoring", pjoin(set_fp, setname)) continue self[setname] = basics.AutoConfigSection({ "class": "pkgcore.pkgsets.filelist.FileList", "location": pjoin(set_fp, setname)}) except FileNotFoundError: pass
def test_load_repos_conf_dir(self): # repo priority sorting and dir/symlink scanning repos_conf_dir = pjoin(self.dir, 'repos.conf') os.mkdir(repos_conf_dir) repos_conf_sym = pjoin(self.dir, 'repos.conf.sym') os.symlink(repos_conf_dir, repos_conf_sym) # add global repos.conf shutil.copyfile( pjoin(const.CONFIG_PATH, 'repos.conf'), pjoin(repos_conf_dir, 'repos.conf')) with open(pjoin(repos_conf_dir, 'z'), 'w') as f: f.write(textwrap.dedent('''\ [bar] location = /var/gentoo/repos/bar [foo] location = /var/gentoo/repos/foo priority = 10''')) f.flush() defaults, repos = load_repos_conf(repos_conf_dir) sym_defaults, sym_repos = load_repos_conf(repos_conf_sym) self.assertEqual(defaults, sym_defaults) self.assertEqual(repos, sym_repos) self.assertEqual('gentoo', defaults['main-repo']) self.assertEqual(['foo', 'bar', 'gentoo'], repos.keys())
def _visibility_limiters(self): path = pjoin(self.base, 'profiles', 'package.mask') pos, neg = [], [] try: if (self.config.eapi.options['has_profile_data_dirs'] or self.config.profile_formats.intersection(['portage-1', 'portage-2'])): paths = sorted_scan(path) else: paths = [path] for path in paths: for line in iter_read_bash(path): line = line.strip() if line in ('-', ''): raise profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', "encountered empty negation: -") if line.startswith('-'): neg.append(atom.atom(line[1:])) else: pos.append(atom.atom(line)) except FileNotFoundError: pass except ebuild_errors.MalformedAtom as e: raise profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', e) from e return tuple(neg), tuple(pos)
def test_licenses(self): licenses = ('GPL-2', 'GPL-3+', 'BSD') ensure_dirs(pjoin(self.dir, 'licenses')) for license in licenses: touch(pjoin(self.dir, 'licenses', license)) repo = self.mk_tree(self.dir) self.assertEqual(sorted(repo.licenses), sorted(licenses))
def _internal_offset_iter_scan(path, chksum_handlers, offset, stat_func=os.lstat, hidden=True, backup=True): offset = normpath(offset) path = normpath(path) dirs = collections.deque([path[len(offset):]]) if dirs[0]: yield gen_obj(dirs[0], chksum_handlers=chksum_handlers, stat_func=stat_func) sep = os.path.sep while dirs: base = dirs.popleft() real_base = pjoin(offset, base.lstrip(sep)) base = base.rstrip(sep) + sep for x in listdir(real_base): if not hidden and x.startswith('.'): continue if not backup and x.endswith('~'): continue path = pjoin(base, x) obj = gen_obj(path, chksum_handlers=chksum_handlers, real_location=pjoin(real_base, x), stat_func=os.lstat) yield obj if obj.is_dir: dirs.append(path)
def _caching_grab_virtuals(repo, cache_basedir): virtuals = {} update = False cache = _read_mtime_cache(pjoin(cache_basedir, 'virtuals.cache')) existing = _get_mtimes(repo.location) for cat, mtime in existing.iteritems(): d = cache.pop(cat, None) if d is not None and long(d[0]) == long(mtime): d = _convert_cached_virtuals(d) if d is not None: _merge_virtuals(virtuals, d) continue update = True _collect_virtuals(virtuals, repo.itermatch( packages.PackageRestriction("category", values.StrExactMatch(cat)))) if update or cache: _write_mtime_cache(existing, virtuals, pjoin(cache_basedir, 'virtuals.cache')) defaults = _collect_default_providers(virtuals) # _finalize_virtuals(virtuals) return defaults, virtuals
def _clean_old_caches(path): for name in ('plugincache2',): try: osutils.unlink_if_exists(pjoin(path, name)) except EnvironmentError, e: logger.error("attempting to clean old plugin cache %r failed with %s", pjoin(path, name), e)
def process_subcommands(self, parser, name, action_group): l = [] h = self._get_formatter(parser, name) h.add_arguments(action_group._group_actions) data = h.format_help().strip() if data: assert len(action_group._group_actions) == 1 l.extend(_rst_header("=", action_group.title)) if action_group.description: l.extend(action_group.description.split("\n")) for subcommand, parser in action_group._group_actions[0].choices.iteritems(): subdir_path = self.name.split()[1:] base = pjoin(self.base_path, *subdir_path) self.__class__(base, "%s %s" % ( self.name, subcommand), parser, mtime=self.mtime, out_name=subcommand).run() toc_path = self.name.split() if subdir_path: toc_path = subdir_path l.append('') l.append(".. toctree::") l.append(" :maxdepth: 2") l.append('') l.extend(" %s %s <%s>" % (name, subcommand, pjoin(*list(toc_path + [subcommand]))) for subcommand in action_group._group_actions[0].choices) l.append('') return l
def _split(self, iterable, observer, engine, cset): debug_store = pjoin(engine.offset, self._debug_storage.lstrip('/')) objcopy_args = [self.objcopy_binary, '--only-keep-debug'] if self._compress: objcopy_args.append('--compress-debug-sections') for fs_objs, ftype in iterable: if 'ar archive' in ftype: continue if 'relocatable' in ftype: if not any(x.basename.endswith(".ko") for x in fs_objs): continue fs_obj = fs_objs[0] debug_loc = pjoin(debug_store, fs_obj.location.lstrip('/') + ".debug") if debug_loc in cset: continue fpath = fs_obj.data.path debug_ondisk = pjoin(os.path.dirname(fpath), os.path.basename(fpath) + ".debug") # note that we tell the UI the final pathway- not the intermediate one. observer.info("splitdebug'ing %s into %s" % (fs_obj.location, debug_loc)) ret = spawn.spawn(objcopy_args + [fpath, debug_ondisk]) if ret != 0: observer.warn("splitdebug'ing %s failed w/ exitcode %s" % (fs_obj.location, ret)) continue # note that the given pathway to the debug file /must/ be relative to ${D}; # it must exist at the time of invocation. ret = spawn.spawn([self.objcopy_binary, '--add-gnu-debuglink', debug_ondisk, fpath]) if ret != 0: observer.warn("splitdebug created debug file %r, but " "failed adding links to %r (%r)" % (debug_ondisk, fpath, ret)) observer.debug("failed splitdebug command was %r", (self.objcopy_binary, '--add-gnu-debuglink', debug_ondisk, fpath)) continue debug_obj = gen_obj(debug_loc, real_location=debug_ondisk, uid=os_data.root_uid, gid=os_data.root_gid) stripped_fsobj = self._strip_fsobj(fs_obj, ftype, observer, quiet=True) self._modified.add(stripped_fsobj) self._modified.add(debug_obj) for fs_obj in fs_objs[1:]: debug_loc = pjoin(debug_store, fs_obj.location.lstrip('/') + ".debug") linked_debug_obj = debug_obj.change_attributes(location=debug_loc) observer.info("splitdebug hardlinking %s to %s" % (debug_obj.location, debug_loc)) self._modified.add(linked_debug_obj) self._modified.add(stripped_fsobj.change_attributes( location=fs_obj.location))
def trigger(self, engine): bin_path = self.get_binary_path() if bin_path is None: return offset = engine.offset locs = [pjoin(offset, x.lstrip(os.path.sep)) for x in self.locations] if engine.phase.startswith('pre_'): self.saved_mtimes.set_state(locs) return elif engine.phase == 'post_merge' and \ engine.mode == const.REPLACE_MODE: # skip post_merge for replace. # we catch it on unmerge... return regens = set(x.location for x in self.saved_mtimes.get_changes(locs)) # force regeneration of any directory lacking the info index. regens.update(x for x in locs if not os.path.isfile(pjoin(x, 'dir'))) bad = [] for x in regens: bad.extend(self.regen(bin_path, x)) if bad and engine.observer is not None: engine.observer.warn("bad info files: %r" % sorted(bad))
def _get_versions(self, catpkg): cppath = pjoin(self.base, catpkg[0], catpkg[1]) pkg = catpkg[-1] + "-" lp = len(pkg) extension = self.extension ext_len = -len(extension) try: ret = tuple(x[lp:ext_len] for x in listdir_files(cppath) if x[ext_len:] == extension and x[:lp] == pkg) if any(('scm' in x or '-try' in x) for x in ret): if not self.ignore_paludis_versioning: for x in ret: if 'scm' in x: raise ebuild_errors.InvalidCPV( "%s/%s-%s has nonstandard -scm " "version component" % (catpkg + (x,))) elif 'try' in x: raise ebuild_errors.InvalidCPV( "%s/%s-%s has nonstandard -try " "version component" % (catpkg + (x,))) raise AssertionError('unreachable codepoint was reached') return tuple(x for x in ret if ('scm' not in x and 'try' not in x)) return ret except EnvironmentError as e: raise_from(KeyError( "failed fetching versions for package %s: %s" % (pjoin(self.base, '/'.join(catpkg)), str(e))))
def add_profile(config, config_dir, profile_override=None): if profile_override is None: profile = _find_profile_link(config_dir) else: profile = normpath(abspath(profile_override)) if not os.path.exists(profile): raise_from(errors.ComplexInstantiationError( "%s doesn't exist" % (profile,))) paths = profiles.OnDiskProfile.split_abspath(profile) if paths is None: raise errors.ComplexInstantiationError( '%s expands to %s, but no profile detected' % (pjoin(config_dir, 'make.profile'), profile)) user_profile_path = pjoin(config_dir, 'profile') if os.path.isdir(user_profile_path): config["profile"] = basics.AutoConfigSection({ "class": "pkgcore.ebuild.profiles.UserProfile", "parent_path": paths[0], "parent_profile": paths[1], "user_path": user_profile_path, }) else: config["profile"] = basics.AutoConfigSection({ "class": "pkgcore.ebuild.profiles.OnDiskProfile", "basepath": paths[0], "profile": paths[1], })
def test_load_make_conf(self): self.assertIn('PORTAGE_TMPDIR', self.make_globals) # nonexistent file d = {} # by default files are required self.assertRaises( errors.ParsingError, load_make_conf, d, pjoin(self.dir, 'make.globals')) # should return empty dict when not required load_make_conf(d, pjoin(self.dir, 'make.conf'), required=False) self.assertEqual({}, d) # unreadable file d = {} with NamedTemporaryFile() as f: os.chmod(f.name, stat.S_IWUSR) self.assertRaises( errors.PermissionDeniedError, load_make_conf, d, f.name) # overrides and incrementals with NamedTemporaryFile() as f: f.write(b'DISTDIR=foo\nACCEPT_LICENSE=foo\n') f.flush() d = {} load_make_conf(d, pjoin(const.CONFIG_PATH, 'make.globals')) load_make_conf(d, f.name, allow_sourcing=True, incrementals=True) self.assertEqual('foo', d['DISTDIR']) self.assertEqual( ' '.join([self.make_globals['ACCEPT_LICENSE'], 'foo']), d['ACCEPT_LICENSE'])
def trigger(self, engine, existing_cset, install_cset): # hackish, but it works. protected_filter = gen_config_protect_filter( engine.offset, self.extra_protects, self.extra_disables).match ignore_filter = gen_collision_ignore_filter(engine.offset).match protected = {} for x in existing_cset.iterfiles(): if not ignore_filter(x.location) and protected_filter(x.location): replacement = install_cset[x] if not simple_chksum_compare(replacement, x): protected.setdefault( pjoin(engine.offset, os.path.dirname(x.location).lstrip(os.path.sep)), []).append((os.path.basename(replacement.location), replacement)) for dir_loc, entries in protected.iteritems(): updates = {x[0]: [] for x in entries} try: existing = sorted(x for x in listdir_files(dir_loc) if x.startswith("._cfg")) except OSError as oe: if oe.errno != errno.ENOENT: raise # this shouldn't occur. continue for x in existing: try: # ._cfg0000_filename count = int(x[5:9]) if x[9] != "_": raise ValueError fn = x[10:] except (ValueError, IndexError): continue if fn in updates: updates[fn].append((count, fn)) # now we rename. for fname, entry in entries: # check for any updates with the same chksums. count = 0 for cfg_count, cfg_fname in updates[fname]: if simple_chksum_compare(livefs.gen_obj( pjoin(dir_loc, cfg_fname)), entry): count = cfg_count break count = max(count, cfg_count + 1) try: install_cset.remove(entry) except KeyError: # this shouldn't occur... continue new_fn = pjoin(dir_loc, "._cfg%04i_%s" % (count, fname)) new_entry = entry.change_attributes(location=new_fn) install_cset.add(new_entry) self.renames[new_entry] = entry del updates
def _visibility_limiters(self): path = pjoin(self.base, 'profiles', 'package.mask') pos, neg = [], [] try: if self.config.profile_formats.intersection(['portage-1', 'portage-2']): paths = sorted_scan(path) else: paths = [path] for path in paths: for line in iter_read_bash(path): line = line.strip() if line in ('-', ''): raise profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', "encountered empty negation: -") if line.startswith('-'): neg.append(atom.atom(line[1:])) else: pos.append(atom.atom(line)) except IOError as i: if i.errno != errno.ENOENT: raise except ebuild_errors.MalformedAtom as ma: raise_from(profiles.ProfileError( pjoin(self.base, 'profiles'), 'package.mask', ma)) return [neg, pos]
def mk_tree(self, path, *args, **kwds): eclasses = kwds.pop('eclass_cache', None) if eclasses is None: epath = pjoin(path, 'eclass') ensure_dirs(epath) eclasses = eclass_cache.cache(epath) ensure_dirs(pjoin(path, 'profiles')) return repository.UnconfiguredTree(path, eclass_cache=eclasses, *args, **kwds)
def test_dangling_symlink(self): src = self.gen_dir("src") self.generate_tree(src, {"dir":["dir"]}) cset = livefs.scan(src, offset=src) dest = self.gen_dir("dest") os.symlink(pjoin(dest, "dest"), pjoin(dest, "dir")) self.assertTrue(ops.merge_contents(cset, offset=dest)) self.assertEqual(cset, livefs.scan(src, offset=dest))
def _install_targets(self, targets): dirs = set() for x in targets: d = pjoin(os.path.splitext(os.path.basename(x))[0], 'LC_MESSAGES') if d not in dirs: self.install_dirs([d]) dirs.add(d) self.install([(x, pjoin(d, f'{self.pkg.PN}.mo'))])
def _get_ebuild_path(self, pkg): if pkg.revision is None: if pkg.fullver not in self.versions[(pkg.category, pkg.package)]: # daft explicit -r0 on disk. return pjoin(self.base, pkg.category, pkg.package, "%s-%s-r0%s" % (pkg.package, pkg.fullver, self.extension)) return pjoin(self.base, pkg.category, pkg.package, \ "%s-%s%s" % (pkg.package, pkg.fullver, self.extension))
def test_extend_path(self): import mod_testplug expected = lists.stable_unique( pjoin(p, 'mod_testplug') for p in sys.path if os.path.isdir(p)) self.assertEqual(expected, mod_testplug.__path__, set(expected) ^ set(mod_testplug.__path__))
def __init__(self, userpriv, sandbox, fakeroot, save_file): """ :param sandbox: enables a sandboxed processor :param userpriv: enables a userpriv'd processor :param fakeroot: enables a fakeroot'd processor- this is a mutually exclusive option to sandbox, and requires userpriv to be enabled. Violating this will result in nastiness. """ self.lock() self.ebd = e_const.EBUILD_DAEMON_PATH spawn_opts = {'umask': 0002} self._preloaded_eclasses = {} self._eclass_caching = False self._outstanding_expects = [] self._metadata_paths = None if fakeroot and (sandbox or not userpriv): traceback.print_stack() logger.error( "Both sandbox and fakeroot cannot be enabled at the same time") raise InitializationError( "cannot initialize with sandbox and fakeroot") if userpriv: self.__userpriv = True spawn_opts.update({ "uid": os_data.portage_uid, "gid": os_data.portage_gid, "groups": [os_data.portage_gid] }) else: if pkgcore.spawn.is_userpriv_capable(): spawn_opts.update({ "gid": os_data.portage_gid, "groups": [0, os_data.portage_gid] }) self.__userpriv = False # open the pipes to be used for chatting with the new daemon cread, cwrite = os.pipe() dread, dwrite = os.pipe() self.__sandbox = False self.__fakeroot = False # since it's questionable which spawn method we'll use (if # sandbox or fakeroot fex), we ensure the bashrc is invalid. env = { x: "/etc/portage/spork/not/valid/ha/ha" for x in ("BASHRC", "BASH_ENV") } if int(os.environ.get('PKGCORE_PERF_DEBUG', 1)) > 1: env["PKGCORE_PERF_DEBUG"] = os.environ['PKGCORE_PERF_DEBUG'] # append script dir to PATH for git repo or unpacked tarball if "PKGCORE_SCRIPT_PATH" in os.environ: env["PATH"] = os.pathsep.join( [os.environ["PATH"], os.environ["PKGCORE_SCRIPT_PATH"]]) args = [] if sandbox: if not pkgcore.spawn.is_sandbox_capable(): raise ValueError("spawn lacks sandbox capabilities") if fakeroot: raise InitializationError( 'fakeroot was on, but sandbox was also on') self.__sandbox = True spawn_func = pkgcore.spawn.spawn_sandbox # env.update({"SANDBOX_DEBUG":"1", "SANDBOX_DEBUG_LOG":"/var/tmp/test"}) elif fakeroot: if not pkgcore.spawn.is_fakeroot_capable(): raise ValueError("spawn lacks fakeroot capabilities") self.__fakeroot = True spawn_func = pkgcore.spawn.spawn_fakeroot args.append(save_file) else: spawn_func = pkgcore.spawn.spawn # force to a neutral dir so that sandbox/fakeroot won't explode if # ran from a nonexistent dir spawn_opts["cwd"] = e_const.EAPI_BIN_PATH # little trick. we force the pipes to be high up fd wise so # nobody stupidly hits 'em. max_fd = min(pkgcore.spawn.max_fd_limit, 1024) env.update({ "PKGCORE_EBD_READ_FD": str(max_fd - 2), "PKGCORE_EBD_WRITE_FD": str(max_fd - 1) }) self.pid = spawn_func(["/bin/bash", self.ebd, "daemonize"], fd_pipes={ 0: 0, 1: 1, 2: 2, max_fd - 2: cread, max_fd - 1: dwrite }, returnpid=True, env=env, *args, **spawn_opts)[0] os.close(cread) os.close(dwrite) self.ebd_write = os.fdopen(cwrite, "w") self.ebd_read = os.fdopen(dread, "r") # basically a quick "yo" to the daemon self.write("dude?") if not self.expect("dude!"): logger.error("error in server coms, bailing.") raise InitializationError( "expected 'dude!' response from ebd, which wasn't received. " "likely a bug") self.write(e_const.EAPI_BIN_PATH) # send PKGCORE_PYTHON_BINARY... self.write(pkgcore.spawn.find_invoking_python()) self.write( os.pathsep.join([ normpath(abspath(pjoin(pkgcore.__file__, os.pardir, os.pardir))), os.environ.get('PYTHONPATH', '') ])) if self.__sandbox: self.write("sandbox_log?") self.__sandbox_log = self.read().split()[0] self.dont_export_vars = self.read().split() # locking isn't used much, but w/ threading this will matter self.unlock()
def fetch(self, target): """Fetch a file. :type target: :obj:`pkgcore.fetch.fetchable` instance :return: None if fetching failed, else on disk location of the copied file """ if not isinstance(target, fetchable): raise TypeError( f"target must be fetchable instance/derivative: {target}") kw = {"mode": 0o775} if self.readonly: kw["mode"] = 0o555 if self.userpriv: kw["gid"] = portage_gid kw["minimal"] = True if not ensure_dirs(self.distdir, **kw): raise errors.DistdirPerms( self.distdir, "if userpriv, uid must be %i, gid must be %i. " "if not readonly, directory must be 0775, else 0555" % (portage_uid, portage_gid)) path = pjoin(self.distdir, target.filename) uris = iter(target.uri) last_exc = RuntimeError("fetching failed for an unknown reason") spawn_opts = {'umask': 0o002, 'env': self.extra_env} if self.userpriv and is_userpriv_capable(): spawn_opts.update({"uid": portage_uid, "gid": portage_gid}) for _attempt in range(self.attempts): try: self._verify(path, target) return path except errors.MissingDistfile as e: command = self.command last_exc = e except errors.ChksumFailure: raise except errors.FetchFailed as e: last_exc = e if not e.resumable: try: os.unlink(path) command = self.command except OSError as e: raise errors.UnmodifiableFile(path, e) from e else: command = self.resume_command # Note we're not even checking the results, the verify portion of # the loop handles this. In other words, don't trust the external # fetcher's exit code, trust our chksums instead. try: spawn_bash( command % { "URI": next(uris), "FILE": target.filename }, **spawn_opts) except StopIteration: raise errors.FetchFailed(target.filename, "ran out of urls to fetch from") else: raise last_exc
def pm_tmpdir(self): """Temporary directory for the package manager.""" return pjoin(self.tmpdir, 'portage')
def __init__(self, path, parent_path): self.override_path = pjoin(path, parent_path) ProfileNode.__init__(self, path, pms_strict=False)
def _get_ebuild_path(self, pkg): return pjoin(self.base, pkg.category, pkg.package, f"{pkg.package}-{pkg.fullver}{self.extension}")
def discern_loc(base, pkg, extension='.tbz2'): return pjoin(base, pkg.category, "%s-%s%s" % (pkg.package, pkg.fullver, extension))
def setUp(self): TempDirMixin.setUp(self) self.pdir = pjoin(self.dir, 'profiles') ensure_dirs(self.pdir)
def test_path_restrict(self): repo_dir = pjoin(self.dir, 'repo') sym_repo_dir = pjoin(self.dir, 'sym_repo') os.symlink(repo_dir, sym_repo_dir) ensure_dirs(pjoin(repo_dir, 'profiles')) with open(pjoin(repo_dir, 'profiles', 'repo_name'), 'w') as f: f.write('testrepo\n') ensure_dirs(pjoin(repo_dir, 'cat', 'foo')) ensure_dirs(pjoin(repo_dir, 'cat', 'bar')) ensure_dirs(pjoin(repo_dir, 'tac', 'oof')) touch(pjoin(repo_dir, 'skel.ebuild')) touch(pjoin(repo_dir, 'cat', 'foo', 'foo-1.ebuild')) touch(pjoin(repo_dir, 'cat', 'foo', 'foo-2.ebuild')) touch(pjoin(repo_dir, 'cat', 'foo', 'Manifest')) touch(pjoin(repo_dir, 'cat', 'bar', 'bar-1.ebuild')) touch(pjoin(repo_dir, 'tac', 'oof', 'oof-1.ebuild')) # specify repo category dirs with open(pjoin(repo_dir, 'profiles', 'categories'), 'w') as f: f.write('cat\n') f.write('tac\n') for d in (repo_dir, sym_repo_dir): repo = self.mk_tree(d) for path in ( self.dir, # path not in repo pjoin(repo.location, 'a'), # nonexistent category dir pjoin(repo.location, 'profiles'), # non-category dir pjoin( repo.location, 'skel.ebuild'), # not in the correct cat/PN dir layout pjoin(repo.location, 'cat', 'a'), # nonexistent package dir pjoin(repo.location, 'cat', 'foo', 'foo-0.ebuild'), # nonexistent ebuild file pjoin(repo.location, 'cat', 'foo', 'Manifest')): # non-ebuild file self.assertRaises(ValueError, repo.path_restrict, path) # repo dir restriction = repo.path_restrict(repo.location) self.assertEqual(len(restriction), 1) self.assertInstance(restriction[0], restricts.RepositoryDep) # matches all 4 ebuilds in the repo self.assertEqual(len(repo.match(restriction)), 4) # category dir restriction = repo.path_restrict(pjoin(repo.location, 'cat')) self.assertEqual(len(restriction), 2) self.assertInstance(restriction[1], restricts.CategoryDep) # matches all 3 ebuilds in the category self.assertEqual(len(repo.match(restriction)), 3) # package dir restriction = repo.path_restrict(pjoin(repo.location, 'cat', 'foo')) self.assertEqual(len(restriction), 3) self.assertInstance(restriction[2], restricts.PackageDep) # matches both ebuilds in the package dir self.assertEqual(len(repo.match(restriction)), 2) # ebuild file restriction = repo.path_restrict( pjoin(repo.location, 'cat', 'foo', 'foo-1.ebuild')) self.assertEqual(len(restriction), 4) self.assertInstance(restriction[3], restricts.VersionMatch) # specific ebuild version match self.assertEqual(len(repo.match(restriction)), 1) # relative ebuild file path with mock.patch('os.getcwd', return_value=os.path.realpath( pjoin(repo.location, 'cat', 'foo'))): restriction = repo.path_restrict('./foo-1.ebuild') self.assertEqual(len(restriction), 4) self.assertInstance(restriction[3], restricts.VersionMatch) # specific ebuild version match self.assertEqual(len(repo.match(restriction)), 1)
def bashrc(self): path = pjoin(self.path, "profile.bashrc") if os.path.exists(path): return local_source(path) return None
def __init__(self, basepath, profile, load_profile_base=True): ProfileStack.__init__(self, pjoin(basepath, profile)) self.basepath = basepath self.load_profile_base = load_profile_base
def _get_manifest(self, category, package): return digest.Manifest(pjoin(self.base, category, package, "Manifest"), thin=self.config.manifests.thin, enforce_gpg=self.enable_gpg)
def finish(self): unknown_pkgs = defaultdict(lambda: defaultdict(list)) unknown_pkg_use = defaultdict(lambda: defaultdict(list)) unknown_use = defaultdict(lambda: defaultdict(list)) unknown_keywords = defaultdict(lambda: defaultdict(list)) def _pkg_atoms(filename, profile, vals): for a in iflatten_instance(vals, atom.atom): if not self.repo.match(a): unknown_pkgs[profile.path][filename].append(a) def _pkg_keywords(filename, profile, vals): for atom, keywords in vals: invalid = set(keywords) - self.valid_keywords if invalid: unknown_keywords[profile.path][filename].append( (atom, invalid)) def _pkg_use(filename, profile, vals): # TODO: give ChunkedDataDict some dict view methods d = vals if isinstance(d, misc.ChunkedDataDict): d = vals.render_to_dict() for _pkg, entries in d.items(): for a, disabled, enabled in entries: pkgs = self.repo.match(a) if not pkgs: unknown_pkgs[profile.path][filename].append(a) else: available = { u for pkg in pkgs for u in pkg.iuse_stripped } unknown_disabled = set(disabled) - available unknown_enabled = set(enabled) - available if unknown_disabled: unknown_pkg_use[profile.path][filename].append( (a, ('-' + u for u in unknown_disabled))) if unknown_enabled: unknown_pkg_use[profile.path][filename].append( (a, unknown_enabled)) def _use(filename, profile, vals): # TODO: give ChunkedDataDict some dict view methods d = vals.render_to_dict() for _, entries in d.items(): for _, disabled, enabled in entries: unknown_disabled = set(disabled) - self.available_iuse unknown_enabled = set(enabled) - self.available_iuse if unknown_disabled: unknown_use[profile.path][filename].extend( ('-' + u for u in unknown_disabled)) if unknown_enabled: unknown_use[profile.path][filename].extend( unknown_enabled) def _deprecated(filename, profile, vals): # make sure replacement profile exists if vals is not None: replacement, msg = vals try: _ProfileNode(pjoin(self.profiles_dir, replacement)) except profiles_mod.ProfileError as e: yield ProfileError( f'nonexistent replacement {replacement!r} ' f'for deprecated profile: {profile.name!r}') file_parse_map = { 'packages': ('packages', _pkg_atoms), 'package.mask': ('masks', _pkg_atoms), 'package.unmask': ('unmasks', _pkg_atoms), 'package.use': ('pkg_use', _pkg_use), 'package.use.force': ('pkg_use_force', _pkg_use), 'package.use.stable.force': ('pkg_use_stable_force', _pkg_use), 'package.use.mask': ('pkg_use_mask', _pkg_use), 'package.use.stable.mask': ('pkg_use_stable_mask', _pkg_use), 'use.force': ('use_force', _use), 'use.stable.force': ('use_stable_force', _use), 'use.mask': ('use_mask', _use), 'use.stable.mask': ('use_stable_mask', _use), 'parent': ('parents', lambda *args: None), 'deprecated': ('deprecated', _deprecated), # non-PMS files 'package.keywords': ('keywords', _pkg_keywords), 'package.accept_keywords': ('accept_keywords', _pkg_keywords), } profile_reports = [] report_profile_warnings = lambda x: profile_reports.append( ProfileWarning(x)) report_profile_errors = lambda x: profile_reports.append( ProfileError(x)) for root, _dirs, files in os.walk(self.profiles_dir): if root not in self.non_profile_dirs: profile = _ProfileNode(root) for f in set(files).intersection(file_parse_map.keys()): attr, func = file_parse_map[f] file_path = pjoin(root[len(self.profiles_dir) + 1:], f) # convert log warnings/errors into reports with patch('pkgcore.log.logger.error', report_profile_errors), \ patch('pkgcore.log.logger.warning', report_profile_warnings): vals = getattr(profile, attr) results = func(f, profile, vals) if results is not None: yield from results yield from profile_reports for path, filenames in sorted(unknown_pkgs.items()): for filename, vals in filenames.items(): pkgs = map(str, vals) yield UnknownProfilePackages( pjoin(path[len(self.profiles_dir):].lstrip('/'), filename), pkgs) for path, filenames in sorted(unknown_pkg_use.items()): for filename, vals in filenames.items(): for pkg, flags in vals: yield UnknownProfilePackageUse( pjoin(path[len(self.profiles_dir):].lstrip('/'), filename), str(pkg), flags) for path, filenames in sorted(unknown_use.items()): for filename, vals in filenames.items(): yield UnknownProfileUse( pjoin(path[len(self.profiles_dir):].lstrip('/'), filename), vals) for path, filenames in sorted(unknown_keywords.items()): for filename, vals in filenames.items(): for pkg, keywords in vals: yield UnknownProfilePackageKeywords( pjoin(path[len(self.profiles_dir):].lstrip('/'), filename), str(pkg), keywords)
pjoin(self.env["D"], self.prefix.lstrip('/'))) + "/" def get_env_source(self): with open(pjoin(self.env["T"], "environment"), "rb") as f: return data_source.bytes_data_source(f.read()) def setup_env_data_source(self): if not ensure_dirs( self.env["T"], mode=0770, gid=portage_gid, minimal=True): raise format.FailedDirectory( self.env['T'], "%s doesn't fulfill minimum mode %o and gid %i" % (self.env['T'], 0770, portage_gid)) if self.env_data_source is not None: fp = pjoin(self.env["T"], "environment") # load data first (might be a local_source), *then* write # if it's a src_ebuild being installed, trying to do two steps # stomps the local_sources data. data = self.env_data_source.bytes_fileobj().read() with open(fp, "wb") as f: f.write(data) del data def _set_per_phase_env(self, phase, env): self._setup_merge_type(phase, env) def _setup_merge_type(self, phase, env): # only allowed in pkg_ phases. if (not self.eapi.phases.get(phase, "").startswith("pkg_")
def finish(self): # don't check for unknown category dirs on overlays if self.options.gentoo_repo: category_dirs = set( filterfalse(self.repo.false_categories.__contains__, (x for x in listdir_dirs(self.repo.location) if x[0] != '.'))) unknown_categories = category_dirs.difference(self.repo.categories) if unknown_categories: yield UnknownCategories(sorted(unknown_categories)) arches_without_profiles = set(self.arches) - set( self.repo.profiles.arches()) if arches_without_profiles: yield ArchesWithoutProfiles(sorted(arches_without_profiles)) root_profile_dirs = {'embedded'} available_profile_dirs = set() for root, _dirs, _files in os.walk(self.profiles_dir): d = root[len(self.profiles_dir):].lstrip('/') if d: available_profile_dirs.add(d) available_profile_dirs -= self.non_profile_dirs | root_profile_dirs profile_reports = [] report_profile_warnings = lambda x: profile_reports.append( ProfileWarning(x)) report_profile_errors = lambda x: profile_reports.append( ProfileError(x)) # don't check for acceptable profile statuses on overlays if self.options.gentoo_repo: known_profile_statuses = self.known_profile_statuses else: known_profile_statuses = None # forcibly parse profiles.desc and convert log warnings/errors into reports with patch('pkgcore.log.logger.error', report_profile_errors), \ patch('pkgcore.log.logger.warning', report_profile_warnings): profiles = Profiles.parse(self.profiles_dir, self.repo.repo_id, known_status=known_profile_statuses, known_arch=self.arches) yield from profile_reports seen_profile_dirs = set() lagging_profile_eapi = defaultdict(list) for p in profiles: try: profile = profiles_mod.ProfileStack( pjoin(self.profiles_dir, p.path)) except profiles_mod.ProfileError: yield NonexistentProfilePath(p.path) continue for parent in profile.stack: seen_profile_dirs.update( dir_parents(parent.path[len(self.profiles_dir):])) # flag lagging profile EAPIs -- assumes EAPIs are sequentially # numbered which should be the case for the gentoo repo if (self.options.gentoo_repo and str(profile.eapi) < str(parent.eapi)): lagging_profile_eapi[profile].append(parent) for profile, parents in lagging_profile_eapi.items(): parent = parents[-1] yield LaggingProfileEapi(profile.name, str(profile.eapi), parent.name, str(parent.eapi)) unused_profile_dirs = available_profile_dirs - seen_profile_dirs if unused_profile_dirs: yield UnusedProfileDirs(sorted(unused_profile_dirs))
def _cmd_implementation_digests(self, domain, matches, observer, mirrors=False, force=False): manifest_config = self.repo.config.manifests if manifest_config.disabled: observer.info(f"repo {self.repo.repo_id} has manifests disabled") return required_chksums = set(manifest_config.required_hashes) write_chksums = manifest_config.hashes distdir = domain.fetcher.distdir ret = set() for key_query in sorted( set(match.unversioned_atom for match in matches)): pkgs = self.repo.match(key_query) # check for pkgs masked by bad metadata bad_metadata = self.repo._bad_masked.match(key_query) if bad_metadata: for pkg in bad_metadata: e = pkg.data error_str = f"{pkg.cpvstr}: {e.msg(verbosity=observer.verbosity)}" observer.error(error_str) ret.add(key_query) continue # Check for bad ebuilds -- mismatched or invalid PNs won't be # matched by regular restrictions so they will otherwise be # ignored. ebuilds = { x for x in listdir_files( pjoin(self.repo.location, str(key_query))) if x.endswith('.ebuild') } unknown_ebuilds = ebuilds.difference( os.path.basename(x.path) for x in pkgs) if unknown_ebuilds: error_str = ( f"{key_query}: invalid ebuild{_pl(unknown_ebuilds)}: " f"{', '.join(unknown_ebuilds)}") observer.error(error_str) ret.add(key_query) continue # empty package dir if not pkgs: continue manifest = pkgs[0].manifest # all pkgdir fetchables pkgdir_fetchables = {} for pkg in pkgs: pkgdir_fetchables.update({ fetchable.filename: fetchable for fetchable in iflatten_instance( pkg._get_attr['fetchables'] (pkg, allow_missing_checksums=True, skip_default_mirrors=(not mirrors)), fetch.fetchable) }) # fetchables targeted for (re-)manifest generation fetchables = {} chksum_set = set(write_chksums) for filename, fetchable in pkgdir_fetchables.items(): if force or not required_chksums.issubset(fetchable.chksums): fetchable.chksums = { k: v for k, v in fetchable.chksums.items() if k in chksum_set } fetchables[filename] = fetchable # Manifest files aren't necessary with thin manifests and no distfiles if manifest_config.thin and not pkgdir_fetchables: if os.path.exists(manifest.path): try: os.remove(manifest.path) except EnvironmentError as e: observer.error( 'failed removing old manifest: ' f'{key_query}::{self.repo.repo_id}: {e}') ret.add(key_query) continue # Manifest file is current and not forcing a refresh if not force and manifest.distfiles.keys( ) == pkgdir_fetchables.keys(): continue pkg_ops = domain.pkg_operations(pkgs[0], observer=observer) if not pkg_ops.supports("fetch"): observer.error( f"pkg {pkg} doesn't support fetching, can't generate manifest" ) ret.add(key_query) continue # fetch distfiles if not pkg_ops.fetch(list(fetchables.values()), observer): ret.add(key_query) continue # calculate checksums for fetched distfiles try: for fetchable in fetchables.values(): chksums = chksum.get_chksums( pjoin(distdir, fetchable.filename), *write_chksums) fetchable.chksums = dict(zip(write_chksums, chksums)) except chksum.MissingChksumHandler as e: observer.error(f'failed generating chksum: {e}') ret.add(key_query) break if key_query not in ret: fetchables.update(pkgdir_fetchables) observer.info( f"generating manifest: {key_query}::{self.repo.repo_id}") manifest.update(sorted(fetchables.values()), chfs=write_chksums) return ret
def test_it(self): with open(pjoin(self.dir, "arch.list"), "w") as f: f.write("\n".join(('amd64', 'ppc', 'x86'))) self.mk_profiles({ "1": ["x86"], "2": ["x86"], "3": ["ppc"], }) with open(pjoin(self.dir, 'profiles', '1', 'package.use.stable.mask'), 'w') as f: f.write('dev-util/diffball foo') with open(pjoin(self.dir, 'profiles', '2', 'package.use.stable.force'), 'w') as f: f.write('=dev-util/diffball-0.1 bar foo') with open(pjoin(self.dir, 'profiles', '3', 'package.use.stable.force'), 'w') as f: f.write('dev-util/diffball bar foo') check = self.get_check('1', '2', '3') def get_rets(ver, attr, KEYWORDS="x86", **data): data["KEYWORDS"] = KEYWORDS pkg = FakePkg(f"dev-util/diffball-{ver}", data=data) return check.collapse_evaluate_depset(pkg, attr, getattr(pkg, attr)) # few notes... for ensuring proper profiles came through, use # sorted(x.name for x in blah); reasoning is that it will catch # if duplicates come through, *and* ensure proper profile collapsing # shouldn't return anything due to no profiles matching the keywords. assert get_rets("0.0.1", "depend", KEYWORDS="foon") == [] l = get_rets("0.0.2", "depend") assert len(l) == 1, f"must collapse all profiles down to one run: got {l!r}" assert len(l[0][1]) == 4, "must have four runs, (arch and ~arch for each profile)" assert sorted(set(x.name for x in l[0][1])) == ['1', '2'], f"must have two profiles: got {l!r}" assert l[0][1][0].key == 'x86' assert l[0][1][1].key == 'x86' l = get_rets( "0.1", "rdepend", RDEPEND="x? ( dev-util/confcache ) foo? ( dev-util/foo ) " "bar? ( dev-util/bar ) !bar? ( dev-util/nobar ) x11-libs/xserver" ) assert len(l) == 3, f"must collapse all profiles down to 3 runs: got {l!r}" # ordering is potentially random; thus pull out which depset result is # which based upon profile l1 = [x for x in l if x[1][0].name == '1'][0] l2 = [x for x in l if x[1][0].name == '2'][0] assert ( set(str(l1[0]).split()) == {'dev-util/confcache', 'dev-util/bar', 'dev-util/nobar', 'x11-libs/xserver'}) assert ( set(str(l2[0]).split()) == {'dev-util/confcache', 'dev-util/foo', 'dev-util/bar', 'x11-libs/xserver'}) # test feed wiping, using an empty depset; if it didn't clear, then # results from a pkg/attr tuple from above would come through rather # then an empty. pkg = FakePkg('dev-util/diffball-0.5') check.feed(pkg) l = get_rets("0.1", "rdepend") assert len(l) == 1, f"feed didn't clear the cache- should be len 1: {l!r}" check.feed(pkg) # ensure it handles arch right. l = get_rets("0", "depend", KEYWORDS="ppc x86") assert len(l) == 1, f"should be len 1, got {l!r}" assert sorted(set(x.name for x in l[0][1])) == ["1", "2", "3"], ( f"should have three profiles of 1-3, got {l[0][1]!r}") # ensure it's caching profile collapsing, iow, keywords for same ver # that's partially cached (single attr at least) should *not* change # things. l = get_rets("0", "depend", KEYWORDS="ppc") assert sorted(set(x.name for x in l[0][1])) == ['1', '2', '3'], ( f"should have 3 profiles, got {l[0][1]!r}\nthis indicates it's " "re-identifying profiles every invocation, which is unwarranted ") l = get_rets("1", "depend", KEYWORDS="ppc x86", DEPEND="ppc? ( dev-util/ppc ) !ppc? ( dev-util/x86 )") assert len(l) == 2, f"should be len 2, got {l!r}" # same issue, figure out what is what l1 = [x[1] for x in l if str(x[0]).strip() == "dev-util/ppc"][0] l2 = [x[1] for x in l if str(x[0]).strip() == "dev-util/x86"][0] assert sorted(set(x.name for x in l1)) == ["3"] assert sorted(set(x.name for x in l2)) == ["1", "2"]
def _get_path(self, pkg): return pjoin(self.base, pkg.category, f"{pkg.package}-{pkg.fullver}.tbz2")
def get_env_source(self): with open(pjoin(self.env["T"], "environment"), "rb") as f: return data_source.bytes_data_source(f.read())
def __init__(self, location, eclass_cache, cache=(), default_mirrors=None, override_repo_id=None, ignore_paludis_versioning=False, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri :param override_repo_id: Either None, or string to force as the repository unique id :param ignore_paludis_versioning: If False, fail when -scm is encountred. if True, silently ignore -scm ebuilds. """ prototype.tree.__init__(self) if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config self._repo_id = override_repo_id self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError("base not a dir: %s" % self.base) except OSError: raise_from( errors.InitializationError("lstat failed on base %s" % (self.base, ))) self.eclass_cache = eclass_cache self.licenses = repo_objs.Licenses(location) fp = pjoin(self.base, metadata_offset, "thirdpartymirrors") mirrors = {} try: for k, v in read_dict(fp, splitter=None).iteritems(): v = v.split() shuffle(v) mirrors[k] = v except EnvironmentError as ee: if ee.errno != errno.ENOENT: raise if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache, ) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self.ignore_paludis_versioning = ignore_paludis_versioning self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory(self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache()
def __init__(self, location, eclass_cache=None, masters=(), cache=(), default_mirrors=None, allow_missing_manifests=False, repo_config=None): """ :param location: on disk location of the tree :param cache: sequence of :obj:`pkgcore.cache.template.database` instances to use for storing metadata :param masters: repo masters this repo inherits from :param eclass_cache: If not None, :obj:`pkgcore.ebuild.eclass_cache` instance representing the eclasses available, if None, generates the eclass_cache itself :param default_mirrors: Either None, or sequence of mirrors to try fetching from first, then falling back to other uri """ super().__init__() self.base = self.location = location try: if not stat.S_ISDIR(os.stat(self.base).st_mode): raise errors.InitializationError( f"base not a dir: {self.base}") except OSError as e: raise errors.InitializationError( f"lstat failed: {self.base}") from e if repo_config is None: repo_config = repo_objs.RepoConfig(location) self.config = repo_config # profiles dir is required by PMS if not os.path.isdir(self.config.profiles_base): raise errors.InvalidRepo( f'missing required profiles dir: {self.location!r}') # verify we support the repo's EAPI if not self.is_supported: raise errors.UnsupportedRepo(self) if eclass_cache is None: eclass_cache = eclass_cache_mod.cache(pjoin( self.location, 'eclass'), location=self.location) self.eclass_cache = eclass_cache self.masters = masters self.trees = tuple(masters) + (self, ) self.licenses = repo_objs.Licenses(self.location) self.profiles = self.config.profiles if masters: self.licenses = repo_objs.OverlayedLicenses(*self.trees) self.profiles = repo_objs.OverlayedProfiles(*self.trees) # use mirrors from masters if not defined in the repo mirrors = dict(self.thirdpartymirrors) for master in masters: for k, v in master.mirrors.items(): if k not in mirrors: mirrors[k] = v if isinstance(cache, (tuple, list)): cache = tuple(cache) else: cache = (cache, ) self.mirrors = mirrors self.default_mirrors = default_mirrors self.cache = cache self._allow_missing_chksums = allow_missing_manifests self.package_class = self.package_factory(self, cache, self.eclass_cache, self.mirrors, self.default_mirrors) self._shared_pkg_cache = WeakValCache() self._bad_masked = RestrictionRepo(repo_id='bad_masked') self.projects_xml = repo_objs.LocalProjectsXml( pjoin(self.location, 'metadata', 'projects.xml'))
def __stage_step_callback__(self, stage): try: open(pjoin(self.builddir, '.%s' % (stage, )), 'w').close() except EnvironmentError: # we really don't care... pass
def __init__(self, pkg, initial_env=None, env_data_source=None, features=None, observer=None, clean=True, tmp_offset=None, use_override=None, allow_fetching=False): """ :param pkg: :class:`pkgcore.ebuild.ebuild_src.package` instance this env is being setup for :param initial_env: initial environment to use for this ebuild :param env_data_source: a :obj:`snakeoil.data_source.base` instance to restore the environment from- used for restoring the state of an ebuild processing, whether for unmerging, or walking phases during building :param features: ebuild features, hold over from portage, will be broken down at some point """ if use_override is not None: use = use_override else: use = pkg.use self.allow_fetching = allow_fetching if not hasattr(self, "observer"): self.observer = observer if not pkg.eapi.is_supported: raise TypeError("package %s uses an unsupported eapi: %s" % (pkg, pkg.eapi)) if initial_env is not None: # copy. self.env = dict(initial_env) for x in ("USE", "ACCEPT_LICENSE"): if x in self.env: del self.env[x] else: self.env = {} if "PYTHONPATH" in os.environ: self.env["PYTHONPATH"] = os.environ["PYTHONPATH"] if features is None: features = self.env.get("FEATURES", ()) # XXX: note this is just EAPI 3 compatibility; not full prefix, soon.. self.env["ROOT"] = self.domain.root self.prefix_mode = pkg.eapi.options.prefix_capable or 'force-prefix' in features self.env["PKGCORE_PREFIX_SUPPORT"] = 'false' self.prefix = '/' if self.prefix_mode: self.prefix = self.domain.prefix self.env['EPREFIX'] = self.prefix.rstrip('/') self.env['EROOT'] = abspath( pjoin(self.domain.root, self.prefix.lstrip('/'))).rstrip('/') + '/' self.env["PKGCORE_PREFIX_SUPPORT"] = 'true' # set the list of internally implemented EAPI specific functions that # shouldn't be exported if os.path.exists(pjoin(const.EBD_PATH, 'funcnames', str(pkg.eapi))): with open(pjoin(const.EBD_PATH, 'funcnames', str(pkg.eapi)), 'r') as f: eapi_funcs = f.readlines() else: ret, eapi_funcs = spawn_get_output([ pjoin(const.EBD_PATH, 'generate_eapi_func_list.bash'), str(pkg.eapi) ]) if ret != 0: raise Exception( "failed to generate list of EAPI %s specific functions" % str(pkg.eapi)) self.env["PKGCORE_EAPI_FUNCS"] = ' '.join(x.strip() for x in eapi_funcs) self.env_data_source = env_data_source if (env_data_source is not None and not isinstance(env_data_source, data_source.base)): raise TypeError( "env_data_source must be None, or a pkgcore.data_source.base " "derivative: %s: %s" % (env_data_source.__class__, env_data_source)) self.features = set(x.lower() for x in features) self.env["FEATURES"] = ' '.join(sorted(self.features)) iuse_effective_regex = (re.escape(x) for x in pkg.iuse_effective) iuse_effective_regex = "^(%s)$" % "|".join(iuse_effective_regex) iuse_effective_regex = iuse_effective_regex.replace("\\.\\*", ".*") self.env["PKGCORE_IUSE_EFFECTIVE"] = iuse_effective_regex expected_ebuild_env(pkg, self.env, env_source_override=self.env_data_source) self.env["PKGCORE_FINALIZED_RESTRICT"] = ' '.join( str(x) for x in pkg.restrict) self.restrict = pkg.restrict for x in ("sandbox", "userpriv"): setattr(self, x, self.feat_or_bool(x) and not (x in self.restrict)) if self.userpriv and os.getuid() != 0: self.userpriv = False if "PORT_LOGDIR" in self.env: self.logging = pjoin( self.env["PORT_LOGDIR"], "%s:%s:%s.log" % (pkg.cpvstr, self.__class__.__name__, time.strftime("%Y%m%d-%H%M%S", time.localtime()))) del self.env["PORT_LOGDIR"] else: self.logging = False self.env["XARGS"] = xargs self.bashrc = self.env.pop("bashrc", ()) self.pkg = pkg self.eapi = pkg.eapi wipes = [ k for k, v in self.env.iteritems() if not isinstance(v, basestring) ] for k in wipes: del self.env[k] self.set_op_vars(tmp_offset) self.clean_at_start = clean self.clean_needed = False
def __init__(self, domain, pkg, verified_files, eclass_cache, observer=None, **kwargs): """ :param pkg: :obj:`pkgcore.ebuild.ebuild_src.package` instance we'll be building :param domain_settings: dict bled down from the domain configuration; basically initial env :param eclass_cache: the :class:`pkgcore.ebuild.eclass_cache` we'll be using :param files: mapping of fetchables mapped to their disk location """ use = kwargs.get("use_override", pkg.use) domain_settings = domain.settings format.build.__init__(self, domain, pkg, verified_files, observer) ebd.__init__(self, pkg, initial_env=domain_settings, features=domain_settings["FEATURES"], **kwargs) self.env["FILESDIR"] = pjoin(os.path.dirname(pkg.ebuild.path), "files") self.eclass_cache = eclass_cache self.env["ECLASSDIR"] = eclass_cache.eclassdir # this needs to be deprecated and dropped from future EAPIs self.env["PORTDIR"] = eclass_cache.location self.run_test = self.feat_or_bool("test", domain_settings) self.allow_failed_test = self.feat_or_bool("test-fail-continue", domain_settings) if "test" in self.restrict: self.run_test = False elif "test" not in use: if self.run_test: logger.warning( "disabling test for %s due to test use flag being disabled" % pkg) self.run_test = False # XXX minor hack path = self.env["PATH"].split(":") for s, default in (("DISTCC", ".distcc"), ("CCACHE", "ccache")): b = (self.feat_or_bool(s, domain_settings) and s not in self.restrict) setattr(self, s.lower(), b) if b: # looks weird I realize, but # pjoin("/foor/bar", "/barr/foo") == "/barr/foo" # and pjoin("/foo/bar", ".asdf") == "/foo/bar/.asdf" self.env.setdefault(s + "_DIR", pjoin(self.tmpdir, default)) # gentoo bug 355283 libdir = self.env.get("ABI") if libdir is not None: libdir = self.env.get("LIBDIR_%s" % (libdir, )) if libdir is not None: libdir = self.env.get(libdir) if libdir is None: libdir = "lib" path.insert(0, "/usr/%s/%s/bin" % (libdir, s.lower())) else: for y in ("_PATH", "_DIR"): if s + y in self.env: del self.env[s + y] path = [piece for piece in path if piece] self.env["PATH"] = os.pathsep.join(path) self.env["A"] = ' '.join(set(x.filename for x in pkg.fetchables)) if self.eapi.options.has_AA: pkg = getattr(self.pkg, '_raw_pkg', self.pkg) self.env["AA"] = ' '.join( set(x.filename for x in iflatten_instance( pkg.fetchables, fetch.fetchable))) if self.eapi.options.has_KV: ret = spawn_get_output(['uname', '-r']) if ret[0] == 0: self.env["KV"] = ret[1][0].strip() if self.eapi.options.has_merge_type: self.env["MERGE_TYPE"] = "source" if self.setup_is_for_src: self.init_distfiles_env()
def bashrcs(self): files = sorted_scan(pjoin(self.config_dir, 'bashrc'), follow_symlinks=True) return tuple(local_source(x) for x in files)
def _cmd_implementation_sanity_check(self, domain, observer): """Various ebuild sanity checks (REQUIRED_USE, pkg_pretend).""" pkg = self.pkg eapi = pkg.eapi # perform REQUIRED_USE checks if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): observer.info( textwrap.dedent(""" REQUIRED_USE requirement wasn't met Failed to match: {} from: {} for USE: {} pkg: {} """.format(node, pkg.required_use, " ".join(use), pkg.cpvstr))) return False # return if running pkg_pretend is not required if 'pretend' not in pkg.mandatory_phases: return True # run pkg_pretend phase commands = None if not pkg.built: commands = { "request_inherit": partial(inherit_handler, self._eclass_cache) } env = expected_ebuild_env(pkg) builddir = pjoin(domain.tmpdir, env["CATEGORY"], env["PF"]) pkg_tmpdir = normpath(pjoin(builddir, "temp")) ensure_dirs(pkg_tmpdir, mode=0770, gid=portage_gid, minimal=True) env["ROOT"] = domain.root env["T"] = pkg_tmpdir # TODO: make colored output easier to achieve from observers msg = [ '>>> Running pkg_pretend for ', observer._output._out.fg('green'), pkg.cpvstr, observer._output._out.reset ] observer._output._out.write(*msg) try: start = time.time() ret = run_generic_phase(pkg, "pretend", env, userpriv=True, sandbox=True, extra_handlers=commands) logger.debug("pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: return False finally: shutil.rmtree(builddir) # try to wipe the cat dir; if not empty, ignore it try: os.rmdir(os.path.dirname(builddir)) except EnvironmentError as e: # POSIX specifies either ENOTEMPTY or EEXIST for non-empty dir # in particular, Solaris uses EEXIST in that case. # https://github.com/pkgcore/pkgcore/pull/181 if e.errno not in (errno.ENOTEMPTY, errno.EEXIST): raise
def test_trigger(self): if self.raw_kls().get_binary_path() is None: raise SkipTest( "can't verify regen behaviour due to install-info not being available") cur = os.environ.get("PATH", self) try: os.environ.pop("PATH", None) # shouldn't run if the binary is missing # although it should warn, and this code will explode when it does. self.engine.phase = 'post_merge' self.assertEqual(None, self.trigger(self.engine, {})) finally: if cur is not self: os.environ["PATH"] = cur # verify it runs when dir is missing. # doesn't create the file since no info files. self.reset_objects() self.assertFalse(self.run_trigger('pre_merge', [])) self.assertFalse(self.run_trigger('post_merge', [self.dir])) # add an info, and verify it generated. with open(pjoin(self.dir, 'foo.info'), 'w') as f: f.write(self.info_data) self.reset_objects() self.trigger.enable_regen = True self.assertFalse(self.run_trigger('pre_merge', [])) self.assertFalse(self.run_trigger('post_merge', [self.dir])) # verify it doesn't; mtime is fine self.reset_objects() self.trigger.enable_regen = True self.assertFalse(self.run_trigger('pre_merge', [])) self.assertFalse(self.run_trigger('post_merge', [])) # verify it handles quoting properly, and that it ignores # complaints about duplicates. self.reset_objects() self.trigger.enable_regen = True self.assertFalse(self.run_trigger('pre_merge', [])) with open(pjoin(self.dir, "blaidd drwg.info"), "w") as f: f.write(self.info_data) self.assertFalse(self.run_trigger('post_merge', [self.dir])) # verify it passes back failures. self.reset_objects() self.trigger.enable_regen = True self.assertFalse(self.run_trigger('pre_merge', [])) with open(pjoin(self.dir, "tiza grande.info"), "w") as f: f.write('\n'.join(self.info_data.splitlines()[:-1])) l = self.run_trigger('post_merge', [self.dir]) self.assertEqual(len(l), 1) self.assertIn('tiza grande.info', l[0]) # verify it holds off on info regen till after unmerge for replaces. self.reset_objects(mode=const.REPLACE_MODE) self.assertFalse(self.run_trigger('pre_merge', [])) self.assertFalse(self.run_trigger('post_merge', [])) self.assertFalse(self.run_trigger('pre_unmerge', [])) os.unlink(pjoin(self.dir, "tiza grande.info")) self.assertFalse(self.run_trigger('post_unmerge', [self.dir]))
data.append("%s='%s'" % (key, val)) else: data.append("%s=$'%s'" % (key, val.replace("'", "\\'"))) return 'export %s' % (' '.join(data), ) def send_env(self, env_dict, async=False, tmpdir=None): """ transfer the ebuild's desired env (env_dict) to the running daemon :type env_dict: mapping with string keys and values. :param env_dict: the bash env. """ data = self._generate_env_str(env_dict) old_umask = os.umask(0002) if tmpdir: path = pjoin(tmpdir, 'ebd-env-transfer') fileutils.write_file(path, 'wb', data) self.write("start_receiving_env file %s\n" % (path, ), append_newline=False) else: self.write("start_receiving_env bytes %i\n%s" % (len(data), data), append_newline=False) os.umask(old_umask) return self.expect("env_received", async=async, flush=True) def set_logfile(self, logfile=''): """ Set the logfile (location to log to). Relevant only when the daemon is sandbox'd,
def _get_metadata_xml(self, category, package): return repo_objs.LocalMetadataXml( pjoin(self.base, category, package, "metadata.xml"))
def make_obj(self, location="/tmp/foo", **kwds): target = kwds.pop("target", pjoin(location, "target")) kwds.setdefault("strict", False) return self.kls(location, target, **kwds)