def __init__(self, raw_repo, domain, domain_settings, fetcher=None): """ :param raw_repo: :obj:`_UnconfiguredTree` instance :param domain_settings: environment settings to bind :param fetcher: :obj:`pkgcore.fetch.base.fetcher` instance to use for getting access to fetchable files """ if "USE" not in domain_settings: raise errors.InitializationError( "%s requires the following settings: 'USE', not supplied" % ( self.__class__,)) elif 'CHOST' not in domain_settings: raise errors.InitializationError( "%s requires the following settings: 'CHOST', not supplied" % ( self.__class__,)) chost = domain_settings['CHOST'] scope_update = {'chost': chost} scope_update.update((x, domain_settings.get(x.upper(), chost)) for x in ('cbuild', 'ctarget')) scope_update['operations_callback'] = self._generate_pkg_operations self.config_wrappables['iuse_effective'] = partial( self._generate_iuse_effective, domain.profile) configured.tree.__init__(self, raw_repo, self.config_wrappables, pkg_kls_injections=scope_update) self._get_pkg_use = domain.get_package_use_unconfigured self._get_pkg_use_for_building = domain.get_package_use_buildable self.domain_settings = domain_settings self.fetcher_override = fetcher self._delayed_iuse = partial(make_kls(InvertedContains), InvertedContains)
def test_size_verification_first(self): self.write_data() chksum_data = dict(chksums.iteritems()) l = [] def f(chf, fp): l.append(chf) return chksum_data[chf] subhandlers = { "size": partial(f, 'size'), known_chksum: partial(f, known_chksum) } # exact size verification self.fetcher._verify(self.fp, self.obj, handlers=subhandlers, all_chksums=False) self.assertEqual(['size', known_chksum], l) for x in (-100, 100): while l: l.pop(-1) chksum_data["size"] = chksums["size"] + x self.assertFailure(self.fetcher._verify, self.fp, self.obj, handlers=subhandlers, all_chksums=False, resumable=x < 0) self.assertEqual(['size'], l)
def test_subcommand(self): class SubParser(commandline.OptionParser): def check_values(self, values, args): values, args = commandline.OptionParser.check_values( self, values, args) values.args = args values.progname = self.prog return values, () def submain(status, options, out, err, subs=('sub', )): self.assertEqual(options.args, ['subarg']) self.assertEqual(options.progname, 'fo %s' % (' '.join(subs), )) return status self.assertMain(0, '', '', {'sub': (SubParser, partial(submain, 0))}, ['sub', 'subarg'], script_name='fo') self.assertMain( 1, '', '', { 'sub': { 'sub2': (SubParser, partial(submain, 1, subs=('sub', 'sub2'))) } }, ['sub', 'sub2', 'subarg'], script_name='fo')
def __init__(self, dbs, per_repo_strategy, global_strategy=None, depset_reorder_strategy=None, process_built_depends=False, drop_cycles=False, debug=False, debug_handle=None): if debug_handle is None: debug_handle = sys.stdout self.debug_handler = debug_handle self._dprint = partial(dprint, debug_handle) if not isinstance(dbs, (list, tuple)): dbs = [dbs] if global_strategy is None: global_strategy = self.default_global_strategy if depset_reorder_strategy is None: depset_reorder_strategy = self.default_depset_reorder_strategy self.depset_reorder = depset_reorder_strategy self.per_repo_strategy = per_repo_strategy self.total_ordering_strategy = global_strategy self.all_raw_dbs = [ misc.caching_repo(x, self.per_repo_strategy) for x in dbs ] self.all_dbs = global_strategy(self.all_raw_dbs) self.default_dbs = self.all_dbs self.state = state.plan_state() vdb_state_filter_restrict = MutableContainmentRestriction( self.state.vdb_filter) self.livefs_dbs = multiplex.tree(*[ visibility.filterTree(x, vdb_state_filter_restrict) for x in self.all_raw_dbs if x.livefs ]) self.insoluble = set() self.vdb_preloaded = False self._ensure_livefs_is_loaded = \ self._ensure_livefs_is_loaded_nonpreloaded self.drop_cycles = drop_cycles self.process_built_depends = process_built_depends self._debugging = debug if debug: self._rec_add_atom = partial(self._stack_debugging_rec_add_atom, self._rec_add_atom) self._debugging_depth = 0 self._debugging_drop_cycles = False
def __init__(self, data, frozen=False, livefs=False): self.installed = [] self.replaced = [] self.uninstalled = [] util.SimpleTree.__init__(self, data, pkg_klass=partial(fake_pkg, self)) self.livefs = livefs self.frozen = frozen
def __init__(self, **kwargs): kwargs.setdefault('observer', None) self._triggers = [] for k, v in kwargs.iteritems(): if callable(v): v = partial(v, self) setattr(self, k, v)
def _cmd_implementation_sanity_check(self, domain): pkg = self.pkg eapi = pkg.eapi_obj if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): print "REQUIRED_USE requirement weren't met\nFailed to match: %s\nfrom: %s\nfor USE: %s\npkg: %s" % \ (node, pkg.required_use, " ".join(use), pkg) return False if 'pretend' not in pkg.mandatory_phases: return True commands = {"request_inherit": partial(inherit_handler, self._eclass_cache)} env = expected_ebuild_env(pkg) env["ROOT"] = domain.root try: logger.debug("running ebuild pkg_pretend sanity check for %s", pkg) start = time.time() ret = run_generic_phase(pkg, "pretend", env, True, True, False, extra_handlers=commands) logger.debug("pkg_pretend sanity check for %s took %2.2f seconds", pkg, time.time() - start) return ret except format.GenericBuildError, e: logger.error("pkg_pretend sanity check for %s failed with exception %r" % (pkg, e)) return False
def archive_to_fsobj(src_tar): psep = os.path.sep for member in src_tar: d = { "uid":member.uid, "gid":member.gid, "mtime":member.mtime, "mode":member.mode} location = psep + member.name.strip(psep) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg(): d["data"] = invokable_data_source.wrap_function(partial( src_tar.extractfile, member.name), returns_text=False, returns_handle=True) # suppress hardlinks until the rest of pkgcore is updated for it. d["dev"] = None d["inode"] = None yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = long(member.major) d["minor"] = long(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def merge_contents(cset, offset=None, callback=None): """ merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs :param cset: :class:`pkgcore.fs.contents.contentsSet` instance :param offset: if not None, offset to prefix all locations with. Think of it as target dir. :param callback: callable to report each entry being merged; given a single arg, the fs object being merged. :raise EnvironmentError: Thrown for permission failures. """ if callback is None: callback = lambda obj:None ensure_perms = get_plugin("fs_ops.ensure_perms") copyfile = get_plugin("fs_ops.copyfile") mkdir = get_plugin("fs_ops.mkdir") if not isinstance(cset, contents.contentsSet): raise TypeError("cset must be a contentsSet, got %r" % (cset,)) if offset is not None: if os.path.exists(offset): if not os.path.isdir(offset): raise TypeError("offset must be a dir, or not exist: %s" % offset) else: mkdir(fs.fsDir(offset, strict=False)) iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep)) else: iterate = iter d = list(iterate(cset.iterdirs())) d.sort() for x in d: callback(x) try: # we pass in the stat ourselves, using stat instead of # lstat gen_obj uses internally; this is the equivalent of # "deference that link" obj = gen_obj(x.location, stat=os.stat(x.location)) if not fs.isdir(obj): raise Exception( "%s exists and needs to be a dir, but is a %s" % (x.location, obj)) ensure_perms(x, obj) except OSError, oe: if oe.errno != errno.ENOENT: raise try: # we do this form to catch dangling symlinks mkdir(x) except OSError, oe: if oe.errno != errno.EEXIST: raise os.unlink(x.location) mkdir(x) ensure_perms(x)
def test_normal(self): self.write_data() self.assertEqual(self.fetcher._verify(self.fp, self.obj), None) self.write_data(data[:-1]) self.assertFailure(self.fetcher._verify, self.fp, self.obj, resumable=True) # verify it returns -2 for missing file paths. os.unlink(self.fp) self.assertFailure(self.fetcher._verify, self.fp, self.obj, kls=errors.MissingDistfile, resumable=True) self.write_data(data + "foon") self.assertFailure(self.fetcher._verify, self.fp, self.obj, resumable=False) # verify they're ran one, and only once l = [] def f(chf, fp): l.append(chf) return chksums[chf] alt_handlers = {chf: partial(f, chf) for chf in chksums} self.assertEqual( self.fetcher._verify(self.fp, self.obj, handlers=alt_handlers), None) self.assertEqual(sorted(l), sorted(alt_handlers))
def _run_depend_like_phase(self, command, package_inst, eclass_cache, extra_commands={}): self._ensure_metadata_paths(const.HOST_NONROOT_PATHS) e = expected_ebuild_env(package_inst, depends=True) data = self._generate_env_str(e) self.write("%s %i\n%s" % (command, len(data), data), append_newline=False) updates = None if self._eclass_caching: updates = set() commands = extra_commands.copy() commands["request_inherit"] = partial(inherit_handler, eclass_cache, updates=updates) val = self.generic_handler(additional_commands=commands) if not val: logger.error("returned val from %s was '%s'", command, str(val)) raise Exception(val) if updates: self.preload_eclasses(eclass_cache, limited_to=updates, async=True)
def archive_to_fsobj(src_tar): psep = os.path.sep for member in src_tar: d = { "uid": member.uid, "gid": member.gid, "mtime": member.mtime, "mode": member.mode } location = psep + member.name.strip(psep) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg(): d["data"] = invokable_data_source.wrap_function( partial(src_tar.extractfile, member.name), returns_text=False, returns_handle=True) # suppress hardlinks until the rest of pkgcore is updated for it. d["dev"] = None d["inode"] = None yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = long(member.major) d["minor"] = long(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def make_keywords_filter(self, arch, default_keys, accept_keywords, profile_keywords, incremental=False): """Generates a restrict that matches iff the keywords are allowed.""" if not accept_keywords and not profile_keywords: return packages.PackageRestriction( "keywords", values.ContainmentMatch(*default_keys)) if "~" + arch.lstrip("~") not in default_keys: # stable; thus empty entries == ~arch unstable = "~" + arch def f(r, v): if not v: return r, unstable return r, v data = collapsed_restrict_to_data( ((packages.AlwaysTrue, default_keys),), (f(*i) for i in accept_keywords)) else: if incremental: f = collapsed_restrict_to_data else: f = non_incremental_collapsed_restrict_to_data data = f(((packages.AlwaysTrue, default_keys),), accept_keywords) if incremental: raise NotImplementedError(self.incremental_apply_keywords_filter) #f = self.incremental_apply_keywords_filter else: f = self.apply_keywords_filter return delegate(partial(f, data, profile_keywords))
def __new__(cls, name, bases, d): if '__del__' in d: d['__finalizer__'] = d.pop("__del__") elif not '__finalizer__' in d and not \ any(hasattr(parent, "__finalizer__") for parent in bases): raise TypeError( "cls %s doesn't have either __del__ nor a __finalizer__" % (name, )) if not '__disable_finalization__' in d and not \ any(hasattr(parent, "__disable_finalization__") for parent in bases): # install tracking d['__disable_finalization__'] = __disable_finalization__ d['__enable_finalization__'] = __enable_finalization__ # install tracking bits. we do this per class- this is intended to avoid any # potential stupid subclasses wiping a parents tracking. d['__finalizer_weakrefs__'] = mappings.defaultdict(dict) new_cls = super(WeakRefFinalizer, cls).__new__(cls, name, bases, d) new_cls.__proxy_class__ = partial(make_kls(new_cls, WeakRefProxy), cls, lambda x: x) new_cls.__proxy_class__.__name__ = name cls.__known_classes__[new_cls] = True return new_cls
class AlwaysBoolTest(TestRestriction): bool_kls = partial(restriction.AlwaysBool, 'foo') def test_true(self): true_r = self.bool_kls(True) false_r = self.bool_kls(False) self.assertMatch(true_r, false_r) self.assertForceTrue(true_r, false_r) self.assertNotForceFalse(true_r, false_r) self.assertNotMatch(false_r, true_r) self.assertNotForceTrue(false_r, true_r) self.assertForceFalse(false_r, true_r) self.assertEqual(str(true_r), "always 'True'") self.assertEqual(str(false_r), "always 'False'") self.assertNotEqual(hash(true_r), hash(false_r)) self.assertEqual(hash(true_r), hash(self.bool_kls(True))) self.assertEqual(hash(false_r), hash(self.bool_kls(False))) self.assertEqual(true_r, self.bool_kls(True)) self.assertEqual(false_r, self.bool_kls(False)) self.assertNotEqual(true_r, false_r)
def __init__(self, dbs, per_repo_strategy, global_strategy=None, depset_reorder_strategy=None, process_built_depends=False, drop_cycles=False, debug=False, debug_handle=None): if debug_handle is None: debug_handle = sys.stdout self.debug_handler = debug_handle self._dprint = partial(dprint, debug_handle) if not isinstance(dbs, (list, tuple)): dbs = [dbs] if global_strategy is None: global_strategy = self.default_global_strategy if depset_reorder_strategy is None: depset_reorder_strategy = self.default_depset_reorder_strategy self.depset_reorder = depset_reorder_strategy self.per_repo_strategy = per_repo_strategy self.total_ordering_strategy = global_strategy self.all_raw_dbs = [misc.caching_repo(x, self.per_repo_strategy) for x in dbs] self.all_dbs = global_strategy(self, self.all_raw_dbs) self.default_dbs = self.all_dbs self.state = state.plan_state() vdb_state_filter_restrict = MutableContainmentRestriction(self.state.vdb_filter) self.livefs_dbs = multiplex.tree( *[visibility.filterTree(x, vdb_state_filter_restrict) for x in self.all_raw_dbs if x.livefs]) self.insoluble = set() self.vdb_preloaded = False self._ensure_livefs_is_loaded = \ self._ensure_livefs_is_loaded_nonpreloaded self.drop_cycles = drop_cycles self.process_built_depends = process_built_depends self._debugging = debug if debug: self._rec_add_atom = partial(self._stack_debugging_rec_add_atom, self._rec_add_atom) self._debugging_depth = 0 self._debugging_drop_cycles = False
def _cmd_implementation_sanity_check(self, domain): pkg = self.pkg eapi = pkg.eapi_obj if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): print( textwrap.dedent(""" REQUIRED_USE requirement wasn't met Failed to match: {} from: {} for USE: {} pkg: {} """.format(node, pkg.required_use, " ".join(use), pkg.cpvstr))) return False if 'pretend' not in pkg.mandatory_phases: return True commands = None if not pkg.built: commands = { "request_inherit": partial(inherit_handler, self._eclass_cache) } env = expected_ebuild_env(pkg) tmpdir = normpath(domain._get_tempspace()) builddir = pjoin(tmpdir, env["CATEGORY"], env["PF"]) pkg_tmpdir = normpath(pjoin(builddir, "temp")) ensure_dirs(pkg_tmpdir, mode=0770, gid=portage_gid, minimal=True) env["ROOT"] = domain.root env["T"] = pkg_tmpdir try: logger.debug("running ebuild pkg_pretend sanity check for %s", pkg.cpvstr) start = time.time() ret = run_generic_phase(pkg, "pretend", env, userpriv=True, sandbox=True, fakeroot=False, extra_handlers=commands) logger.debug("pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: logger.error( "pkg_pretend sanity check for %s failed with exception %r" % (pkg.cpvstr, e)) return False finally: shutil.rmtree(builddir) # try to wipe the cat dir; if not empty, ignore it try: os.rmdir(os.path.dirname(builddir)) except EnvironmentError as e: if e.errno != errno.ENOTEMPTY: raise
def itermatch(self, restrict, **kwds): kwds.setdefault("force", True) o = kwds.get("pkg_klass_override") if o is not None: kwds["pkg_klass_override"] = partial(self.package_class, o) else: kwds["pkg_klass_override"] = self.package_class return (x for x in self.raw_repo.itermatch(restrict, **kwds) if x.is_supported)
def __init__(self, **kwargs): self._called = [] if isinstance(kwargs.get('_hooks', False), basestring): kwargs['_hooks'] = (kwargs['_hooks'],) for k, v in kwargs.iteritems(): if callable(v): v = partial(v, self) setattr(self, k, v)
class Class(base.base): __slotting_intentionally_disabled__ = True _get_attr = { str(x): partial((lambda a, s: a), x) for x in xrange(10) } _get_attr["a"] = lambda s: "foo" __getattr__ = base.dynamic_getattr_dict
def _mask_filter(masks, negate=False): atoms = defaultdict(list) globs = [] for m in masks: if isinstance(m, _atom): atoms[m.key].append(m) else: globs.append(m) return delegate(partial(apply_mask_filter, globs, atoms), negate=negate)
def generate_collapsed_restriction(atoms, negate=False): d = {} for a in atoms: k = a.key if k not in d: d[k] = [a] else: d[k].append(a) return delegate(partial(_collapsed_restrict_match, d), negate=negate)
def generate_providers(self): rdep = AndRestriction(self.versioned_atom) func = partial(virtual_ebuild, self._parent, self, {"rdepends":rdep, "slot":"%s-%s" % (self.category, self.version)}) # re-enable license at some point. #, "license":self.license}) return conditionals.DepSet.parse( self.data.pop("PROVIDE", ""), virtual_ebuild, element_func=func, operators={"":boolean.AndRestriction})
def clone(self, unfreeze=False): obj = self.__class__() if self.frozen and not unfreeze: obj._dict = self._dict obj._global_settings = self._global_settings return obj obj._dict = defaultdict(partial(list, self._global_settings)) for key, values in self._dict.iteritems(): obj._dict[key].extend(values) obj._global_settings = list(self._global_settings) return obj
def parse_owns(value): "Value is a comma delimited set of paths to search contents for" # yes it would be easier to do this without using parserestrict- # we use defer to using it for the sake of a common parsing # exposed to the commandline however. # the problem here is we don't want to trigger fs* module loadup # unless needed- hence this function. parser = parserestrict.comma_separated_containment('contents', values_kls=contents_module.contentsSet, token_kls=partial(fs_module.fsBase, strict=False)) return parser(value)
def parse_owns(value): "Value is a comma delimited set of paths to search contents for" # yes it would be easier to do this without using parserestrict- # we use defer to using it for the sake of a common parsing # exposed to the commandline however. # the problem here is we don't want to trigger fs* module loadup # unless needed- hence this function. parser = parserestrict.comma_separated_containment( 'contents', values_kls=contents_module.contentsSet, token_kls=partial(fs_module.fsBase, strict=False)) return parser(value)
def test_subcommand(self): class SubParser(commandline.OptionParser): def check_values(self, values, args): values, args = commandline.OptionParser.check_values( self, values, args) values.args = args values.progname = self.prog return values, () def submain(status, options, out, err, subs=('sub',)): self.assertEqual(options.args, ['subarg']) self.assertEqual(options.progname, 'fo %s' % (' '.join(subs),)) return status self.assertMain( 0, '', '', {'sub': (SubParser, partial(submain,0))}, ['sub', 'subarg'], script_name='fo') self.assertMain( 1, '', '', {'sub': {'sub2': (SubParser, partial(submain, 1, subs=('sub', 'sub2')))}}, ['sub', 'sub2', 'subarg'], script_name='fo')
def __init__(self, mode, tempdir, hooks, csets, preserves, observer, offset=None, disable_plugins=False, parallelism=None): if observer is None: observer = observer_mod.repo_observer(observer_mod.null_output) self.observer = observer self.mode = mode if tempdir is not None: tempdir = normpath(tempdir) + '/' self.tempdir = tempdir if parallelism is None: parallelism = get_proc_count() self.parallelism = parallelism self.hooks = ImmutableDict((x, []) for x in hooks) self.preserve_csets = [] self.cset_sources = {} # instantiate these separately so their values are preserved self.preserved_csets = LazyValDict( self.preserve_csets, self._get_cset_source) for k, v in csets.iteritems(): if isinstance(v, basestring): v = getattr(self, v, v) if not callable(v): raise TypeError( "cset values must be either the string name of " "existing methods, or callables (got %s)" % v) if k in preserves: self.add_preserved_cset(k, v) else: self.add_cset(k, v) if offset is None: offset = "/" self.offset = offset if not disable_plugins: # merge in default triggers first. for trigger in get_plugins('triggers'): t = trigger() t.register(self) # merge in overrides for hook, triggers in hooks.iteritems(): for trigger in triggers: self.add_trigger(hook, trigger) self.regenerate_csets() for x in hooks: setattr(self, x, currying.partial(self.execute_hook, x))
def revdep_pkgs_finalize(sequence, namespace): if not sequence: return [] l = [] for atom_inst in sequence: for repo in namespace.repos: l.extend(repo.itermatch(atom_inst)) # have our pkgs; now build the restrict. any_restrict = values.AnyMatch(values.FunctionRestriction( partial(_revdep_pkgs_match, tuple(l)))) r = values.FlatteningRestriction(atom.atom, any_restrict) return list(packages.PackageRestriction(dep, r) for dep in ('depends', 'rdepends', 'post_rdepends'))
def generate_providers(self): rdep = AndRestriction(self.versioned_atom) func = partial(virtual_ebuild, self._parent, self, { "rdepends": rdep, "slot": "%s-%s" % (self.category, self.version) }) # re-enable license at some point. #, "license":self.license}) return conditionals.DepSet.parse(self.data.pop("PROVIDE", ""), virtual_ebuild, element_func=func, operators={"": boolean.AndRestriction})
def __iter__(self): matcher = partial(is_disjoint, self.eclasses) for atom in VersionedInstalled.__iter__(self): pkgs = self.portdir.match(atom) if not pkgs: # This thing is in the vdb but no longer in portdir # (or someone misconfigured us to use a bogus # portdir). Just ignore it. continue assert len(pkgs) == 1, "I do not know what I am doing: %r" % (pkgs,) pkg = pkgs[0] if matcher(pkg.data.get("_eclasses_", ())): yield atom
def test_size_verification_first(self): self.write_data() chksum_data = dict(chksums.iteritems()) l = [] def f(chf, fp): l.append(chf) return chksum_data[chf] subhandlers = {"size":partial(f, 'size'), known_chksum:partial(f, known_chksum)} # exact size verification self.fetcher._verify(self.fp, self.obj, handlers=subhandlers, all_chksums=False) self.assertEqual(['size', known_chksum], l) for x in (-100, 100): while l: l.pop(-1) chksum_data["size"] = chksums["size"] + x self.assertFailure(self.fetcher._verify, self.fp, self.obj, handlers=subhandlers, all_chksums=False, resumable=x < 0) self.assertEqual(['size'], l)
def test_get_metadata(self): ec = FakeEclassCache('/nonexistent/path') pkg = malleable_obj(_mtime_=100, cpvstr='dev-util/diffball-0.71', path='bollocks') class fake_cache(dict): readonly = False validate_result = False def validate_entry(self, *args): return self.validate_result cache1 = fake_cache({pkg.cpvstr: {'_mtime_':100, 'marker':1} }) cache2 = fake_cache({}) class explode_kls(AssertionError): pass def explode(name, *args, **kwargs): raise explode_kls("%s was called with %r and %r, " "shouldn't be invoked." % (name, args, kwargs)) pf = self.mkinst(cache=(cache2, cache1), eclasses=ec, _update_metadata=partial(explode, '_update_metadata')) cache1.validate_result = True self.assertEqual(pf._get_metadata(pkg), {'marker':1, '_mtime_':100}, reflective=False) self.assertEqual(cache1.keys(), [pkg.cpvstr]) self.assertFalse(cache2) # mtime was wiped, thus no longer is usable. # note also, that the caches are writable. cache1.validate_result = False self.assertRaises(explode_kls, pf._get_metadata, pkg) self.assertFalse(cache2) self.assertFalse(cache1) # Note that this is known crap eclass data; partially lazyness, partially # to validate the eclass validation is left to ec cache only. cache2.update({pkg.cpvstr: {'_mtime_':200, '_eclasses_':{'eclass1':(None, 100)}, 'marker':2} }) cache2.readonly = True self.assertRaises(explode_kls, pf._get_metadata, pkg) self.assertEqual(cache2.keys(), [pkg.cpvstr]) # keep in mind the backend assumes it gets its own copy of the data. # thus, modifying (popping _mtime_) _is_ valid self.assertEqual(cache2[pkg.cpvstr], {'_eclasses_':{'eclass1':(None, 100)}, 'marker':2, '_mtime_':200})
def test_get_metadata(self): ec = FakeEclassCache('/nonexistent/path') pkg = malleable_obj(_mtime_=100, cpvstr='dev-util/diffball-0.71', path='bollocks') class fake_cache(dict): readonly = False validate_result = False def validate_entry(self, *args): return self.validate_result cache1 = fake_cache({pkg.cpvstr: {'_mtime_':100, 'marker':1} }) cache2 = fake_cache({}) class explode_kls(AssertionError): pass def explode(name, *args, **kwargs): raise explode_kls("%s was called with %r and %r, " "shouldn't be invoked." % (name, args, kwargs)) pf = self.mkinst(cache=(cache2, cache1), eclasses=ec, _update_metadata=partial(explode, '_update_metadata')) cache1.validate_result = True self.assertEqual(pf._get_metadata(pkg), {'marker':1, '_mtime_':100}, reflective=False) self.assertEqual(cache1.keys(), [pkg.cpvstr]) self.assertFalse(cache2) # mtime was wiped, thus no longer is usable. # note also, that the caches are writable. cache1.validate_result = False self.assertRaises(explode_kls, pf._get_metadata, pkg) self.assertFalse(cache2) self.assertFalse(cache1) # Note that this is known crap eclass data; partially lazyness, partially # to validate the eclass validation is left to ec cache only. cache2.update({pkg.cpvstr: {'_mtime_':200, '_eclasses_':{'eclass1':(None, 100)}, 'marker':2} }) cache2.readonly = True self.assertRaises(explode_kls, pf._get_metadata, pkg) self.assertEqual(cache2.keys(), [pkg.cpvstr]) # keep in mind the backend assumes it gets it's own copy of the data. # thus, modifying (popping _mtime_) _is_ valid self.assertEqual(cache2[pkg.cpvstr], {'_eclasses_':{'eclass1':(None, 100)}, 'marker':2, '_mtime_':200})
def __init__(self, raw_repo, domain, domain_settings, fetcher=None): """ :param raw_repo: :obj:`_UnconfiguredTree` instance :param domain_settings: environment settings to bind :param fetcher: :obj:`pkgcore.fetch.base.fetcher` instance to use for getting access to fetchable files """ if "USE" not in domain_settings: raise errors.InitializationError( "%s requires the following settings: 'USE', not supplied" % (self.__class__, )) elif 'CHOST' not in domain_settings: raise errors.InitializationError( "%s requires the following settings: 'CHOST', not supplied" % (self.__class__, )) chost = domain_settings['CHOST'] scope_update = {'chost': chost} scope_update.update((x, domain_settings.get(x.upper(), chost)) for x in ('cbuild', 'ctarget')) scope_update.update((x, domain_settings[x.upper()]) for x in ('cflags', 'cxxflags', 'ldflags')) scope_update['operations_callback'] = self._generate_pkg_operations self.config_wrappables['iuse_effective'] = partial( self._generate_iuse_effective, domain.profile) configured.tree.__init__(self, raw_repo, self.config_wrappables, pkg_kls_injections=scope_update) self._get_pkg_use = domain.get_package_use_unconfigured self._get_pkg_use_for_building = domain.get_package_use_buildable self.domain_settings = domain_settings self.fetcher_override = fetcher self._delayed_iuse = partial(make_kls(InvertedContains), InvertedContains)
def provides_repo(self): d = {} for pkg in self._collapse_generic("pkg_provided"): d.setdefault(pkg.category, {}).setdefault(pkg.package, []).append(pkg.fullver) intermediate_parent = PkgProvidedParent() obj = util.SimpleTree(d, pkg_klass=currying.partial(PkgProvided, intermediate_parent), livefs=True, frozen=True) intermediate_parent._parent_repo = obj if not d: obj.match = obj.itermatch = _empty_provides_iterable obj.has_match = _empty_provides_has_match return obj
def wrap_function(cls, invokable, returns_text=True, returns_handle=False, encoding_hint=None): """ Helper function to automatically convert a function that returns text or bytes into appropriate callable :param invokable: a callable that returns either text, or bytes, taking no args :param returns_text: True if the data returned is text/basestring, False if Not :param returns_handle: True if the object returned is a handle, False if not. Note that returns_text still has meaning here- returns_text indicates what sort of data the handle returns from read invocations. :param encoding_hint: the preferred encoding to use for encoding :return: invokable_data_source instance """ return cls(partial(cls._simple_wrapper, invokable, encoding_hint, returns_text, returns_handle))
def revdep_pkgs_finalize(sequence, namespace): if not sequence: return [] l = [] for atom_inst in sequence: for repo in namespace.repos: l.extend(repo.itermatch(atom_inst)) # have our pkgs; now build the restrict. any_restrict = values.AnyMatch( values.FunctionRestriction(partial(_revdep_pkgs_match, tuple(l)))) r = values.FlatteningRestriction(atom.atom, any_restrict) return list( packages.PackageRestriction(dep, r) for dep in ('depends', 'rdepends', 'post_rdepends'))
def _cmd_implementation_sanity_check(self, domain): pkg = self.pkg eapi = pkg.eapi_obj if eapi.options.has_required_use: use = pkg.use for node in pkg.required_use: if not node.match(use): print(textwrap.dedent( """ REQUIRED_USE requirement wasn't met Failed to match: {} from: {} for USE: {} pkg: {} """.format(node, pkg.required_use, " ".join(use), pkg.cpvstr) )) return False if 'pretend' not in pkg.mandatory_phases: return True commands = None if not pkg.built: commands = {"request_inherit": partial(inherit_handler, self._eclass_cache)} env = expected_ebuild_env(pkg) tmpdir = normpath(domain._get_tempspace()) builddir = pjoin(tmpdir, env["CATEGORY"], env["PF"]) pkg_tmpdir = normpath(pjoin(builddir, "temp")) ensure_dirs(pkg_tmpdir, mode=0770, gid=portage_gid, minimal=True) env["ROOT"] = domain.root env["T"] = pkg_tmpdir try: logger.debug("running ebuild pkg_pretend sanity check for %s", pkg.cpvstr) start = time.time() ret = run_generic_phase(pkg, "pretend", env, userpriv=True, sandbox=True, fakeroot=False, extra_handlers=commands) logger.debug("pkg_pretend sanity check for %s took %2.2f seconds", pkg.cpvstr, time.time() - start) return ret except format.GenericBuildError as e: logger.error("pkg_pretend sanity check for %s failed with exception %r" % (pkg.cpvstr, e)) return False finally: shutil.rmtree(builddir) # try to wipe the cat dir; if not empty, ignore it try: os.rmdir(os.path.dirname(builddir)) except EnvironmentError as e: if e.errno != errno.ENOTEMPTY: raise
def __init__(self, *args, **kwargs): self.priority = int(kwargs.pop("priority", self.default_priority)) self.config_type = kwargs.pop("config_type", None) if self.config_type is None or not isinstance(self.config_type, str): raise ValueError("config_type must specified, and be a string") if kwargs.pop("get_default", False): kwargs["default"] = DelayedValue(currying.partial(self.store_default, self.config_type, option_string=kwargs.get('option_strings', [None])[0]), self.priority) self.store_name = kwargs.pop("store_name", False) self.writable = kwargs.pop("writable", None) self.target = argparse._StoreAction(*args, **kwargs) super(StoreConfigObject, self).__init__(*args, **kwargs)
def setup(self, setup_phase_override=None): self.setup_logging() additional_commands = {} phase_name = "setup-binpkg" if self.setup_is_for_src: phase_name = "setup" if setup_phase_override is not None: phase_name = setup_phase_override if self.setup_is_for_src: additional_commands["request_inherit"] = partial(inherit_handler, self.eclass_cache) return self._generic_phase(phase_name, False, True, False, extra_handlers=additional_commands)
def generic_check(self, attr): # should puke a metadata error for empty license chk = self.mk_check() mk_pkg = partial(self.mk_pkg, attr) self.assertNoReport(chk, mk_pkg()) self.assertNoReport(chk, mk_pkg("|| ( dev-util/foo ) dev-foo/bugger ")) r = self.assertIsInstance( self.assertReport(self.mk_check(), mk_pkg("|| (")), metadata_checks.MetadataError) self.assertEqual(r.attr, attr) if 'depend' not in attr: return self.assertNoReport(chk, mk_pkg("!dev-util/blah")) r = self.assertIsInstance(self.assertReport(self.mk_check(), mk_pkg("!dev-util/diffball")), metadata_checks.MetadataError) self.assertIn(r.msg, "blocks itself")
def provides_repo(self): d = {} for pkg in self._collapse_generic("pkg_provided"): d.setdefault(pkg.category, {}).setdefault(pkg.package, []).append(pkg.fullver) intermediate_parent = PkgProvidedParent() obj = util.SimpleTree(d, pkg_klass=currying.partial( PkgProvided, intermediate_parent), livefs=True, frozen=True) intermediate_parent._parent_repo = obj if not d: obj.match = obj.itermatch = _empty_provides_iterable obj.has_match = _empty_provides_has_match return obj
def generate_fetchables(self, allow_missing_checksums=False): chksums_can_be_missing = allow_missing_checksums or \ bool(getattr(self.repo, '_allow_missing_chksums', False)) chksums_can_be_missing, chksums = self.repo._get_digests(self, allow_missing=chksums_can_be_missing) mirrors = getattr(self._parent, "mirrors", {}) default_mirrors = getattr(self._parent, "default_mirrors", None) common = {} func = partial(create_fetchable_from_uri, self, chksums, chksums_can_be_missing, mirrors, default_mirrors, common) d = conditionals.DepSet.parse( self.data.pop("SRC_URI", ""), fetchable, operators={}, element_func=func, allow_src_uri_file_renames=self.eapi_obj.options.src_uri_renames) for v in common.itervalues(): v.uri.finalize() return d
def demandload(scope, *imports): """Import modules into scope when each is first used. scope should be the value of C{globals()} in the module calling this function. (using C{locals()} may work but is not recommended since mutating that is not safe). Other args are strings listing module names. names are handled like this:: foo import foo foo@bar import foo as bar foo:bar from foo import bar foo:bar,quux from foo import bar, quux foo.bar:quux from foo.bar import quux foo:baz@quux from foo import baz as quux """ for source, target in parse_imports(imports): scope[target] = Placeholder(scope, target, partial(load_any, source))
def generate_fetchables(self, allow_missing_checksums=False): chksums_can_be_missing = allow_missing_checksums or \ bool(getattr(self.repo, '_allow_missing_chksums', False)) chksums_can_be_missing, chksums = self.repo._get_digests( self, allow_missing=chksums_can_be_missing) mirrors = getattr(self._parent, "mirrors", {}) default_mirrors = getattr(self._parent, "default_mirrors", None) common = {} func = partial(create_fetchable_from_uri, self, chksums, chksums_can_be_missing, mirrors, default_mirrors, common) d = conditionals.DepSet.parse( self.data.pop("SRC_URI", ""), fetchable, operators={}, element_func=func, allow_src_uri_file_renames=self.eapi_obj.options.src_uri_renames) for v in common.itervalues(): v.uri.finalize() return d
class Test_native_generic_equality(TestCase): op_prefix = "native_" kls = currying.partial(klass.generic_equality, ne=klass.native_generic_attr_ne, eq=klass.native_generic_attr_eq) def test_it(self): class c(object): __attr_comparison__ = ("foo", "bar") __metaclass__ = self.kls def __init__(self, foo, bar): self.foo, self.bar = foo, bar def __repr__(self): return "<c: foo=%r, bar=%r, %i>" % (getattr( self, 'foo', 'unset'), getattr(self, 'bar', 'unset'), id(self)) self.assertEqual(c(1, 2), c(1, 2)) c1 = c(1, 3) self.assertEqual(c1, c1) del c1 self.assertNotEqual(c(2, 1), c(1, 2)) c1 = c(1, 2) del c1.foo c2 = c(1, 2) self.assertNotEqual(c1, c2) del c2.foo self.assertEqual(c1, c2) def test_call(self): def mk_class(meta): class c(object): __metaclass__ = meta return c self.assertRaises(TypeError, mk_class)
def _run_depend_like_phase(self, command, package_inst, eclass_cache, extra_commands={}): self._ensure_metadata_paths(const.HOST_NONROOT_PATHS) e = expected_ebuild_env(package_inst, depends=True) data = self._generate_env_str(e) self.write("%s %i\n%s" % (command, len(data), data), append_newline=False) updates = None if self._eclass_caching: updates = set() commands = extra_commands.copy() commands["request_inherit"] = partial(inherit_handler, eclass_cache, updates=updates) val = self.generic_handler(additional_commands=commands) if not val: logger.error("returned val from %s was '%s'" % (command, str(val))) raise Exception(val) if updates: self.preload_eclasses(eclass_cache, limited_to=updates, async=True)
def unmerge_contents(cset, offset=None, callback=None): """ unmerge a :obj:`pkgcore.fs.contents.contentsSet` instance to the livefs :param cset: :obj:`pkgcore.fs.contents.contentsSet` instance :param offset: if not None, offset to prefix all locations with. Think of it as target dir. :param callback: callable to report each entry being unmerged :return: True, or an exception is thrown on failure (OSError, although see default_copyfile for specifics). :raise EnvironmentError: see :func:`default_copyfile` and :func:`default_mkdir` """ if callback is None: callback = lambda obj: None iterate = iter if offset is not None: iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep)) for x in iterate(cset.iterdirs(invert=True)): callback(x) unlink_if_exists(x.location) # this is a fair sight faster then using sorted/reversed l = list(iterate(cset.iterdirs())) l.sort(reverse=True) for x in l: try: os.rmdir(x.location) except OSError as e: if not e.errno in (errno.ENOTEMPTY, errno.ENOENT, errno.ENOTDIR, errno.EBUSY, errno.EEXIST): raise else: callback(x) return True