def _manifest(options, out, err): failed = options.repo.operations.manifest( domain=options.domain, restriction=options.restriction, observer=observer_mod.formatter_output(out), mirrors=options.mirrors, force=options.force, distdir=options.distdir) return int(any(failed))
def digest_main(options, out, err): repo = options.repo failed = repo.operations.digests( domain=options.domain, restriction=options.restriction, observer=observer_mod.formatter_output(out), mirrors=options.mirrors, force=options.force) return int(any(failed))
def main(options, out, err): pkgs = options.domain.all_repos.match(options.atom) if not pkgs: err.write('got no matches for %s\n' % (options.atom, )) return 1 if len(pkgs) > 1: err.write('got multiple matches for %s:' % (options.atom, )) if len(set((pkg.slot, pkg.repo) for pkg in pkgs)) != 1: for pkg in sorted(pkgs): err.write("repo %r, slot %r, %s" % ( getattr(pkg.repo, 'repo_id', 'unknown'), pkg.slot, pkg.cpvstr, ), prefix=" ") err.write() err.write( "please refine your restriction to match only one slot/repo pair\n" ) return 1 pkgs = [max(pkgs)] err.write("choosing %r, slot %r, %s" % (getattr( pkgs[0].repo, 'repo_id', 'unknown'), pkgs[0].slot, pkgs[0].cpvstr), prefix=' ') kwds = {} build_obs = observer.build_observer(observer.formatter_output(out), not options.debug) phases = [x for x in options.phase if x != 'clean'] clean = (len(phases) != len(options.phase)) if options.no_auto: kwds["ignore_deps"] = True if "setup" in phases: phases.insert(0, "fetch") # by default turn off startup cleans; we clean by ourselves if # told to do so via an arg build = options.domain.build_pkg(pkgs[0], build_obs, clean=False, allow_fetching=True) if clean: build.cleanup(force=True) build._reload_state() phase_funcs = (getattr(build, x) for x in phases) for phase, f in izip(phases, phase_funcs): out.write() out.write('executing phase %s' % (phase, )) f(**kwds)
def unmerge(out, err, vdb, targets, options, formatter, world_set=None): """Unmerge tokens. hackish, should be rolled back into the resolver""" matches = set() unknown = set() for token, restriction in targets: # Catch restrictions matching across more than one category. # Multiple matches in the same category are acceptable. # The point is that matching across more than one category is # nearly always unintentional ("pmerge -C spork" without # realising there are sporks in more than one category), but # matching more than one cat/pkg is impossible without # explicit wildcards. installed = vdb.match(restriction) if not installed: unknown.add(token) categories = set(pkg.category for pkg in installed) if len(categories) > 1: raise parserestrict.ParseError( "%r is in multiple categories (%s)" % (token, ', '.join(sorted(set(pkg.key for pkg in installed))))) matches.update(installed) # fail out if no matches are found, otherwise just output a notification if unknown: if matches: err.write("Skipping unknown matches: %s\n" % ', '.join(map(repr, unknown))) else: raise Failure("no matches found: %s" % ', '.join(map(repr, unknown))) out.write(out.bold, 'The following packages are to be unmerged:') out.prefix = [out.bold, ' * ', out.reset] for pkg in matches: out.write(pkg.cpvstr) out.prefix = [] repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug) if options.pretend: return if (options.ask and not formatter.ask("Would you like to unmerge these packages?")): return return do_unmerge(options, out, err, vdb, matches, world_set, repo_obs)
def main(options, out, err): pkgs = options.domain.all_repos.match(options.atom) if not pkgs: err.write('got no matches for %s\n' % (options.atom,)) return 1 if len(pkgs) > 1: err.write('got multiple matches for %s:' % (options.atom,)) if len(set((pkg.slot, pkg.repo) for pkg in pkgs)) != 1: for pkg in sorted(pkgs): err.write("repo %r, slot %r, %s" % (getattr(pkg.repo, 'repo_id', 'unknown'), pkg.slot, pkg.cpvstr,), prefix=" ") err.write() err.write("please refine your restriction to match only one slot/repo pair\n"); return 1 pkgs = [max(pkgs)] err.write("choosing %r, slot %r, %s" % (getattr(pkgs[0].repo, 'repo_id', 'unknown'), pkgs[0].slot, pkgs[0].cpvstr), prefix=' ') kwds = {} build_obs = observer.build_observer(observer.formatter_output(out), not options.debug) phases = [x for x in options.phase if x != 'clean'] clean = (len(phases) != len(options.phase)) if options.no_auto: kwds["ignore_deps"] = True if "setup" in phases: phases.insert(0, "fetch") # by default turn off startup cleans; we clean by ourselves if # told to do so via an arg build = options.domain.build_pkg(pkgs[0], build_obs, clean=False, allow_fetching=True) if clean: build.cleanup(force=True) build._reload_state() phase_funcs = list(getattr(build, x) for x in phases) for phase, f in zip(phases, phase_funcs): out.write() out.write('executing phase %s' % (phase,)) f(**kwds)
def unmerge(out, err, vdb, restrictions, options, formatter, world_set=None): """Unmerge tokens. hackish, should be rolled back into the resolver""" all_matches = set() for restriction in restrictions: # Catch restrictions matching across more than one category. # Multiple matches in the same category are acceptable. # The point is that matching across more than one category is # nearly always unintentional ("pmerge -C spork" without # realising there are sporks in more than one category), but # matching more than one cat/pkg is impossible without # explicit wildcards. matches = vdb.match(restriction) if not matches: raise Failure('Nothing matches %s' % (restriction, )) categories = set(pkg.category for pkg in matches) if len(categories) > 1: raise parserestrict.ParseError( '%s is in multiple categories (%s)' % (restriction, ', '.join(set(pkg.key for pkg in matches)))) all_matches.update(matches) matches = sorted(all_matches) out.write(out.bold, 'The following packages are to be unmerged:') out.prefix = [out.bold, ' * ', out.reset] for match in matches: out.write(match.cpvstr) out.prefix = [] repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug) if options.pretend: return if (options.ask and not formatter.ask("Would you like to unmerge these packages?")): return return do_unmerge(options, out, err, vdb, matches, world_set, repo_obs)
def regen_main(options, out, err): """Regenerate a repository cache.""" ret = [] observer = observer_mod.formatter_output(out) for repo in iter_stable_unique(options.repos): if not repo.operations.supports("regen_cache"): out.write(f"repo {repo} doesn't support cache regeneration") continue elif not getattr(repo, 'cache', False) and not options.force: out.write(f"skipping repo {repo}: cache disabled") continue start_time = time.time() ret.append(repo.operations.regen_cache( threads=options.threads, observer=observer, force=options.force, eclass_caching=(not options.disable_eclass_caching))) end_time = time.time() if options.verbosity > 0: out.write( "finished %d nodes in %.2f seconds" % (len(repo), end_time - start_time)) if options.rsync: timestamp = pjoin(repo.location, "metadata", "timestamp.chk") try: with open(timestamp, "w") as f: f.write(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())) except IOError as e: err.write(f"Unable to update timestamp file {timestamp!r}: {e.strerror}") ret.append(os.EX_IOERR) if options.use_local_desc: ret.append(update_use_local_desc(repo, observer)) if options.pkg_desc_index: ret.append(update_pkg_desc_index(repo, observer)) return int(any(ret))
def unmerge(out, err, vdb, restrictions, options, formatter, world_set=None): """Unmerge tokens. hackish, should be rolled back into the resolver""" all_matches = set() for restriction in restrictions: # Catch restrictions matching across more than one category. # Multiple matches in the same category are acceptable. # The point is that matching across more than one category is # nearly always unintentional ("pmerge -C spork" without # realising there are sporks in more than one category), but # matching more than one cat/pkg is impossible without # explicit wildcards. matches = vdb.match(restriction) if not matches: raise Failure('Nothing matches %s' % (restriction,)) categories = set(pkg.category for pkg in matches) if len(categories) > 1: raise parserestrict.ParseError( '%s is in multiple categories (%s)' % ( restriction, ', '.join(set(pkg.key for pkg in matches)))) all_matches.update(matches) matches = sorted(all_matches) out.write(out.bold, 'The following packages are to be unmerged:') out.prefix = [out.bold, ' * ', out.reset] for match in matches: out.write(match.cpvstr) out.prefix = [] repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug) if options.pretend: return if (options.ask and not formatter.ask("Would you like to unmerge these packages?")): return return do_unmerge(options, out, err, vdb, matches, world_set, repo_obs)
def main(options, out, err): domain = options.domain kwds = {} phase_obs = observer.phase_observer(observer.formatter_output(out), not options.debug) phases = [x for x in options.phase if x != 'clean'] clean = (len(phases) != len(options.phase)) if options.no_auto: kwds["ignore_deps"] = True if "setup" in phases: phases.insert(0, "fetch") # by default turn off startup cleans; we clean by ourselves if # told to do so via an arg build = domain.build_pkg(options.pkg, phase_obs, clean=False, allow_fetching=True) if clean: build.cleanup(force=True) build._reload_state() phase_funcs = [(p, getattr(build, p, None)) for p in phases] unknown_phases = [p for p, func in phase_funcs if func is None] if unknown_phases: argparser.error( "unknown phase%s: %s" % (pluralism(unknown_phases), ', '.join(map(repr, unknown_phases)))) try: for phase, func in phase_funcs: out.write('executing phase %s' % (phase, )) func(**kwds) except format.errors as e: out.error("caught exception executing phase %s: %s" % (phase, e)) return 1
def main(options, out, err): pkg = options.pkg repos = None if os.path.isfile(pkg) and pkg.endswith('.ebuild'): ebuild_path = os.path.abspath(pkg) repo_path = os.path.abspath( os.path.join(pkg, os.pardir, os.pardir, os.pardir)) # find the ebuild's repo # TODO: iterating through the repos feels wrong, we could use a # multi-keyed dict with repo IDs and paths as keys with repo # objects as values (same thing we need for better portage-2 # profile support) for x in options.domain.repos: if getattr(x, 'repository_type', None) == 'source' and \ x.raw_repo.location == repo_path: repos = x break if repos is None: err.write('no configured repo contains: %s' % ebuild_path) return 1 ebuild_P = os.path.basename(os.path.splitext(ebuild_path)[0]) ebuild_category = ebuild_path.split(os.sep)[-3] pkg = atom.atom('=%s/%s' % (ebuild_category, ebuild_P)) else: try: pkg = atom.atom(pkg) repos = options.domain.all_repos except MalformedAtom: err.write('not a valid atom or ebuild: "%s"' % pkg) return 1 pkgs = repos.match(pkg) if not pkgs: err.write('got no matches for %s\n' % (pkg, )) return 1 if len(pkgs) > 1: err.write('got multiple matches for %s:' % (pkg, )) if len(set((pkg.slot, pkg.repo) for pkg in pkgs)) != 1: for pkg in sorted(pkgs): err.write("repo %r, slot %r, %s" % ( getattr(pkg.repo, 'repo_id', 'unknown'), pkg.slot, pkg.cpvstr, ), prefix=" ") err.write() err.write( "please refine your restriction to match only one slot/repo pair\n" ) return 1 pkgs = [max(pkgs)] err.write("choosing %r, slot %r, %s" % (getattr( pkgs[0].repo, 'repo_id', 'unknown'), pkgs[0].slot, pkgs[0].cpvstr), prefix=' ') kwds = {} build_obs = observer.build_observer(observer.formatter_output(out), not options.debug) phases = [x for x in options.phase if x != 'clean'] clean = (len(phases) != len(options.phase)) if options.no_auto: kwds["ignore_deps"] = True if "setup" in phases: phases.insert(0, "fetch") # by default turn off startup cleans; we clean by ourselves if # told to do so via an arg build = options.domain.build_pkg(pkgs[0], build_obs, clean=False, allow_fetching=True) if clean: build.cleanup(force=True) build._reload_state() phase_funcs = (getattr(build, x) for x in phases) for phase, f in izip(phases, phase_funcs): out.write('executing phase %s' % (phase, )) f(**kwds)
def main(options, out, err): config = options.config if options.debug: resolver.plan.limiters.add(None) domain = options.domain livefs_repos = domain.all_livefs_repos world_set = world_list = options.world if options.oneshot: world_set = None formatter = options.formatter( out=out, err=err, unstable_arch=domain.unstable_arch, domain_settings=domain.settings, use_expand=domain.profile.use_expand, use_expand_hidden=domain.profile.use_expand_hidden, pkg_get_use=domain.get_package_use_unconfigured, world_list=world_list, verbose=options.verbose, livefs_repos=livefs_repos, distdir=domain.fetcher.get_storage_path(), quiet_repo_display=options.quiet_repo_display) # This mode does not care about sets and packages so bypass all that. if options.unmerge: if not options.oneshot: if world_set is None: err.write( "Disable world updating via --oneshot, or fix your configuration" ) return 1 try: unmerge(out, err, livefs_repos, options.targets, options, formatter, world_set) except (parserestrict.ParseError, Failure) as e: out.error(str(e)) return 1 return source_repos = domain.source_repositories installed_repos = domain.installed_repositories if options.usepkgonly: source_repos = source_repos.change_repos( x for x in source_repos if getattr(x, 'repository_type', None) != 'source') elif options.usepkg: repo_types = [(getattr(x, 'repository_type', None) == 'built', x) for x in source_repos] source_repos = source_repos.change_repos( [x[1] for x in repo_types if x[0]] + [x[1] for x in repo_types if not x[0]]) elif options.source_only: source_repos = source_repos.change_repos( x for x in source_repos if getattr(x, 'repository_type', None) == 'source') atoms = [] for setname, pkgset in options.set: if pkgset is None: return 1 l = list(pkgset) if not l: out.write("skipping set %s: set is empty, nothing to update" % setname) else: atoms.extend(l) for token in options.targets: try: a = parse_atom(token, source_repos.combined, livefs_repos, return_none=True) except parserestrict.ParseError as e: out.error(str(e)) return 1 if a is None: if token in config.pkgset: out.error('No package matches %r, but there is a set with ' 'that name. Use -s to specify a set.' % (token, )) return 2 elif not options.ignore_failures: out.error('No matches for %r; ignoring it' % token) else: return -1 else: atoms.append(a) if not atoms and not options.newuse: out.error('No targets specified; nothing to do') return 1 atoms = stable_unique(atoms) if (not options.set or options.clean) and not options.oneshot: if world_set is None: err.write( "Disable world updating via --oneshot, or fix your configuration" ) return 1 if options.upgrade: resolver_kls = resolver.upgrade_resolver else: resolver_kls = resolver.min_install_resolver extra_kwargs = {} if options.empty: extra_kwargs['resolver_cls'] = resolver.empty_tree_merge_plan if options.debug: extra_kwargs['debug'] = True # XXX: This should recurse on deep if options.newuse: out.write(out.bold, ' * ', out.reset, 'Scanning for changed USE...') out.title('Scanning for changed USE...') for inst_pkg in installed_repos.itermatch(OrRestriction(*atoms)): src_pkgs = source_repos.match(inst_pkg.versioned_atom) if src_pkgs: src_pkg = max(src_pkgs) inst_iuse = set(use.lstrip("+-") for use in inst_pkg.iuse) src_iuse = set(use.lstrip("+-") for use in src_pkg.iuse) inst_flags = inst_iuse.intersection(inst_pkg.use) src_flags = src_iuse.intersection(src_pkg.use) if inst_flags.symmetric_difference(src_flags) or \ inst_pkg.iuse.symmetric_difference(src_pkg.iuse): atoms.append(src_pkg.unversioned_atom) # left intentionally in place for ease of debugging. # from guppy import hpy # hp = hpy() # hp.setrelheap() resolver_inst = resolver_kls( installed_repos.repositories, source_repos.repositories, verify_vdb=options.deep, nodeps=options.nodeps, drop_cycles=options.ignore_cycles, force_replace=options.replace, process_built_depends=options.with_built_depends, **extra_kwargs) if options.preload_vdb_state: out.write(out.bold, ' * ', out.reset, 'Preloading vdb... ') vdb_time = time() resolver_inst.load_vdb_state() vdb_time = time() - vdb_time else: vdb_time = 0.0 failures = [] resolve_time = time() out.title('Resolving...') out.write(out.bold, ' * ', out.reset, 'Resolving...') ret = resolver_inst.add_atoms(atoms, finalize=True) while ret: out.error('resolution failed') restrict = ret[0][0] just_failures = reduce_to_failures(ret[1]) display_failures(out, just_failures, debug=options.debug) failures.append(restrict) if not options.ignore_failures: break out.write("restarting resolution") atoms = [x for x in atoms if x != restrict] resolver_inst.reset() ret = resolver_inst.add_atoms(atoms, finalize=True) resolve_time = time() - resolve_time if options.debug: out.write(out.bold, " * ", out.reset, "resolution took %.2f seconds" % resolve_time) if failures: out.write() out.write('Failures encountered:') for restrict in failures: out.error("failed '%s'" % (restrict, )) out.write('potentials:') match_count = 0 for r in repo_utils.get_raw_repos(source_repos.repositories): l = r.match(restrict) if l: out.write("repo %s: [ %s ]" % (r, ", ".join(str(x) for x in l))) match_count += len(l) if not match_count: out.write("No matches found in %s" % (source_repos.repositories, )) out.write() if not options.ignore_failures: return 1 resolver_inst.free_caches() if options.clean: out.write(out.bold, ' * ', out.reset, 'Packages to be removed:') vset = set(installed_repos.combined) len_vset = len(vset) vset.difference_update(x.pkg for x in resolver_inst.state.iter_ops(True)) wipes = sorted(x for x in vset if x.package_is_real) for x in wipes: out.write("Remove %s" % x) out.write() if wipes: out.write("removing %i packages of %i installed, %0.2f%%." % (len(wipes), len_vset, 100 * (len(wipes) / float(len_vset)))) else: out.write("no packages to remove") if options.pretend: return 0 if options.ask: if not formatter.ask("Do you wish to proceed?", default_answer=False): return 1 out.write() repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug) do_unmerge(options, out, err, installed_repos.combined, wipes, world_set, repo_obs) return 0 if options.debug: out.write() out.write(out.bold, ' * ', out.reset, 'debug: all ops') out.first_prefix.append(" ") plan_len = len(str(len(resolver_inst.state.plan))) for pos, op in enumerate(resolver_inst.state.plan): out.write(str(pos + 1).rjust(plan_len), ': ', str(op)) out.first_prefix.pop() out.write(out.bold, ' * ', out.reset, 'debug: end all ops') out.write() changes = resolver_inst.state.ops(only_real=True) build_obs = observer.build_observer(observer.formatter_output(out), not options.debug) repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug) if options.debug: out.write(out.bold, " * ", out.reset, "running sanity checks") start_time = time() if not changes.run_sanity_checks(domain, build_obs): out.error("sanity checks failed. please resolve them and try again.") return 1 if options.debug: out.write( out.bold, " * ", out.reset, "finished sanity checks in %.2f seconds" % (time() - start_time)) out.write() if options.ask or options.pretend: for op in changes: formatter.format(op) formatter.end() if vdb_time: out.write(out.bold, 'Took %.2f' % (vdb_time, ), out.reset, ' seconds to preload vdb state') if not changes: out.write("Nothing to merge.") return if options.pretend: if options.verbose: out.write( out.bold, ' * ', out.reset, "resolver plan required %i ops (%.2f seconds)\n" % (len(resolver_inst.state.plan), resolve_time)) return if (options.ask and not formatter.ask("Would you like to merge these packages?")): return change_count = len(changes) # left in place for ease of debugging. cleanup = [] try: for count, op in enumerate(changes): for func in cleanup: func() cleanup = [] out.write("\nProcessing %i of %i: %s" % (count + 1, change_count, op.pkg.cpvstr)) out.title("%i/%i: %s" % (count + 1, change_count, op.pkg.cpvstr)) if op.desc != "remove": cleanup = [op.pkg.release_cached_data] if not options.fetchonly and options.debug: out.write("Forcing a clean of workdir") pkg_ops = domain.pkg_operations(op.pkg, observer=build_obs) out.write("\n%i files required-" % len(op.pkg.fetchables)) try: ret = pkg_ops.run_if_supported("fetch", or_return=True) except IGNORED_EXCEPTIONS: raise except Exception as e: ret = e if ret is not True: if ret is False: ret = None commandline.dump_error( out, ret, "\nfetching failed for %s" % (op.pkg.cpvstr, )) if not options.ignore_failures: return 1 continue if options.fetchonly: continue buildop = pkg_ops.run_if_supported("build", or_return=None) pkg = op.pkg if buildop is not None: out.write("building %s" % (op.pkg.cpvstr, )) result = False try: result = buildop.finalize() except format.errors as e: out.error("caught exception building %s: % s" % (op.pkg.cpvstr, e)) else: if result is False: out.error("failed building %s" % (op.pkg.cpvstr, )) if result is False: if not options.ignore_failures: return 1 continue pkg = result cleanup.append(pkg.release_cached_data) pkg_ops = domain.pkg_operations(pkg, observer=build_obs) cleanup.append(buildop.cleanup) cleanup.append(partial(pkg_ops.run_if_supported, "cleanup")) pkg = pkg_ops.run_if_supported("localize", or_return=pkg) # wipe this to ensure we don't inadvertantly use it further down; # we aren't resetting it after localizing, so could have the wrong # set of ops. del pkg_ops out.write() if op.desc == "replace": if op.old_pkg == pkg: out.write(">>> Reinstalling %s" % (pkg.cpvstr)) else: out.write(">>> Replacing %s with %s" % (op.old_pkg.cpvstr, pkg.cpvstr)) i = domain.replace_pkg(op.old_pkg, pkg, repo_obs) cleanup.append(op.old_pkg.release_cached_data) else: out.write(">>> Installing %s" % (pkg.cpvstr, )) i = domain.install_pkg(pkg, repo_obs) # force this explicitly- can hold onto a helluva lot more # then we would like. else: out.write(">>> Removing %s" % op.pkg.cpvstr) i = domain.uninstall_pkg(op.pkg, repo_obs) try: ret = i.finish() except merge_errors.BlockModification as e: out.error("Failed to merge %s: %s" % (op.pkg, e)) if not options.ignore_failures: return 1 continue # while this does get handled through each loop, wipe it now; we don't need # that data, thus we punt it now to keep memory down. # for safety sake, we let the next pass trigger a release also- # mainly to protect against any code following triggering reloads # basically, be protective if world_set is not None: if op.desc == "remove": out.write('>>> Removing %s from world file' % op.pkg.cpvstr) removal_pkg = slotatom_if_slotted(source_repos.combined, op.pkg.versioned_atom) update_worldset(world_set, removal_pkg, remove=True) elif not options.oneshot and any( x.match(op.pkg) for x in atoms): if not options.upgrade: out.write('>>> Adding %s to world file' % op.pkg.cpvstr) add_pkg = slotatom_if_slotted(source_repos.combined, op.pkg.versioned_atom) update_worldset(world_set, add_pkg) # again... left in place for ease of debugging. # except KeyboardInterrupt: # import pdb;pdb.set_trace() # else: # import pdb;pdb.set_trace() finally: pass # the final run from the loop above doesn't invoke cleanups; # we could ignore it, but better to run it to ensure nothing is inadvertantly # held on the way out of this function. # makes heappy analysis easier if we're careful about it. for func in cleanup: func() # and wipe the reference to the functions to allow things to fall out of # memory. cleanup = [] out.write("finished") return 0
def main(options, out, err): pkg = options.pkg repos = None if os.path.isfile(pkg) and pkg.endswith('.ebuild'): ebuild_path = os.path.abspath(pkg) repo_path = os.path.abspath(os.path.join( pkg, os.pardir, os.pardir, os.pardir)) # find the ebuild's repo # TODO: iterating through the repos feels wrong, we could use a # multi-keyed dict with repo IDs and paths as keys with repo # objects as values (same thing we need for better portage-2 # profile support) for x in options.domain.ebuild_repos: if x.raw_repo.location == repo_path: repos = x break if repos is None: err.write('no configured repo contains: %s' % ebuild_path) return 1 ebuild_P = os.path.basename(os.path.splitext(ebuild_path)[0]) ebuild_category = ebuild_path.split(os.sep)[-3] pkg = atom.atom('=%s/%s' % (ebuild_category, ebuild_P)) else: try: pkg = atom.atom(pkg) repos = options.domain.all_repos except MalformedAtom: err.write('not a valid atom or ebuild: "%s"' % pkg) return 1 pkgs = repos.match(pkg) if not pkgs: err.write('got no matches for %s\n' % (pkg,)) return 1 if len(pkgs) > 1: err.write('got multiple matches for %s:' % (pkg,)) if len(set((pkg.slot, pkg.repo) for pkg in pkgs)) != 1: for pkg in sorted(pkgs): err.write("repo %r, slot %r, %s" % (getattr(pkg.repo, 'repo_id', 'unknown'), pkg.slot, pkg.cpvstr,), prefix=" ") err.write() err.write("please refine your restriction to match only one slot/repo pair\n") return 1 pkgs = [max(pkgs)] err.write("choosing %r, slot %r, %s" % (getattr(pkgs[0].repo, 'repo_id', 'unknown'), pkgs[0].slot, pkgs[0].cpvstr), prefix=' ') kwds = {} build_obs = observer.build_observer(observer.formatter_output(out), not options.debug) phases = [x for x in options.phase if x != 'clean'] clean = (len(phases) != len(options.phase)) if options.no_auto: kwds["ignore_deps"] = True if "setup" in phases: phases.insert(0, "fetch") # by default turn off startup cleans; we clean by ourselves if # told to do so via an arg build = options.domain.build_pkg(pkgs[0], build_obs, clean=False, allow_fetching=True) if clean: build.cleanup(force=True) build._reload_state() phase_funcs = (getattr(build, x) for x in phases) for phase, f in izip(phases, phase_funcs): out.write('executing phase %s' % (phase,)) f(**kwds)
def main(options, out, err): config = options.config if options.debug: resolver.plan.limiters.add(None) domain = options.domain livefs_repos = domain.all_livefs_repos world_set = world_list = options.world if options.oneshot: world_set = None formatter = options.formatter( out=out, err=err, unstable_arch=domain.unstable_arch, domain_settings=domain.settings, use_expand=domain.profile.use_expand, use_expand_hidden=domain.profile.use_expand_hidden, pkg_get_use=domain.get_package_use_unconfigured, world_list=world_list, verbose=options.verbose, livefs_repos=livefs_repos, distdir=domain.fetcher.get_storage_path(), quiet_repo_display=options.quiet_repo_display) # This mode does not care about sets and packages so bypass all that. if options.unmerge: if not options.oneshot: if world_set is None: err.write("Disable world updating via --oneshot, " "or fix your configuration") return 1 try: unmerge(out, err, livefs_repos, options.targets, options, formatter, world_set) except (parserestrict.ParseError, Failure) as e: out.error(str(e)) return 1 return source_repos = domain.source_repos installed_repos = domain.installed_repos if options.usepkgonly: source_repos = domain.binary_repos elif options.usepkg: source_repos = domain.binary_repos + domain.ebuild_repos elif options.source_only: source_repos = domain.ebuild_repos atoms = [] for setname, pkgset in options.sets: if pkgset is None: return 1 l = list(pkgset) if not l: out.write("skipping set %s: set is empty, nothing to update" % setname) else: atoms.extend(l) for token, restriction in options.targets: try: matches = parse_target(restriction, source_repos.combined, livefs_repos, return_none=True) except parserestrict.ParseError as e: out.error(str(e)) return 1 if matches is None: if token in config.pkgset: out.error( "No package matches '%s', but there is a set with " 'that name. Use @set to specify a set.' % (token,)) return 2 elif not options.ignore_failures: out.error("No matches for '%s'; ignoring it" % (token,)) else: return -1 else: atoms.extend(matches) if not atoms and not options.newuse: out.error('No targets specified; nothing to do') return 1 atoms = stable_unique(atoms) if options.clean and not options.oneshot: if world_set is None: err.write("Disable world updating via --oneshot, or fix your configuration") return 1 if options.upgrade: resolver_kls = resolver.upgrade_resolver else: resolver_kls = resolver.min_install_resolver extra_kwargs = {} if options.empty: extra_kwargs['resolver_cls'] = resolver.empty_tree_merge_plan if options.debug: extra_kwargs['debug'] = True # XXX: This should recurse on deep if options.newuse: out.write(out.bold, ' * ', out.reset, 'Scanning for changed USE...') out.title('Scanning for changed USE...') for inst_pkg in installed_repos.itermatch(OrRestriction(*atoms)): src_pkgs = source_repos.match(inst_pkg.versioned_atom) if src_pkgs: src_pkg = max(src_pkgs) inst_iuse = inst_pkg.iuse_stripped src_iuse = src_pkg.iuse_stripped inst_flags = inst_iuse.intersection(inst_pkg.use) src_flags = src_iuse.intersection(src_pkg.use) if inst_flags.symmetric_difference(src_flags) or \ inst_iuse.symmetric_difference(src_iuse): atoms.append(src_pkg.unversioned_atom) # left intentionally in place for ease of debugging. # from guppy import hpy # hp = hpy() # hp.setrelheap() resolver_inst = resolver_kls( vdbs=installed_repos.repos, dbs=source_repos.repos, verify_vdb=options.deep, nodeps=options.nodeps, drop_cycles=options.ignore_cycles, force_replace=options.replace, process_built_depends=options.with_bdeps, **extra_kwargs) if options.preload_vdb_state: out.write(out.bold, ' * ', out.reset, 'Preloading vdb... ') vdb_time = time() resolver_inst.load_vdb_state() vdb_time = time() - vdb_time else: vdb_time = 0.0 failures = [] resolve_time = time() out.title('Resolving...') out.write(out.bold, ' * ', out.reset, 'Resolving...') ret = resolver_inst.add_atoms(atoms, finalize=True) while ret: out.error('resolution failed') restrict = ret[0][0] just_failures = reduce_to_failures(ret[1]) display_failures(out, just_failures, debug=options.debug) failures.append(restrict) if not options.ignore_failures: break out.write("restarting resolution") atoms = [x for x in atoms if x != restrict] resolver_inst.reset() ret = resolver_inst.add_atoms(atoms, finalize=True) resolve_time = time() - resolve_time if options.debug: out.write(out.bold, " * ", out.reset, "resolution took %.2f seconds" % resolve_time) if failures: out.write() out.write('Failures encountered:') for restrict in failures: out.error("failed '%s'" % (restrict,)) out.write('potentials:') match_count = 0 for r in repo_utils.get_raw_repos(source_repos.repos): l = r.match(restrict) if l: out.write( "repo %s: [ %s ]" % (r, ", ".join(str(x) for x in l))) match_count += len(l) if not match_count: out.write("No matches found in %s" % (source_repos.repos,)) out.write() if not options.ignore_failures: return 1 resolver_inst.free_caches() if options.clean: out.write(out.bold, ' * ', out.reset, 'Packages to be removed:') vset = set(installed_repos.combined) len_vset = len(vset) vset.difference_update(x.pkg for x in resolver_inst.state.iter_ops(True)) wipes = sorted(x for x in vset if x.package_is_real) for x in wipes: out.write("Remove %s" % x) out.write() if wipes: out.write("removing %i packages of %i installed, %0.2f%%." % (len(wipes), len_vset, 100*(len(wipes)/float(len_vset)))) else: out.write("no packages to remove") if options.pretend: return 0 if options.ask: if not formatter.ask("Do you wish to proceed?", default_answer=False): return 1 out.write() repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug) do_unmerge(options, out, err, installed_repos.combined, wipes, world_set, repo_obs) return 0 if options.debug: out.write() out.write(out.bold, ' * ', out.reset, 'debug: all ops') out.first_prefix.append(" ") plan_len = len(str(len(resolver_inst.state.plan))) for pos, op in enumerate(resolver_inst.state.plan): out.write(str(pos + 1).rjust(plan_len), ': ', str(op)) out.first_prefix.pop() out.write(out.bold, ' * ', out.reset, 'debug: end all ops') out.write() changes = resolver_inst.state.ops(only_real=True) build_obs = observer.build_observer(observer.formatter_output(out), not options.debug) repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug) if options.ask or options.pretend: for op in changes: formatter.format(op) formatter.end() if vdb_time: out.write(out.bold, 'Took %.2f' % (vdb_time,), out.reset, ' seconds to preload vdb state') if not changes: out.write("Nothing to merge.") return if options.pretend: if options.verbose: out.write( out.bold, ' * ', out.reset, "resolver plan required %i ops (%.2f seconds)\n" % (len(resolver_inst.state.plan), resolve_time)) return if (options.ask and not formatter.ask("Would you like to merge these packages?")): return if options.debug: out.write(out.bold, " * ", out.reset, "running sanity checks") start_time = time() if not changes.run_sanity_checks(domain, build_obs): out.error("sanity checks failed. please resolve them and try again.") return 1 if options.debug: out.write( out.bold, " * ", out.reset, "finished sanity checks in %.2f seconds" % (time() - start_time)) out.write() change_count = len(changes) # left in place for ease of debugging. cleanup = [] try: for count, op in enumerate(changes): for func in cleanup: func() cleanup = [] out.write("\nProcessing %i of %i: %s" % (count + 1, change_count, op.pkg.cpvstr)) out.title("%i/%i: %s" % (count + 1, change_count, op.pkg.cpvstr)) if op.desc != "remove": cleanup = [op.pkg.release_cached_data] if not options.fetchonly and options.debug: out.write("Forcing a clean of workdir") pkg_ops = domain.pkg_operations(op.pkg, observer=build_obs) out.write("\n%i files required-" % len(op.pkg.fetchables)) try: ret = pkg_ops.run_if_supported("fetch", or_return=True) except IGNORED_EXCEPTIONS: raise except Exception as e: ret = e if ret is not True: if ret is False: ret = None commandline.dump_error(out, ret, "\nfetching failed for %s" % (op.pkg.cpvstr,)) if not options.ignore_failures: return 1 continue if options.fetchonly: continue buildop = pkg_ops.run_if_supported("build", or_return=None) pkg = op.pkg if buildop is not None: out.write("building %s" % (op.pkg.cpvstr,)) result = False try: result = buildop.finalize() except format.errors as e: out.error("caught exception building %s: % s" % (op.pkg.cpvstr, e)) else: if result is False: out.error("failed building %s" % (op.pkg.cpvstr,)) if result is False: if not options.ignore_failures: return 1 continue pkg = result cleanup.append(pkg.release_cached_data) pkg_ops = domain.pkg_operations(pkg, observer=build_obs) cleanup.append(buildop.cleanup) cleanup.append(partial(pkg_ops.run_if_supported, "cleanup")) pkg = pkg_ops.run_if_supported("localize", or_return=pkg) # wipe this to ensure we don't inadvertantly use it further down; # we aren't resetting it after localizing, so could have the wrong # set of ops. del pkg_ops out.write() if op.desc == "replace": if op.old_pkg == pkg: out.write(">>> Reinstalling %s" % (pkg.cpvstr)) else: out.write(">>> Replacing %s with %s" % ( op.old_pkg.cpvstr, pkg.cpvstr)) i = domain.replace_pkg(op.old_pkg, pkg, repo_obs) cleanup.append(op.old_pkg.release_cached_data) else: out.write(">>> Installing %s" % (pkg.cpvstr,)) i = domain.install_pkg(pkg, repo_obs) # force this explicitly- can hold onto a helluva lot more # then we would like. else: out.write(">>> Removing %s" % op.pkg.cpvstr) i = domain.uninstall_pkg(op.pkg, repo_obs) try: ret = i.finish() except merge_errors.BlockModification as e: out.error("Failed to merge %s: %s" % (op.pkg, e)) if not options.ignore_failures: return 1 continue # while this does get handled through each loop, wipe it now; we don't need # that data, thus we punt it now to keep memory down. # for safety sake, we let the next pass trigger a release also- # mainly to protect against any code following triggering reloads # basically, be protective if world_set is not None: if op.desc == "remove": out.write('>>> Removing %s from world file' % op.pkg.cpvstr) removal_pkg = slotatom_if_slotted(source_repos.combined, op.pkg.versioned_atom) update_worldset(world_set, removal_pkg, remove=True) elif not options.oneshot and any(x.match(op.pkg) for x in atoms): if not options.upgrade: out.write('>>> Adding %s to world file' % op.pkg.cpvstr) add_pkg = slotatom_if_slotted(source_repos.combined, op.pkg.versioned_atom) update_worldset(world_set, add_pkg) # again... left in place for ease of debugging. # except KeyboardInterrupt: # import pdb;pdb.set_trace() # else: # import pdb;pdb.set_trace() finally: pass # the final run from the loop above doesn't invoke cleanups; # we could ignore it, but better to run it to ensure nothing is # inadvertantly held on the way out of this function. # makes heappy analysis easier if we're careful about it. for func in cleanup: func() # and wipe the reference to the functions to allow things to fall out of # memory. cleanup = [] out.write("finished") return 0
wipes = sorted(x for x in vset if x.package_is_real) for x in wipes: out.write("Remove %s" % x) out.write() if wipes: out.write("removing %i packages of %i installed, %0.2f%%." % (len(wipes), len_vset, 100*(len(wipes)/float(len_vset)))) else: out.write("no packages to remove") if options.pretend: return 0 if options.ask: if not formatter.ask("Do you wish to proceed?", default_answer=False): return 1 out.write() repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug) do_unmerge(options, out, err, installed_repos.combined, wipes, world_set, repo_obs) return 0 if options.debug: out.write() out.write(out.bold, ' * ', out.reset, 'debug: all ops') out.first_prefix.append(" ") plan_len = len(str(len(resolver_inst.state.plan))) for pos, op in enumerate(resolver_inst.state.plan): out.write(str(pos + 1).rjust(plan_len), ': ', str(op)) out.first_prefix.pop() out.write(out.bold, ' * ', out.reset, 'debug: end all ops') out.write()
# pull all matches and drop untracked ebuilds p = git.run( 'status', '--porcelain=v1', '-u', '-z', "*.ebuild", cwd=repo.location, stdout=subprocess.PIPE) for path in p.stdout.strip('\x00').split('\x00'): if mo := _untracked_ebuild_re.match(path): try: untracked = atom_cls(f"={mo.group('category')}/{mo.group('package')}") pkgs.discard(untracked) except MalformedAtom: continue # manifest all staged or committed packages failed = repo.operations.manifest( options.domain, packages.OrRestriction(*pkgs), observer=observer_mod.formatter_output(out)) if any(failed): return 1 # include existing Manifest files for staging manifests = (pjoin(repo.location, f'{x.key}/Manifest') for x in atoms) git_add_files.extend(filter(os.path.exists, manifests)) # mangle files if options.mangle: # don't mangle FILESDIR content skip_regex = re.compile(rf'^{repo.location}/[^/]+/[^/]+/files/.+$') mangler = GentooMangler if options.gentoo_repo else Mangler paths = (pjoin(repo.location, x) for x in changes.paths) git_add_files.extend(mangler(paths, skip_regex=skip_regex))
def main(options, out, err): token, restriction = options.target[0] domain = options.domain try: pkgs = options.repo.match(restriction, pkg_filter=None) except MetadataException as e: error = e.msg(verbosity=options.verbosity) argparser.error(f'{e.pkg.cpvstr}::{e.pkg.repo.repo_id}: {error}') if not pkgs: argparser.error(f"no matches: {token!r}") pkg = max(pkgs) if len(pkgs) > 1: argparser.err.write(f"got multiple matches for {token!r}:") if len(set((p.slot, p.repo) for p in pkgs)) != 1: for p in pkgs: repo_id = getattr(p.repo, 'repo_id', 'unknown') argparser.err.write(f"{p.cpvstr}:{p.slot}::{repo_id}", prefix=' ') argparser.err.write() argparser.error("please refine your restriction to one match") repo_id = getattr(pkg.repo, 'repo_id', 'unknown') argparser.err.write(f"choosing {pkg.cpvstr}:{pkg.slot}::{repo_id}", prefix=' ') sys.stderr.flush() kwds = {} phase_obs = observer.phase_observer(observer.formatter_output(out), options.debug) phases = [x for x in options.phase if x != 'clean'] clean = (len(phases) != len(options.phase)) if options.no_auto: kwds["ignore_deps"] = True if "setup" in phases: phases.insert(0, "fetch") # forcibly run test phase if selected force_test = 'test' in phases if force_test and 'test' in pkg.iuse: pkg.use.add('test') # by default turn off startup cleans; we clean by ourselves if # told to do so via an arg build = domain.build_pkg(pkg, failed=True, clean=False, allow_fetching=True, observer=phase_obs, force_test=force_test) if clean: build.cleanup(force=True) build._reload_state() phase_funcs = [(p, getattr(build, p, None)) for p in phases] unknown_phases = [p for p, func in phase_funcs if func is None] if unknown_phases: argparser.error( "unknown phase%s: %s" % (pluralism(unknown_phases), ', '.join(map(repr, unknown_phases)))) try: for phase, func in phase_funcs: out.write(f'executing phase {phase}') func(**kwds) except OperationError as e: raise ExitException( f"caught exception executing phase {phase}: {e}") from e
def unmerge(out, err, installed_repos, targets, options, formatter, world_set=None): """Unmerge tokens. hackish, should be rolled back into the resolver""" # split real and virtual repos vdb = installed_repos.real.combined fake_vdb = installed_repos.virtual.combined matches = set() fake = set() unknown = set() for token, restriction in targets: # Catch restrictions matching across more than one category. # Multiple matches in the same category are acceptable. # The point is that matching across more than one category is # nearly always unintentional ("pmerge -C spork" without # realising there are sporks in more than one category), but # matching more than one cat/pkg is impossible without # explicit wildcards. installed = vdb.match(restriction) if not installed: fake_pkgs = fake_vdb.match(restriction) if fake_pkgs: fake.update(fake_pkgs) else: unknown.add(token) continue categories = set(pkg.category for pkg in installed) if len(categories) > 1: raise parserestrict.ParseError( "%r is in multiple categories (%s)" % (token, ', '.join(sorted(set(pkg.key for pkg in installed))))) matches.update(installed) # fail out if no matches are found, otherwise just output a notification if unknown: unknowns = ', '.join(map(repr, unknown)) if matches: err.write(f"Skipping unknown matches: {unknowns}\n") else: raise Failure(f"no matches found: {unknowns}") if fake: err.write( 'Skipping virtual pkg%s: %s' % (pluralism(fake_pkgs), ', '.join(f'{x.versioned_atom}::{x.repo_id}' for x in fake))) if matches: out.write(out.bold, 'The following packages are to be unmerged:') out.prefix = [out.bold, ' * ', out.reset] for pkg in matches: out.write(pkg.cpvstr) out.prefix = [] repo_obs = observer.repo_observer(observer.formatter_output(out), debug=options.debug) if options.pretend: return if (options.ask and not formatter.ask("Would you like to unmerge these packages?") ): return return do_unmerge(options, out, err, vdb, matches, world_set, repo_obs)
def main(options, out, err): target = options.target domain = options.domain repo = domain.ebuild_repos_raw if target.endswith('.ebuild'): if not os.path.isfile(target): argparser.error("ebuild doesn't exist: '%s'" % target) try: restriction = repo.path_restrict(target) except ValueError as e: argparser.error(e) else: try: restriction = atom.atom(target) except MalformedAtom: if os.path.isfile(target): argparser.error("file not an ebuild: '%s'" % target) else: argparser.error("invalid package atom: '%s'" % target) pkgs = repo.match(restriction) if not pkgs: argparser.error("no matches: '%s'" % (target,)) pkg = max(pkgs) if len(pkgs) > 1: err.write("got multiple matches for '%s':" % (target,)) if len(set((p.slot, p.repo) for p in pkgs)) != 1: for p in pkgs: err.write( "%s:%s::%s" % (p.cpvstr, p.slot, getattr(p.repo, 'repo_id', 'unknown')), prefix=' ') err.write() argparser.error("please refine your restriction to one match") err.write( "choosing %s:%s::%s" % (pkg.cpvstr, pkg.slot, getattr(pkg.repo, 'repo_id', 'unknown')), prefix=' ') kwds = {} phase_obs = observer.phase_observer(observer.formatter_output(out), not options.debug) phases = [x for x in options.phase if x != 'clean'] clean = (len(phases) != len(options.phase)) if options.no_auto: kwds["ignore_deps"] = True if "setup" in phases: phases.insert(0, "fetch") # by default turn off startup cleans; we clean by ourselves if # told to do so via an arg build = domain.build_pkg(pkg, phase_obs, clean=False, allow_fetching=True) if clean: build.cleanup(force=True) build._reload_state() phase_funcs = [] for phase in phases: p = getattr(build, phase, None) if p is None: argparser.error("unknown phase: '%s'" % phase) phase_funcs.append(p) try: for phase, f in izip(phases, phase_funcs): out.write('executing phase %s' % (phase,)) f(**kwds) except format.errors: return 1
def main(options, out, err): if options.list_sets: display_pkgsets(out, options) return 0 config = options.config if options.debug: resolver.plan.limiters.add(None) domain = options.domain world_set = world_list = options.world if options.oneshot: world_set = None formatter = options.formatter( out=out, err=err, unstable_arch=domain.unstable_arch, use_expand=domain.profile.use_expand, use_expand_hidden=domain.profile.use_expand_hidden, pkg_get_use=domain.get_package_use_unconfigured, world_list=world_list, verbosity=options.verbosity, installed_repos=domain.all_installed_repos, distdir=domain.fetcher.get_storage_path(), quiet_repo_display=options.quiet_repo_display) # This mode does not care about sets and packages so bypass all that. if options.unmerge: if not options.oneshot: if world_set is None: argparser.error("disable world updating via --oneshot, " "or fix your configuration") try: unmerge(out, err, domain.installed_repos, options.targets, options, formatter, world_set) except (parserestrict.ParseError, Failure) as e: argparser.error(e) return source_repos = domain.source_repos installed_repos = domain.installed_repos pkg_type = 'ebuilds' if options.usepkgonly: source_repos = domain.binary_repos pkg_type = 'binpkgs' elif options.usepkg: # binary repos are checked for matches first before ebuild repos source_repos = domain.binary_repos + domain.ebuild_repos pkg_type = 'ebuilds or binpkgs' elif options.source_only: source_repos = domain.ebuild_repos atoms = [] for setname, pkgset in options.sets: if pkgset is None: return 1 l = list(pkgset) if not l: out.write( f"skipping set {setname!r}: set is empty, nothing to update") else: atoms.extend(l) for token, restriction in options.targets: try: matches = parse_target(restriction, source_repos.combined, installed_repos, return_none=True) except parserestrict.ParseError as e: e.token = token argparser.error(e) if matches is None: if not options.ignore_failures: error_msg = [f"no matching {pkg_type}: {token!r}"] if token in config.pkgset: error_msg.append( f"use '@{token}' instead for the package set") elif options.usepkgonly: matches = parse_target(restriction, domain.ebuild_repos.combined, installed_repos, return_none=True) if matches: error_msg.append( "try re-running without -K/--usepkgonly " "enabled to rebuild from source") argparser.error(' -- '.join(error_msg)) else: atoms.extend(matches) if not atoms and not options.newuse: err.write(f"{options.prog}: no targets specified; nothing to do") return 1 atoms = stable_unique(atoms) if options.clean and not options.oneshot: if world_set is None: argparser.error( "disable world updating via --oneshot, or fix your configuration" ) extra_kwargs = {} if options.empty: extra_kwargs['resolver_cls'] = resolver.empty_tree_merge_plan if options.debug: extra_kwargs['debug'] = True # XXX: This should recurse on deep if options.newuse: out.write(out.bold, ' * ', out.reset, 'Scanning for changed USE...') out.title('Scanning for changed USE...') for inst_pkg in installed_repos.itermatch(OrRestriction(*atoms)): src_pkgs = source_repos.match(inst_pkg.versioned_atom) if src_pkgs: src_pkg = max(src_pkgs) inst_iuse = inst_pkg.iuse_stripped src_iuse = src_pkg.iuse_stripped inst_flags = inst_iuse.intersection(inst_pkg.use) src_flags = src_iuse.intersection(src_pkg.use) if inst_flags.symmetric_difference(src_flags) or \ inst_iuse.symmetric_difference(src_iuse): atoms.append(src_pkg.unversioned_atom) excludes = [restriction for token, restriction in options.excludes] if options.onlydeps: excludes.extend(atoms) if excludes: injected_repo = RestrictionRepo(repo_id='injected', restrictions=excludes, frozen=True, livefs=True) installed_repos = injected_repo + installed_repos # left intentionally in place for ease of debugging. # from guppy import hpy # hp = hpy() # hp.setrelheap() resolver_inst = options.resolver_kls( vdbs=installed_repos, dbs=source_repos, verify_vdb=options.deep, nodeps=options.nodeps, drop_cycles=options.ignore_cycles, force_replace=options.replace, process_built_depends=options.with_bdeps, **extra_kwargs) if options.preload_vdb_state: out.write(out.bold, ' * ', out.reset, 'Preloading vdb... ') vdb_time = time() resolver_inst.load_vdb_state() vdb_time = time() - vdb_time else: vdb_time = 0.0 # flush warning messages before dep resolution begins out.flush() err.flush() failures = [] resolve_time = time() if sys.stdout.isatty(): out.title('Resolving...') out.write(out.bold, ' * ', out.reset, 'Resolving...') out.flush() ret = resolver_inst.add_atoms(atoms, finalize=True) while ret: out.error('resolution failed') restrict = ret[0][0] just_failures = reduce_to_failures(ret[1]) display_failures(out, just_failures, debug=options.debug) failures.append(restrict) if not options.ignore_failures: break out.write("restarting resolution") atoms = [x for x in atoms if x != restrict] resolver_inst.reset() ret = resolver_inst.add_atoms(atoms, finalize=True) resolve_time = time() - resolve_time if failures: out.write() out.write('Failures encountered:') for restrict in failures: out.error(f"failed '{restrict}'") out.write('potentials:') match_count = 0 for r in get_raw_repos(source_repos): l = r.match(restrict) if l: out.write(f"repo {r}: [ {', '.join(map(str, l))} ]") match_count += len(l) if not match_count: out.write("No matches found") if not options.ignore_failures: return 1 out.write() resolver_inst.free_caches() if options.clean: out.write(out.bold, ' * ', out.reset, 'Packages to be removed:') vset = set(installed_repos.real.combined) len_vset = len(vset) vset.difference_update(x.pkg for x in resolver_inst.state.iter_ops(True)) wipes = sorted(x for x in vset if x.package_is_real) for x in wipes: out.write(f"Remove {x}") out.write() if wipes: out.write("removing %i packages of %i installed, %0.2f%%." % (len(wipes), len_vset, 100 * (len(wipes) / float(len_vset)))) else: out.write("no packages to remove") if options.pretend: return 0 if options.ask: if not formatter.ask("Do you wish to proceed?", default_answer=False): return 1 out.write() repo_obs = observer.repo_observer(observer.formatter_output(out), debug=options.debug) do_unmerge(options, out, err, installed_repos.real.combined, wipes, world_set, repo_obs) return 0 if options.debug: out.write() out.write(out.bold, ' * ', out.reset, 'debug: all ops') out.first_prefix.append(" ") plan_len = len(str(len(resolver_inst.state.plan))) for pos, op in enumerate(resolver_inst.state.plan): out.write(str(pos + 1).rjust(plan_len), ': ', str(op)) out.first_prefix.pop() out.write(out.bold, ' * ', out.reset, 'debug: end all ops') out.write() changes = resolver_inst.state.ops(only_real=True) build_obs = observer.phase_observer(observer.formatter_output(out), debug=options.debug) repo_obs = observer.repo_observer(observer.formatter_output(out), debug=options.debug) # show pkgs to merge in selected format if (options.ask or options.pretend) and changes: for op in changes: formatter.format(op) formatter.end() if vdb_time: out.write(out.bold, 'Took %.2f' % (vdb_time, ), out.reset, ' seconds to preload vdb state') if changes: if not options.fetchonly: # run sanity checks for pkgs -- pkg_pretend, REQUIRED_USE, etc out.write() out.write(out.bold, " * ", out.reset, "Running sanity checks...") if options.debug: start_time = time() # flush output so bash spawned errors are shown in the correct order of events out.flush() sanity_failures = run_sanity_checks((x.pkg for x in changes), domain, threads=1) if sanity_failures: for pkg, errors in sanity_failures.items(): out.write(pkg.cpvstr) out.write('\n'.join( e.msg(verbosity=options.verbosity) for e in errors)) out.write() if options.ignore_failures: out.write(out.fg('red'), out.bold, "!!! ", out.reset, "Skipping failed sanity checks...") else: out.write(out.fg('red'), out.bold, "!!! ", out.reset, "Sanity checks failed, exiting...") return 1 else: out.write() if options.debug: out.write( out.bold, " * ", out.reset, "finished sanity checks in %.2f seconds" % (time() - start_time)) out.write() elif options.verbosity > 0: # show skipped virtuals virtual_pkgs = set() for x in atoms: matches = installed_repos.virtual.match(x) if matches: virtual_pkgs.add(sorted(matches)[-1]) if virtual_pkgs: out.write("Skipping virtual pkgs:\n%s\n" % '\n'.join(str(x.versioned_atom) for x in virtual_pkgs)) out.write("Nothing to merge.") return if options.pretend: if options.verbosity > 0: out.write( out.bold, ' * ', out.reset, "resolver plan required %i ops (%.2f seconds)" % (len(resolver_inst.state.plan), resolve_time)) return action = 'merge' if options.fetchonly: action = 'fetch' if (options.ask and not formatter.ask(f"Would you like to {action} these packages?")): return change_count = len(changes) # left in place for ease of debugging. cleanup = [] try: for count, op in enumerate(changes): for func in cleanup: func() cleanup = [] out.write(f"\nProcessing {count + 1} of {change_count}: " f"{op.pkg.cpvstr}::{op.pkg.repo}") out.title(f"{count + 1}/{change_count}: {op.pkg.cpvstr}") if op.desc != "remove": cleanup.append(op.pkg.release_cached_data) if not options.fetchonly and options.debug: out.write("Forcing a clean of workdir") pkg_ops = domain.pkg_operations(op.pkg, observer=build_obs) out.write( f"\n{len(op.pkg.distfiles)} file{pluralism(op.pkg.distfiles)} required-" ) if not pkg_ops.run_if_supported("fetch", or_return=True): out.error(f"fetching failed for {op.pkg.cpvstr}") if not options.ignore_failures: return 1 continue if options.fetchonly: continue buildop = pkg_ops.run_if_supported("build", or_return=None) pkg = op.pkg if buildop is not None: out.write(f"building {op.pkg.cpvstr}") result = False exc = None try: result = buildop.finalize() except format.BuildError as e: out.error( f"caught exception building {op.pkg.cpvstr}: {e}") exc = e else: if result is False: out.error(f"failed building {op.pkg.cpvstr}") if result is False: if not options.ignore_failures: raise ExitException(1) from exc continue pkg = result cleanup.append(pkg.release_cached_data) pkg_ops = domain.pkg_operations(pkg, observer=build_obs) cleanup.append(buildop.cleanup) cleanup.append(partial(pkg_ops.run_if_supported, "cleanup")) pkg = pkg_ops.run_if_supported("localize", or_return=pkg) # wipe this to ensure we don't inadvertantly use it further down; # we aren't resetting it after localizing, so could have the wrong # set of ops. del pkg_ops out.write() if op.desc == "replace": if op.old_pkg == pkg: out.write(f">>> Reinstalling {pkg.cpvstr}") else: out.write( f">>> Replacing {op.old_pkg.cpvstr} with {pkg.cpvstr}" ) i = domain.replace_pkg(op.old_pkg, pkg, repo_obs) cleanup.append(op.old_pkg.release_cached_data) else: out.write(f">>> Installing {pkg.cpvstr}") i = domain.install_pkg(pkg, repo_obs) # force this explicitly- can hold onto a helluva lot more # then we would like. else: out.write(f">>> Removing {op.pkg.cpvstr}") i = domain.uninstall_pkg(op.pkg, repo_obs) try: ret = i.finish() except merge_errors.BlockModification as e: out.error(f"Failed to merge {op.pkg}: {e}") if not options.ignore_failures: return 1 continue # while this does get handled through each loop, wipe it now; we don't need # that data, thus we punt it now to keep memory down. # for safety sake, we let the next pass trigger a release also- # mainly to protect against any code following triggering reloads # basically, be protective if world_set is not None: if op.desc == "remove": out.write(f'>>> Removing {op.pkg.cpvstr} from world file') removal_pkg = slotatom_if_slotted(source_repos.combined, op.pkg.versioned_atom) update_worldset(world_set, removal_pkg, remove=True) elif not options.oneshot and any( x.match(op.pkg) for x in atoms): if not (options.upgrade or options.downgrade): out.write(f'>>> Adding {op.pkg.cpvstr} to world file') add_pkg = slotatom_if_slotted(source_repos.combined, op.pkg.versioned_atom) update_worldset(world_set, add_pkg) # again... left in place for ease of debugging. # except KeyboardInterrupt: # import pdb;pdb.set_trace() # else: # import pdb;pdb.set_trace() finally: pass # the final run from the loop above doesn't invoke cleanups; # we could ignore it, but better to run it to ensure nothing is # inadvertantly held on the way out of this function. # makes heappy analysis easier if we're careful about it. for func in cleanup: func() # and wipe the reference to the functions to allow things to fall out of # memory. cleanup = [] return 0