def main(options, out, err):
config = options.config
if options.debug:
resolver.plan.limiters.add(None)
domain = options.domain
livefs_repos = domain.all_livefs_repos
world_set = world_list = options.world
if options.oneshot:
world_set = None
formatter = options.formatter(
out=out, err=err,
unstable_arch=domain.unstable_arch,
domain_settings=domain.settings,
use_expand=domain.profile.use_expand,
use_expand_hidden=domain.profile.use_expand_hidden,
pkg_get_use=domain.get_package_use_unconfigured,
world_list=world_list,
verbose=options.verbose,
livefs_repos=livefs_repos,
distdir=domain.fetcher.get_storage_path(),
quiet_repo_display=options.quiet_repo_display)
# This mode does not care about sets and packages so bypass all that.
if options.unmerge:
if not options.oneshot:
if world_set is None:
err.write("Disable world updating via --oneshot, "
"or fix your configuration")
return 1
try:
unmerge(out, err, livefs_repos, options.targets, options, formatter, world_set)
except (parserestrict.ParseError, Failure) as e:
out.error(str(e))
return 1
return
source_repos = domain.source_repos
installed_repos = domain.installed_repos
if options.usepkgonly:
source_repos = domain.binary_repos
elif options.usepkg:
source_repos = domain.binary_repos + domain.ebuild_repos
elif options.source_only:
source_repos = domain.ebuild_repos
atoms = []
for setname, pkgset in options.sets:
if pkgset is None:
return 1
l = list(pkgset)
if not l:
out.write("skipping set %s: set is empty, nothing to update" % setname)
else:
atoms.extend(l)
for token, restriction in options.targets:
try:
matches = parse_target(restriction, source_repos.combined, livefs_repos, return_none=True)
except parserestrict.ParseError as e:
out.error(str(e))
return 1
if matches is None:
if token in config.pkgset:
out.error(
"No package matches '%s', but there is a set with "
'that name. Use @set to specify a set.' % (token,))
return 2
elif not options.ignore_failures:
out.error("No matches for '%s'; ignoring it" % (token,))
else:
return -1
else:
atoms.extend(matches)
if not atoms and not options.newuse:
out.error('No targets specified; nothing to do')
return 1
atoms = stable_unique(atoms)
if options.clean and not options.oneshot:
if world_set is None:
err.write("Disable world updating via --oneshot, or fix your configuration")
return 1
if options.upgrade:
resolver_kls = resolver.upgrade_resolver
else:
resolver_kls = resolver.min_install_resolver
extra_kwargs = {}
if options.empty:
extra_kwargs['resolver_cls'] = resolver.empty_tree_merge_plan
if options.debug:
extra_kwargs['debug'] = True
# XXX: This should recurse on deep
if options.newuse:
out.write(out.bold, ' * ', out.reset, 'Scanning for changed USE...')
out.title('Scanning for changed USE...')
for inst_pkg in installed_repos.itermatch(OrRestriction(*atoms)):
src_pkgs = source_repos.match(inst_pkg.versioned_atom)
if src_pkgs:
src_pkg = max(src_pkgs)
inst_iuse = inst_pkg.iuse_stripped
src_iuse = src_pkg.iuse_stripped
inst_flags = inst_iuse.intersection(inst_pkg.use)
src_flags = src_iuse.intersection(src_pkg.use)
if inst_flags.symmetric_difference(src_flags) or \
inst_iuse.symmetric_difference(src_iuse):
atoms.append(src_pkg.unversioned_atom)
# left intentionally in place for ease of debugging.
# from guppy import hpy
# hp = hpy()
# hp.setrelheap()
resolver_inst = resolver_kls(
vdbs=installed_repos.repos, dbs=source_repos.repos,
verify_vdb=options.deep, nodeps=options.nodeps,
drop_cycles=options.ignore_cycles, force_replace=options.replace,
process_built_depends=options.with_bdeps, **extra_kwargs)
if options.preload_vdb_state:
out.write(out.bold, ' * ', out.reset, 'Preloading vdb... ')
vdb_time = time()
resolver_inst.load_vdb_state()
vdb_time = time() - vdb_time
else:
vdb_time = 0.0
failures = []
resolve_time = time()
out.title('Resolving...')
out.write(out.bold, ' * ', out.reset, 'Resolving...')
ret = resolver_inst.add_atoms(atoms, finalize=True)
while ret:
out.error('resolution failed')
restrict = ret[0][0]
just_failures = reduce_to_failures(ret[1])
display_failures(out, just_failures, debug=options.debug)
failures.append(restrict)
if not options.ignore_failures:
break
out.write("restarting resolution")
atoms = [x for x in atoms if x != restrict]
resolver_inst.reset()
ret = resolver_inst.add_atoms(atoms, finalize=True)
resolve_time = time() - resolve_time
if options.debug:
out.write(out.bold, " * ", out.reset, "resolution took %.2f seconds" % resolve_time)
if failures:
out.write()
out.write('Failures encountered:')
for restrict in failures:
out.error("failed '%s'" % (restrict,))
out.write('potentials:')
match_count = 0
for r in repo_utils.get_raw_repos(source_repos.repos):
l = r.match(restrict)
if l:
out.write(
"repo %s: [ %s ]" % (r, ", ".join(str(x) for x in l)))
match_count += len(l)
if not match_count:
out.write("No matches found in %s" % (source_repos.repos,))
out.write()
if not options.ignore_failures:
return 1
resolver_inst.free_caches()
if options.clean:
out.write(out.bold, ' * ', out.reset, 'Packages to be removed:')
vset = set(installed_repos.combined)
len_vset = len(vset)
vset.difference_update(x.pkg for x in resolver_inst.state.iter_ops(True))
wipes = sorted(x for x in vset if x.package_is_real)
for x in wipes:
out.write("Remove %s" % x)
out.write()
if wipes:
out.write("removing %i packages of %i installed, %0.2f%%." %
(len(wipes), len_vset, 100*(len(wipes)/float(len_vset))))
else:
out.write("no packages to remove")
if options.pretend:
return 0
if options.ask:
if not formatter.ask("Do you wish to proceed?", default_answer=False):
return 1
out.write()
repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug)
do_unmerge(options, out, err, installed_repos.combined, wipes, world_set, repo_obs)
return 0
if options.debug:
out.write()
out.write(out.bold, ' * ', out.reset, 'debug: all ops')
out.first_prefix.append(" ")
plan_len = len(str(len(resolver_inst.state.plan)))
for pos, op in enumerate(resolver_inst.state.plan):
out.write(str(pos + 1).rjust(plan_len), ': ', str(op))
out.first_prefix.pop()
out.write(out.bold, ' * ', out.reset, 'debug: end all ops')
out.write()
changes = resolver_inst.state.ops(only_real=True)
build_obs = observer.build_observer(observer.formatter_output(out), not options.debug)
repo_obs = observer.repo_observer(observer.formatter_output(out), not options.debug)
if options.ask or options.pretend:
for op in changes:
formatter.format(op)
formatter.end()
if vdb_time:
out.write(out.bold, 'Took %.2f' % (vdb_time,), out.reset,
' seconds to preload vdb state')
if not changes:
out.write("Nothing to merge.")
return
if options.pretend:
if options.verbose:
out.write(
out.bold, ' * ', out.reset,
"resolver plan required %i ops (%.2f seconds)\n" %
(len(resolver_inst.state.plan), resolve_time))
return
if (options.ask and not formatter.ask("Would you like to merge these packages?")):
return
if options.debug:
out.write(out.bold, " * ", out.reset, "running sanity checks")
start_time = time()
if not changes.run_sanity_checks(domain, build_obs):
out.error("sanity checks failed. please resolve them and try again.")
return 1
if options.debug:
out.write(
out.bold, " * ", out.reset,
"finished sanity checks in %.2f seconds" % (time() - start_time))
out.write()
change_count = len(changes)
# left in place for ease of debugging.
cleanup = []
try:
for count, op in enumerate(changes):
for func in cleanup:
func()
cleanup = []
out.write("\nProcessing %i of %i: %s" % (count + 1, change_count, op.pkg.cpvstr))
out.title("%i/%i: %s" % (count + 1, change_count, op.pkg.cpvstr))
if op.desc != "remove":
cleanup = [op.pkg.release_cached_data]
if not options.fetchonly and options.debug:
out.write("Forcing a clean of workdir")
pkg_ops = domain.pkg_operations(op.pkg, observer=build_obs)
out.write("\n%i files required-" % len(op.pkg.fetchables))
try:
ret = pkg_ops.run_if_supported("fetch", or_return=True)
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
ret = e
if ret is not True:
if ret is False:
ret = None
commandline.dump_error(out, ret, "\nfetching failed for %s" % (op.pkg.cpvstr,))
if not options.ignore_failures:
return 1
continue
if options.fetchonly:
continue
buildop = pkg_ops.run_if_supported("build", or_return=None)
pkg = op.pkg
if buildop is not None:
out.write("building %s" % (op.pkg.cpvstr,))
result = False
try:
result = buildop.finalize()
except format.errors as e:
out.error("caught exception building %s: % s" % (op.pkg.cpvstr, e))
else:
if result is False:
out.error("failed building %s" % (op.pkg.cpvstr,))
if result is False:
if not options.ignore_failures:
return 1
continue
pkg = result
cleanup.append(pkg.release_cached_data)
pkg_ops = domain.pkg_operations(pkg, observer=build_obs)
cleanup.append(buildop.cleanup)
cleanup.append(partial(pkg_ops.run_if_supported, "cleanup"))
pkg = pkg_ops.run_if_supported("localize", or_return=pkg)
# wipe this to ensure we don't inadvertantly use it further down;
# we aren't resetting it after localizing, so could have the wrong
# set of ops.
del pkg_ops
out.write()
if op.desc == "replace":
if op.old_pkg == pkg:
out.write(">>> Reinstalling %s" % (pkg.cpvstr))
else:
out.write(">>> Replacing %s with %s" % (
op.old_pkg.cpvstr, pkg.cpvstr))
i = domain.replace_pkg(op.old_pkg, pkg, repo_obs)
cleanup.append(op.old_pkg.release_cached_data)
else:
out.write(">>> Installing %s" % (pkg.cpvstr,))
i = domain.install_pkg(pkg, repo_obs)
# force this explicitly- can hold onto a helluva lot more
# then we would like.
else:
out.write(">>> Removing %s" % op.pkg.cpvstr)
i = domain.uninstall_pkg(op.pkg, repo_obs)
try:
ret = i.finish()
except merge_errors.BlockModification as e:
out.error("Failed to merge %s: %s" % (op.pkg, e))
if not options.ignore_failures:
return 1
continue
# while this does get handled through each loop, wipe it now; we don't need
# that data, thus we punt it now to keep memory down.
# for safety sake, we let the next pass trigger a release also-
# mainly to protect against any code following triggering reloads
# basically, be protective
if world_set is not None:
if op.desc == "remove":
out.write('>>> Removing %s from world file' % op.pkg.cpvstr)
removal_pkg = slotatom_if_slotted(source_repos.combined, op.pkg.versioned_atom)
update_worldset(world_set, removal_pkg, remove=True)
elif not options.oneshot and any(x.match(op.pkg) for x in atoms):
if not options.upgrade:
out.write('>>> Adding %s to world file' % op.pkg.cpvstr)
add_pkg = slotatom_if_slotted(source_repos.combined, op.pkg.versioned_atom)
update_worldset(world_set, add_pkg)
# again... left in place for ease of debugging.
# except KeyboardInterrupt:
# import pdb;pdb.set_trace()
# else:
# import pdb;pdb.set_trace()
finally:
pass
# the final run from the loop above doesn't invoke cleanups;
# we could ignore it, but better to run it to ensure nothing is
# inadvertantly held on the way out of this function.
# makes heappy analysis easier if we're careful about it.
for func in cleanup:
func()
# and wipe the reference to the functions to allow things to fall out of
# memory.
cleanup = []
out.write("finished")
return 0
def main(options, out, err):
if options.list_sets:
display_pkgsets(out, options)
return 0
config = options.config
if options.debug:
resolver.plan.limiters.add(None)
domain = options.domain
world_set = world_list = options.world
if options.oneshot:
world_set = None
formatter = options.formatter(
out=out,
err=err,
unstable_arch=domain.unstable_arch,
use_expand=domain.profile.use_expand,
use_expand_hidden=domain.profile.use_expand_hidden,
pkg_get_use=domain.get_package_use_unconfigured,
world_list=world_list,
verbosity=options.verbosity,
installed_repos=domain.all_installed_repos,
distdir=domain.fetcher.get_storage_path(),
quiet_repo_display=options.quiet_repo_display)
# This mode does not care about sets and packages so bypass all that.
if options.unmerge:
if not options.oneshot:
if world_set is None:
argparser.error("disable world updating via --oneshot, "
"or fix your configuration")
try:
unmerge(out, err, domain.installed_repos, options.targets, options,
formatter, world_set)
except (parserestrict.ParseError, Failure) as e:
argparser.error(e)
return
source_repos = domain.source_repos
installed_repos = domain.installed_repos
pkg_type = 'ebuilds'
if options.usepkgonly:
source_repos = domain.binary_repos
pkg_type = 'binpkgs'
elif options.usepkg:
# binary repos are checked for matches first before ebuild repos
source_repos = domain.binary_repos + domain.ebuild_repos
pkg_type = 'ebuilds or binpkgs'
elif options.source_only:
source_repos = domain.ebuild_repos
atoms = []
for setname, pkgset in options.sets:
if pkgset is None:
return 1
l = list(pkgset)
if not l:
out.write(
f"skipping set {setname!r}: set is empty, nothing to update")
else:
atoms.extend(l)
for token, restriction in options.targets:
try:
matches = parse_target(restriction,
source_repos.combined,
installed_repos,
return_none=True)
except parserestrict.ParseError as e:
e.token = token
argparser.error(e)
if matches is None:
if not options.ignore_failures:
error_msg = [f"no matching {pkg_type}: {token!r}"]
if token in config.pkgset:
error_msg.append(
f"use '@{token}' instead for the package set")
elif options.usepkgonly:
matches = parse_target(restriction,
domain.ebuild_repos.combined,
installed_repos,
return_none=True)
if matches:
error_msg.append(
"try re-running without -K/--usepkgonly "
"enabled to rebuild from source")
argparser.error(' -- '.join(error_msg))
else:
atoms.extend(matches)
if not atoms and not options.newuse:
err.write(f"{options.prog}: no targets specified; nothing to do")
return 1
atoms = stable_unique(atoms)
if options.clean and not options.oneshot:
if world_set is None:
argparser.error(
"disable world updating via --oneshot, or fix your configuration"
)
extra_kwargs = {}
if options.empty:
extra_kwargs['resolver_cls'] = resolver.empty_tree_merge_plan
if options.debug:
extra_kwargs['debug'] = True
# XXX: This should recurse on deep
if options.newuse:
out.write(out.bold, ' * ', out.reset, 'Scanning for changed USE...')
out.title('Scanning for changed USE...')
for inst_pkg in installed_repos.itermatch(OrRestriction(*atoms)):
src_pkgs = source_repos.match(inst_pkg.versioned_atom)
if src_pkgs:
src_pkg = max(src_pkgs)
inst_iuse = inst_pkg.iuse_stripped
src_iuse = src_pkg.iuse_stripped
inst_flags = inst_iuse.intersection(inst_pkg.use)
src_flags = src_iuse.intersection(src_pkg.use)
if inst_flags.symmetric_difference(src_flags) or \
inst_iuse.symmetric_difference(src_iuse):
atoms.append(src_pkg.unversioned_atom)
excludes = [restriction for token, restriction in options.excludes]
if options.onlydeps:
excludes.extend(atoms)
if excludes:
injected_repo = RestrictionRepo(repo_id='injected',
restrictions=excludes,
frozen=True,
livefs=True)
installed_repos = injected_repo + installed_repos
# left intentionally in place for ease of debugging.
# from guppy import hpy
# hp = hpy()
# hp.setrelheap()
resolver_inst = options.resolver_kls(
vdbs=installed_repos,
dbs=source_repos,
verify_vdb=options.deep,
nodeps=options.nodeps,
drop_cycles=options.ignore_cycles,
force_replace=options.replace,
process_built_depends=options.with_bdeps,
**extra_kwargs)
if options.preload_vdb_state:
out.write(out.bold, ' * ', out.reset, 'Preloading vdb... ')
vdb_time = time()
resolver_inst.load_vdb_state()
vdb_time = time() - vdb_time
else:
vdb_time = 0.0
# flush warning messages before dep resolution begins
out.flush()
err.flush()
failures = []
resolve_time = time()
if sys.stdout.isatty():
out.title('Resolving...')
out.write(out.bold, ' * ', out.reset, 'Resolving...')
out.flush()
ret = resolver_inst.add_atoms(atoms, finalize=True)
while ret:
out.error('resolution failed')
restrict = ret[0][0]
just_failures = reduce_to_failures(ret[1])
display_failures(out, just_failures, debug=options.debug)
failures.append(restrict)
if not options.ignore_failures:
break
out.write("restarting resolution")
atoms = [x for x in atoms if x != restrict]
resolver_inst.reset()
ret = resolver_inst.add_atoms(atoms, finalize=True)
resolve_time = time() - resolve_time
if failures:
out.write()
out.write('Failures encountered:')
for restrict in failures:
out.error(f"failed '{restrict}'")
out.write('potentials:')
match_count = 0
for r in get_raw_repos(source_repos):
l = r.match(restrict)
if l:
out.write(f"repo {r}: [ {', '.join(map(str, l))} ]")
match_count += len(l)
if not match_count:
out.write("No matches found")
if not options.ignore_failures:
return 1
out.write()
resolver_inst.free_caches()
if options.clean:
out.write(out.bold, ' * ', out.reset, 'Packages to be removed:')
vset = set(installed_repos.real.combined)
len_vset = len(vset)
vset.difference_update(x.pkg
for x in resolver_inst.state.iter_ops(True))
wipes = sorted(x for x in vset if x.package_is_real)
for x in wipes:
out.write(f"Remove {x}")
out.write()
if wipes:
out.write("removing %i packages of %i installed, %0.2f%%." %
(len(wipes), len_vset, 100 *
(len(wipes) / float(len_vset))))
else:
out.write("no packages to remove")
if options.pretend:
return 0
if options.ask:
if not formatter.ask("Do you wish to proceed?",
default_answer=False):
return 1
out.write()
repo_obs = observer.repo_observer(observer.formatter_output(out),
debug=options.debug)
do_unmerge(options, out, err, installed_repos.real.combined, wipes,
world_set, repo_obs)
return 0
if options.debug:
out.write()
out.write(out.bold, ' * ', out.reset, 'debug: all ops')
out.first_prefix.append(" ")
plan_len = len(str(len(resolver_inst.state.plan)))
for pos, op in enumerate(resolver_inst.state.plan):
out.write(str(pos + 1).rjust(plan_len), ': ', str(op))
out.first_prefix.pop()
out.write(out.bold, ' * ', out.reset, 'debug: end all ops')
out.write()
changes = resolver_inst.state.ops(only_real=True)
build_obs = observer.phase_observer(observer.formatter_output(out),
debug=options.debug)
repo_obs = observer.repo_observer(observer.formatter_output(out),
debug=options.debug)
# show pkgs to merge in selected format
if (options.ask or options.pretend) and changes:
for op in changes:
formatter.format(op)
formatter.end()
if vdb_time:
out.write(out.bold, 'Took %.2f' % (vdb_time, ), out.reset,
' seconds to preload vdb state')
if changes:
if not options.fetchonly:
# run sanity checks for pkgs -- pkg_pretend, REQUIRED_USE, etc
out.write()
out.write(out.bold, " * ", out.reset, "Running sanity checks...")
if options.debug:
start_time = time()
# flush output so bash spawned errors are shown in the correct order of events
out.flush()
sanity_failures = run_sanity_checks((x.pkg for x in changes),
domain,
threads=1)
if sanity_failures:
for pkg, errors in sanity_failures.items():
out.write(pkg.cpvstr)
out.write('\n'.join(
e.msg(verbosity=options.verbosity) for e in errors))
out.write()
if options.ignore_failures:
out.write(out.fg('red'), out.bold, "!!! ", out.reset,
"Skipping failed sanity checks...")
else:
out.write(out.fg('red'), out.bold, "!!! ", out.reset,
"Sanity checks failed, exiting...")
return 1
else:
out.write()
if options.debug:
out.write(
out.bold, " * ", out.reset,
"finished sanity checks in %.2f seconds" %
(time() - start_time))
out.write()
elif options.verbosity > 0:
# show skipped virtuals
virtual_pkgs = set()
for x in atoms:
matches = installed_repos.virtual.match(x)
if matches:
virtual_pkgs.add(sorted(matches)[-1])
if virtual_pkgs:
out.write("Skipping virtual pkgs:\n%s\n" %
'\n'.join(str(x.versioned_atom) for x in virtual_pkgs))
out.write("Nothing to merge.")
return
if options.pretend:
if options.verbosity > 0:
out.write(
out.bold, ' * ', out.reset,
"resolver plan required %i ops (%.2f seconds)" %
(len(resolver_inst.state.plan), resolve_time))
return
action = 'merge'
if options.fetchonly:
action = 'fetch'
if (options.ask and
not formatter.ask(f"Would you like to {action} these packages?")):
return
change_count = len(changes)
# left in place for ease of debugging.
cleanup = []
try:
for count, op in enumerate(changes):
for func in cleanup:
func()
cleanup = []
out.write(f"\nProcessing {count + 1} of {change_count}: "
f"{op.pkg.cpvstr}::{op.pkg.repo}")
out.title(f"{count + 1}/{change_count}: {op.pkg.cpvstr}")
if op.desc != "remove":
cleanup.append(op.pkg.release_cached_data)
if not options.fetchonly and options.debug:
out.write("Forcing a clean of workdir")
pkg_ops = domain.pkg_operations(op.pkg, observer=build_obs)
out.write(
f"\n{len(op.pkg.distfiles)} file{pluralism(op.pkg.distfiles)} required-"
)
if not pkg_ops.run_if_supported("fetch", or_return=True):
out.error(f"fetching failed for {op.pkg.cpvstr}")
if not options.ignore_failures:
return 1
continue
if options.fetchonly:
continue
buildop = pkg_ops.run_if_supported("build", or_return=None)
pkg = op.pkg
if buildop is not None:
out.write(f"building {op.pkg.cpvstr}")
result = False
exc = None
try:
result = buildop.finalize()
except format.BuildError as e:
out.error(
f"caught exception building {op.pkg.cpvstr}: {e}")
exc = e
else:
if result is False:
out.error(f"failed building {op.pkg.cpvstr}")
if result is False:
if not options.ignore_failures:
raise ExitException(1) from exc
continue
pkg = result
cleanup.append(pkg.release_cached_data)
pkg_ops = domain.pkg_operations(pkg, observer=build_obs)
cleanup.append(buildop.cleanup)
cleanup.append(partial(pkg_ops.run_if_supported, "cleanup"))
pkg = pkg_ops.run_if_supported("localize", or_return=pkg)
# wipe this to ensure we don't inadvertantly use it further down;
# we aren't resetting it after localizing, so could have the wrong
# set of ops.
del pkg_ops
out.write()
if op.desc == "replace":
if op.old_pkg == pkg:
out.write(f">>> Reinstalling {pkg.cpvstr}")
else:
out.write(
f">>> Replacing {op.old_pkg.cpvstr} with {pkg.cpvstr}"
)
i = domain.replace_pkg(op.old_pkg, pkg, repo_obs)
cleanup.append(op.old_pkg.release_cached_data)
else:
out.write(f">>> Installing {pkg.cpvstr}")
i = domain.install_pkg(pkg, repo_obs)
# force this explicitly- can hold onto a helluva lot more
# then we would like.
else:
out.write(f">>> Removing {op.pkg.cpvstr}")
i = domain.uninstall_pkg(op.pkg, repo_obs)
try:
ret = i.finish()
except merge_errors.BlockModification as e:
out.error(f"Failed to merge {op.pkg}: {e}")
if not options.ignore_failures:
return 1
continue
# while this does get handled through each loop, wipe it now; we don't need
# that data, thus we punt it now to keep memory down.
# for safety sake, we let the next pass trigger a release also-
# mainly to protect against any code following triggering reloads
# basically, be protective
if world_set is not None:
if op.desc == "remove":
out.write(f'>>> Removing {op.pkg.cpvstr} from world file')
removal_pkg = slotatom_if_slotted(source_repos.combined,
op.pkg.versioned_atom)
update_worldset(world_set, removal_pkg, remove=True)
elif not options.oneshot and any(
x.match(op.pkg) for x in atoms):
if not (options.upgrade or options.downgrade):
out.write(f'>>> Adding {op.pkg.cpvstr} to world file')
add_pkg = slotatom_if_slotted(source_repos.combined,
op.pkg.versioned_atom)
update_worldset(world_set, add_pkg)
# again... left in place for ease of debugging.
# except KeyboardInterrupt:
# import pdb;pdb.set_trace()
# else:
# import pdb;pdb.set_trace()
finally:
pass
# the final run from the loop above doesn't invoke cleanups;
# we could ignore it, but better to run it to ensure nothing is
# inadvertantly held on the way out of this function.
# makes heappy analysis easier if we're careful about it.
for func in cleanup:
func()
# and wipe the reference to the functions to allow things to fall out of
# memory.
cleanup = []
return 0
def main(options, out, err):
config = options.config
if options.debug:
resolver.plan.limiters.add(None)
domain = options.domain
livefs_repos = domain.all_livefs_repos
world_set = world_list = options.world
if options.oneshot:
world_set = None
formatter = options.formatter(
out=out,
err=err,
unstable_arch=domain.unstable_arch,
domain_settings=domain.settings,
use_expand=domain.profile.use_expand,
use_expand_hidden=domain.profile.use_expand_hidden,
pkg_get_use=domain.get_package_use_unconfigured,
world_list=world_list,
verbose=options.verbose,
livefs_repos=livefs_repos,
distdir=domain.fetcher.get_storage_path(),
quiet_repo_display=options.quiet_repo_display)
# This mode does not care about sets and packages so bypass all that.
if options.unmerge:
if not options.oneshot:
if world_set is None:
err.write(
"Disable world updating via --oneshot, or fix your configuration"
)
return 1
try:
unmerge(out, err, livefs_repos, options.targets, options,
formatter, world_set)
except (parserestrict.ParseError, Failure) as e:
out.error(str(e))
return 1
return
source_repos = domain.source_repositories
installed_repos = domain.installed_repositories
if options.usepkgonly:
source_repos = source_repos.change_repos(
x for x in source_repos
if getattr(x, 'repository_type', None) != 'source')
elif options.usepkg:
repo_types = [(getattr(x, 'repository_type', None) == 'built', x)
for x in source_repos]
source_repos = source_repos.change_repos(
[x[1] for x in repo_types if x[0]] +
[x[1] for x in repo_types if not x[0]])
elif options.source_only:
source_repos = source_repos.change_repos(
x for x in source_repos
if getattr(x, 'repository_type', None) == 'source')
atoms = []
for setname, pkgset in options.set:
if pkgset is None:
return 1
l = list(pkgset)
if not l:
out.write("skipping set %s: set is empty, nothing to update" %
setname)
else:
atoms.extend(l)
for token in options.targets:
try:
a = parse_atom(token,
source_repos.combined,
livefs_repos,
return_none=True)
except parserestrict.ParseError as e:
out.error(str(e))
return 1
if a is None:
if token in config.pkgset:
out.error('No package matches %r, but there is a set with '
'that name. Use -s to specify a set.' % (token, ))
return 2
elif not options.ignore_failures:
out.error('No matches for %r; ignoring it' % token)
else:
return -1
else:
atoms.append(a)
if not atoms and not options.newuse:
out.error('No targets specified; nothing to do')
return 1
atoms = stable_unique(atoms)
if (not options.set or options.clean) and not options.oneshot:
if world_set is None:
err.write(
"Disable world updating via --oneshot, or fix your configuration"
)
return 1
if options.upgrade:
resolver_kls = resolver.upgrade_resolver
else:
resolver_kls = resolver.min_install_resolver
extra_kwargs = {}
if options.empty:
extra_kwargs['resolver_cls'] = resolver.empty_tree_merge_plan
if options.debug:
extra_kwargs['debug'] = True
# XXX: This should recurse on deep
if options.newuse:
out.write(out.bold, ' * ', out.reset, 'Scanning for changed USE...')
out.title('Scanning for changed USE...')
for inst_pkg in installed_repos.itermatch(OrRestriction(*atoms)):
src_pkgs = source_repos.match(inst_pkg.versioned_atom)
if src_pkgs:
src_pkg = max(src_pkgs)
inst_iuse = set(use.lstrip("+-") for use in inst_pkg.iuse)
src_iuse = set(use.lstrip("+-") for use in src_pkg.iuse)
inst_flags = inst_iuse.intersection(inst_pkg.use)
src_flags = src_iuse.intersection(src_pkg.use)
if inst_flags.symmetric_difference(src_flags) or \
inst_pkg.iuse.symmetric_difference(src_pkg.iuse):
atoms.append(src_pkg.unversioned_atom)
# left intentionally in place for ease of debugging.
# from guppy import hpy
# hp = hpy()
# hp.setrelheap()
resolver_inst = resolver_kls(
installed_repos.repositories,
source_repos.repositories,
verify_vdb=options.deep,
nodeps=options.nodeps,
drop_cycles=options.ignore_cycles,
force_replace=options.replace,
process_built_depends=options.with_built_depends,
**extra_kwargs)
if options.preload_vdb_state:
out.write(out.bold, ' * ', out.reset, 'Preloading vdb... ')
vdb_time = time()
resolver_inst.load_vdb_state()
vdb_time = time() - vdb_time
else:
vdb_time = 0.0
failures = []
resolve_time = time()
out.title('Resolving...')
out.write(out.bold, ' * ', out.reset, 'Resolving...')
ret = resolver_inst.add_atoms(atoms, finalize=True)
while ret:
out.error('resolution failed')
restrict = ret[0][0]
just_failures = reduce_to_failures(ret[1])
display_failures(out, just_failures, debug=options.debug)
failures.append(restrict)
if not options.ignore_failures:
break
out.write("restarting resolution")
atoms = [x for x in atoms if x != restrict]
resolver_inst.reset()
ret = resolver_inst.add_atoms(atoms, finalize=True)
resolve_time = time() - resolve_time
if options.debug:
out.write(out.bold, " * ", out.reset,
"resolution took %.2f seconds" % resolve_time)
if failures:
out.write()
out.write('Failures encountered:')
for restrict in failures:
out.error("failed '%s'" % (restrict, ))
out.write('potentials:')
match_count = 0
for r in repo_utils.get_raw_repos(source_repos.repositories):
l = r.match(restrict)
if l:
out.write("repo %s: [ %s ]" %
(r, ", ".join(str(x) for x in l)))
match_count += len(l)
if not match_count:
out.write("No matches found in %s" %
(source_repos.repositories, ))
out.write()
if not options.ignore_failures:
return 1
resolver_inst.free_caches()
if options.clean:
out.write(out.bold, ' * ', out.reset, 'Packages to be removed:')
vset = set(installed_repos.combined)
len_vset = len(vset)
vset.difference_update(x.pkg
for x in resolver_inst.state.iter_ops(True))
wipes = sorted(x for x in vset if x.package_is_real)
for x in wipes:
out.write("Remove %s" % x)
out.write()
if wipes:
out.write("removing %i packages of %i installed, %0.2f%%." %
(len(wipes), len_vset, 100 *
(len(wipes) / float(len_vset))))
else:
out.write("no packages to remove")
if options.pretend:
return 0
if options.ask:
if not formatter.ask("Do you wish to proceed?",
default_answer=False):
return 1
out.write()
repo_obs = observer.repo_observer(observer.formatter_output(out),
not options.debug)
do_unmerge(options, out, err, installed_repos.combined, wipes,
world_set, repo_obs)
return 0
if options.debug:
out.write()
out.write(out.bold, ' * ', out.reset, 'debug: all ops')
out.first_prefix.append(" ")
plan_len = len(str(len(resolver_inst.state.plan)))
for pos, op in enumerate(resolver_inst.state.plan):
out.write(str(pos + 1).rjust(plan_len), ': ', str(op))
out.first_prefix.pop()
out.write(out.bold, ' * ', out.reset, 'debug: end all ops')
out.write()
changes = resolver_inst.state.ops(only_real=True)
build_obs = observer.build_observer(observer.formatter_output(out),
not options.debug)
repo_obs = observer.repo_observer(observer.formatter_output(out),
not options.debug)
if options.debug:
out.write(out.bold, " * ", out.reset, "running sanity checks")
start_time = time()
if not changes.run_sanity_checks(domain, build_obs):
out.error("sanity checks failed. please resolve them and try again.")
return 1
if options.debug:
out.write(
out.bold, " * ", out.reset,
"finished sanity checks in %.2f seconds" % (time() - start_time))
out.write()
if options.ask or options.pretend:
for op in changes:
formatter.format(op)
formatter.end()
if vdb_time:
out.write(out.bold, 'Took %.2f' % (vdb_time, ), out.reset,
' seconds to preload vdb state')
if not changes:
out.write("Nothing to merge.")
return
if options.pretend:
if options.verbose:
out.write(
out.bold, ' * ', out.reset,
"resolver plan required %i ops (%.2f seconds)\n" %
(len(resolver_inst.state.plan), resolve_time))
return
if (options.ask
and not formatter.ask("Would you like to merge these packages?")):
return
change_count = len(changes)
# left in place for ease of debugging.
cleanup = []
try:
for count, op in enumerate(changes):
for func in cleanup:
func()
cleanup = []
out.write("\nProcessing %i of %i: %s" %
(count + 1, change_count, op.pkg.cpvstr))
out.title("%i/%i: %s" % (count + 1, change_count, op.pkg.cpvstr))
if op.desc != "remove":
cleanup = [op.pkg.release_cached_data]
if not options.fetchonly and options.debug:
out.write("Forcing a clean of workdir")
pkg_ops = domain.pkg_operations(op.pkg, observer=build_obs)
out.write("\n%i files required-" % len(op.pkg.fetchables))
try:
ret = pkg_ops.run_if_supported("fetch", or_return=True)
except IGNORED_EXCEPTIONS:
raise
except Exception as e:
ret = e
if ret is not True:
if ret is False:
ret = None
commandline.dump_error(
out, ret,
"\nfetching failed for %s" % (op.pkg.cpvstr, ))
if not options.ignore_failures:
return 1
continue
if options.fetchonly:
continue
buildop = pkg_ops.run_if_supported("build", or_return=None)
pkg = op.pkg
if buildop is not None:
out.write("building %s" % (op.pkg.cpvstr, ))
result = False
try:
result = buildop.finalize()
except format.errors as e:
out.error("caught exception building %s: % s" %
(op.pkg.cpvstr, e))
else:
if result is False:
out.error("failed building %s" % (op.pkg.cpvstr, ))
if result is False:
if not options.ignore_failures:
return 1
continue
pkg = result
cleanup.append(pkg.release_cached_data)
pkg_ops = domain.pkg_operations(pkg, observer=build_obs)
cleanup.append(buildop.cleanup)
cleanup.append(partial(pkg_ops.run_if_supported, "cleanup"))
pkg = pkg_ops.run_if_supported("localize", or_return=pkg)
# wipe this to ensure we don't inadvertantly use it further down;
# we aren't resetting it after localizing, so could have the wrong
# set of ops.
del pkg_ops
out.write()
if op.desc == "replace":
if op.old_pkg == pkg:
out.write(">>> Reinstalling %s" % (pkg.cpvstr))
else:
out.write(">>> Replacing %s with %s" %
(op.old_pkg.cpvstr, pkg.cpvstr))
i = domain.replace_pkg(op.old_pkg, pkg, repo_obs)
cleanup.append(op.old_pkg.release_cached_data)
else:
out.write(">>> Installing %s" % (pkg.cpvstr, ))
i = domain.install_pkg(pkg, repo_obs)
# force this explicitly- can hold onto a helluva lot more
# then we would like.
else:
out.write(">>> Removing %s" % op.pkg.cpvstr)
i = domain.uninstall_pkg(op.pkg, repo_obs)
try:
ret = i.finish()
except merge_errors.BlockModification as e:
out.error("Failed to merge %s: %s" % (op.pkg, e))
if not options.ignore_failures:
return 1
continue
# while this does get handled through each loop, wipe it now; we don't need
# that data, thus we punt it now to keep memory down.
# for safety sake, we let the next pass trigger a release also-
# mainly to protect against any code following triggering reloads
# basically, be protective
if world_set is not None:
if op.desc == "remove":
out.write('>>> Removing %s from world file' %
op.pkg.cpvstr)
removal_pkg = slotatom_if_slotted(source_repos.combined,
op.pkg.versioned_atom)
update_worldset(world_set, removal_pkg, remove=True)
elif not options.oneshot and any(
x.match(op.pkg) for x in atoms):
if not options.upgrade:
out.write('>>> Adding %s to world file' %
op.pkg.cpvstr)
add_pkg = slotatom_if_slotted(source_repos.combined,
op.pkg.versioned_atom)
update_worldset(world_set, add_pkg)
# again... left in place for ease of debugging.
# except KeyboardInterrupt:
# import pdb;pdb.set_trace()
# else:
# import pdb;pdb.set_trace()
finally:
pass
# the final run from the loop above doesn't invoke cleanups;
# we could ignore it, but better to run it to ensure nothing is inadvertantly
# held on the way out of this function.
# makes heappy analysis easier if we're careful about it.
for func in cleanup:
func()
# and wipe the reference to the functions to allow things to fall out of
# memory.
cleanup = []
out.write("finished")
return 0
out.write(out.bold, ' * ', out.reset, 'Preloading vdb... ')
vdb_time = time()
resolver_inst.load_vdb_state()
vdb_time = time() - vdb_time
else:
vdb_time = 0.0
failures = []
resolve_time = time()
out.title('Resolving...')
out.write(out.bold, ' * ', out.reset, 'Resolving...')
ret = resolver_inst.add_atoms(atoms, finalize=True)
while ret:
out.error('resolution failed')
restrict = ret[0][0]
just_failures = reduce_to_failures(ret[1])
display_failures(out, just_failures, debug=options.debug)
failures.append(restrict)
if not options.ignore_failures:
break
out.write("restarting resolution")
atoms = [x for x in atoms if x != restrict]
resolver_inst.reset()
ret = resolver_inst.add_atoms(atoms, finalize=True)
resolve_time = time() - resolve_time
if options.debug:
out.write(out.bold, " * ", out.reset, "resolution took %.2f seconds" % resolve_time)
if failures:
out.write()