def calcdep(config):
settings = config.target_config.settings
trees = config.trees
opts = config.opts
action = config.action
params = create_depgraph_params(opts, action)
files = config.args
spinner = stdout_spinner()
# spinner.update = spinner.update_quiet
return _backtrack_depgraph(settings, trees, opts, params, action, files,
spinner)
def _run_builder(self, dirs_cleanup_queue):
"""
This method is called by _run and executes the whole package build
logic, including constraints validation given by argv parameters.
NOTE: negative errors indicate warnings that can be skipped.
"""
if self._packages:
first_package = self._packages[0]
else:
first_package = "_empty_"
log_dir = mkdtemp(prefix="matter_build.",
suffix="." + first_package.replace("/", "_").lstrip("<>=~"))
dirs_cleanup_queue.append(log_dir)
emerge_settings, emerge_trees, mtimedb = self._emerge_config
# reset settings to original state, variables will be reconfigured
# while others may remain saved due to backup_changes().
emerge_settings.unlock()
emerge_settings.reset()
emerge_settings.lock()
# Setup stable/unstable keywords, must be done on
# emerge_settings bacause the reference is spread everywhere
# in emerge_trees.
# This is not thread-safe, but Portage isn't either, so
# who cares!
# ACCEPT_KEYWORDS is not saved and reset every time by the
# reset() call above.
portdb = emerge_trees[emerge_settings["ROOT"]]["porttree"].dbapi
self._setup_keywords(portdb, emerge_settings)
portdb.freeze()
vardb = emerge_trees[emerge_settings["ROOT"]]["vartree"].dbapi
vardb.settings.unlock()
vardb.settings["PORT_LOGDIR"] = log_dir
vardb.settings.backup_changes("PORT_LOGDIR")
vardb.settings.lock()
# Load the most current variables from /etc/profile.env, which
# has been re-generated by the env-update call in _run()
emerge_settings.unlock()
emerge_settings.reload()
emerge_settings.regenerate()
emerge_settings.lock()
sets = self._get_sets_mod() # can be None
sets_conf = None
if sets is not None:
sets_conf = sets.load_default_config(
emerge_settings,
emerge_trees[emerge_settings["ROOT"]])
packages = []
# execute basic, pre-graph generation filters against each
# package dependency in self._packages.
# This is just fast pruning of obvious obviousness.
for package in self._packages:
expanded_pkgs = []
# package sets support
if package.startswith("@") and sets_conf:
try:
set_pkgs = sets_conf.getSetAtoms(package[1:])
expanded_pkgs.extend(sorted(set_pkgs))
except sets.PackageSetNotFound:
# make it fail, add set directly
expanded_pkgs.append(package)
else:
expanded_pkgs.append(package)
for exp_pkg in expanded_pkgs:
accepted = self._pre_graph_filters(
exp_pkg, portdb, vardb)
for best_visible in accepted:
packages.append((exp_pkg, best_visible))
if not packages:
print_warning("No remaining packages in queue, aborting.")
return 0
# at this point we can go ahead building packages
print_info("starting to build:")
for package, best_visible in packages:
print_info(": %s -> %s" % (
package, best_visible,))
if not getcolor():
portage.output.nocolor()
# non interactive properties, this is not really required
# accept-properties just sets os.environ...
build_args = list(self._setup_build_args(self._params))
build_args += ["=" + best_v for _x, best_v in packages]
myaction, myopts, myfiles = parse_opts(build_args)
adjust_configs(myopts, emerge_trees)
apply_priorities(emerge_settings)
spinner = stdout_spinner()
if "--quiet" in myopts:
spinner.update = spinner.update_basic
elif "--nospinner" in myopts:
spinner.update = spinner.update_basic
if emerge_settings.get("TERM") == "dumb" or not is_stdout_a_tty():
spinner.update = spinner.update_basic
print_info("emerge args: %s" % (" ".join(build_args),))
params = create_depgraph_params(myopts, myaction)
success, graph, favorites = backtrack_depgraph(emerge_settings,
emerge_trees, myopts, params, myaction, myfiles, spinner)
if not success:
# print issues to stdout and give up
print_warning("dependencies calculation failed, aborting")
graph.display_problems()
# try to collect some info about the failure
bt_config = (graph.get_backtrack_infos() or {}).get("config", {})
for k, v in bt_config.items():
if k == "needed_use_config_changes":
for tup in v:
try:
pkg, (new_use, new_changes) = tup
except (ValueError, TypeError):
print_error(
"unsupported needed_use_config_changes: %s" % (
tup,))
continue
obj = self._missing_use_packages.setdefault(
"%s" % (pkg.cpv,), {})
obj["cp:slot"] = "%s" % (pkg.slot_atom,)
changes = obj.setdefault("changes", {})
changes.update(copy.deepcopy(new_changes))
elif k == "needed_unstable_keywords":
for pkg in v:
self._needed_unstable_keywords.add("%s" % (pkg.cpv,))
elif k == "needed_p_mask_changes":
for pkg in v:
self._needed_package_mask_changes.add(
"%s" % (pkg.cpv,))
elif k == "needed_license_changes":
for pkg, lics in v:
obj = self._needed_license_changes.setdefault(
"%s" % (pkg.cpv,), set())
obj.update(lics)
else:
print_warning("unsupported backtrack info: %s -> %s" % (
k, v,))
return 0
print_info("dependency graph generated successfully")
real_queue = self._post_graph_filters(graph, vardb, portdb)
if real_queue is None:
# post-graph filters not passed, giving up
return 0
merge_queue = [x for x in real_queue if x.operation == "merge"]
unmerge_queue = [x for x in real_queue if x.operation == "uninstall"]
if merge_queue:
print_info("about to build the following packages:")
for pkg in merge_queue:
print_info(" %s" % (pkg.cpv,))
if unmerge_queue:
print_info("about to uninstall the following packages:")
for pkg in unmerge_queue:
print_info(" %s" % (pkg.cpv,))
if self._pretend:
print_info("portage spawned with --pretend, done!")
return 0
# re-calling action_build(), deps are re-calculated though
validate_ebuild_environment(emerge_trees)
mergetask = Scheduler(emerge_settings, emerge_trees, mtimedb,
myopts, spinner, favorites=favorites,
graph_config=graph.schedulerGraph())
del graph
self.clear_caches(self._emerge_config)
retval = mergetask.merge()
not_merged = []
real_queue_map = dict((pkg.cpv, pkg) for pkg in real_queue)
failed_package = None
if retval != 0:
merge_list = mtimedb.get("resume", {}).get("mergelist")
for _merge_type, _merge_root, merge_atom, _merge_act in merge_list:
merge_atom = "%s" % (merge_atom,)
if failed_package is None:
# we consider the first encountered package the one
# that failed. It makes sense since packages are built
# serially as of today.
# Also, the package object must be available in our
# package queue, so grab it from there.
failed_package = real_queue_map.get(merge_atom)
not_merged.append(merge_atom)
self._not_merged_packages.append(merge_atom)
for pkg in real_queue:
cpv = pkg.cpv
if not cpv:
print_warning("package: %s, has broken cpv: '%s', ignoring" % (
pkg, cpv,))
elif cpv not in not_merged:
if pkg.operation == "merge":
# add to build queue
print_info("package: %s, successfully built" % (cpv,))
self._built_packages.append("%s" % (cpv,))
else:
# add to uninstall queue
print_info("package: %s, successfully uninstalled" % (cpv,))
self._uninstalled_packages.append("%s" % (cpv,))
post_emerge(myaction, myopts, myfiles, emerge_settings["ROOT"],
emerge_trees, mtimedb, retval)
subprocess.call(["env-update"])
if failed_package is not None:
print_warning("failed package: %s::%s" % (failed_package.cpv,
failed_package.repo,))
if self._params["buildfail"] and (failed_package is not None):
std_env = self._build_standard_environment(
repository=self._params["repository"])
std_env["MATTER_PACKAGE_NAMES"] = " ".join(self._packages)
std_env["MATTER_PORTAGE_FAILED_PACKAGE_NAME"] = failed_package.cpv
std_env["MATTER_PORTAGE_REPOSITORY"] = failed_package.repo
# call pkgfail hook if defined
std_env["MATTER_PORTAGE_BUILD_LOG_DIR"] = os.path.join(log_dir,
"build")
buildfail = self._params["buildfail"]
print_info("spawning buildfail: %s" % (buildfail,))
tmp_fd, tmp_path = mkstemp()
with os.fdopen(tmp_fd, "wb") as tmp_f:
with open(buildfail, "rb") as buildfail_f:
tmp_f.write(buildfail_f.read())
try:
# now execute
os.chmod(tmp_path, 0o700)
exit_st = subprocess.call([tmp_path], env = std_env)
if exit_st != 0:
return exit_st
finally:
os.remove(tmp_path)
print_info("portage spawned, return value: %d" % (retval,))
return retval
def emerge_main():
global portage # NFC why this is necessary now - genone
portage._disable_legacy_globals()
# Disable color until we're sure that it should be enabled (after
# EMERGE_DEFAULT_OPTS has been parsed).
portage.output.havecolor = 0
# This first pass is just for options that need to be known as early as
# possible, such as --config-root. They will be parsed again later,
# together with EMERGE_DEFAULT_OPTS (which may vary depending on the
# the value of --config-root).
myaction, myopts, myfiles = parse_opts(sys.argv[1:], silent=True)
if "--debug" in myopts:
os.environ["PORTAGE_DEBUG"] = "1"
if "--config-root" in myopts:
os.environ["PORTAGE_CONFIGROOT"] = myopts["--config-root"]
if "--root" in myopts:
os.environ["ROOT"] = myopts["--root"]
if "--accept-properties" in myopts:
os.environ["ACCEPT_PROPERTIES"] = myopts["--accept-properties"]
# Portage needs to ensure a sane umask for the files it creates.
os.umask(0o22)
settings, trees, mtimedb = load_emerge_config()
portdb = trees[settings["ROOT"]]["porttree"].dbapi
rval = profile_check(trees, myaction)
if rval != os.EX_OK:
return rval
if myaction not in ('help', 'info', 'version') and \
_global_updates(trees, mtimedb["updates"]):
mtimedb.commit()
# Reload the whole config from scratch.
settings, trees, mtimedb = load_emerge_config(trees=trees)
portdb = trees[settings["ROOT"]]["porttree"].dbapi
xterm_titles = "notitles" not in settings.features
if xterm_titles:
xtermTitle("emerge")
tmpcmdline = []
if "--ignore-default-opts" not in myopts:
tmpcmdline.extend(settings["EMERGE_DEFAULT_OPTS"].split())
tmpcmdline.extend(sys.argv[1:])
myaction, myopts, myfiles = parse_opts(tmpcmdline)
if "--digest" in myopts:
os.environ["FEATURES"] = os.environ.get("FEATURES","") + " digest"
# Reload the whole config from scratch so that the portdbapi internal
# config is updated with new FEATURES.
settings, trees, mtimedb = load_emerge_config(trees=trees)
portdb = trees[settings["ROOT"]]["porttree"].dbapi
adjust_configs(myopts, trees)
apply_priorities(settings)
if myaction == 'version':
writemsg_stdout(getportageversion(
settings["PORTDIR"], settings["ROOT"],
settings.profile_path, settings["CHOST"],
trees[settings["ROOT"]]["vartree"].dbapi) + '\n', noiselevel=-1)
return 0
elif myaction == 'help':
_emerge.help.help(myopts, portage.output.havecolor)
return 0
spinner = stdout_spinner()
if "candy" in settings.features:
spinner.update = spinner.update_scroll
if "--quiet" not in myopts:
portage.deprecated_profile_check(settings=settings)
repo_name_check(trees)
repo_name_duplicate_check(trees)
config_protect_check(trees)
check_procfs()
if "getbinpkg" in settings.features:
myopts["--getbinpkg"] = True
if "--getbinpkgonly" in myopts:
myopts["--getbinpkg"] = True
if "--getbinpkgonly" in myopts:
myopts["--usepkgonly"] = True
if "--getbinpkg" in myopts:
myopts["--usepkg"] = True
if "--usepkgonly" in myopts:
myopts["--usepkg"] = True
if "buildpkg" in settings.features or "--buildpkgonly" in myopts:
myopts["--buildpkg"] = True
if "--buildpkgonly" in myopts:
# --buildpkgonly will not merge anything, so
# it cancels all binary package options.
for opt in ("--getbinpkg", "--getbinpkgonly",
"--usepkg", "--usepkgonly"):
myopts.pop(opt, None)
for mytrees in trees.values():
mydb = mytrees["porttree"].dbapi
# Freeze the portdbapi for performance (memoize all xmatch results).
mydb.freeze()
if "--usepkg" in myopts:
# Populate the bintree with current --getbinpkg setting.
# This needs to happen before expand_set_arguments(), in case
# any sets use the bintree.
mytrees["bintree"].populate(
getbinpkgs="--getbinpkg" in myopts)
del mytrees, mydb
if "moo" in myfiles:
print("""
Larry loves Gentoo (""" + platform.system() + """)
_______________________
< Have you mooed today? >
-----------------------
\ ^__^
\ (oo)\_______
(__)\ )\/\
||----w |
|| ||
""")
for x in myfiles:
ext = os.path.splitext(x)[1]
if (ext == ".ebuild" or ext == ".tbz2") and os.path.exists(os.path.abspath(x)):
print(colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n"))
break
root_config = trees[settings["ROOT"]]["root_config"]
if myaction == "list-sets":
writemsg_stdout("".join("%s\n" % s for s in sorted(root_config.sets)))
return os.EX_OK
ensure_required_sets(trees)
# only expand sets for actions taking package arguments
oldargs = myfiles[:]
if myaction in ("clean", "config", "depclean", "info", "prune", "unmerge", None):
myfiles, retval = expand_set_arguments(myfiles, myaction, root_config)
if retval != os.EX_OK:
return retval
# Need to handle empty sets specially, otherwise emerge will react
# with the help message for empty argument lists
if oldargs and not myfiles:
print("emerge: no targets left after set expansion")
return 0
if ("--tree" in myopts) and ("--columns" in myopts):
print("emerge: can't specify both of \"--tree\" and \"--columns\".")
return 1
if '--emptytree' in myopts and '--noreplace' in myopts:
writemsg_level("emerge: can't specify both of " + \
"\"--emptytree\" and \"--noreplace\".\n",
level=logging.ERROR, noiselevel=-1)
return 1
if ("--quiet" in myopts):
spinner.update = spinner.update_quiet
portage.util.noiselimit = -1
if "--fetch-all-uri" in myopts:
myopts["--fetchonly"] = True
if "--skipfirst" in myopts and "--resume" not in myopts:
myopts["--resume"] = True
# Allow -p to remove --ask
if "--pretend" in myopts:
myopts.pop("--ask", None)
# forbid --ask when not in a terminal
# note: this breaks `emerge --ask | tee logfile`, but that doesn't work anyway.
if ("--ask" in myopts) and (not sys.stdin.isatty()):
portage.writemsg("!!! \"--ask\" should only be used in a terminal. Exiting.\n",
noiselevel=-1)
return 1
if settings.get("PORTAGE_DEBUG", "") == "1":
spinner.update = spinner.update_quiet
portage.debug=1
if "python-trace" in settings.features:
import portage.debug
portage.debug.set_trace(True)
if not ("--quiet" in myopts):
if '--nospinner' in myopts or \
settings.get('TERM') == 'dumb' or \
not sys.stdout.isatty():
spinner.update = spinner.update_basic
if "--debug" in myopts:
print("myaction", myaction)
print("myopts", myopts)
if not myaction and not myfiles and "--resume" not in myopts:
_emerge.help.help(myopts, portage.output.havecolor)
return 1
pretend = "--pretend" in myopts
fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts
buildpkgonly = "--buildpkgonly" in myopts
# check if root user is the current user for the actions where emerge needs this
if portage.secpass < 2:
# We've already allowed "--version" and "--help" above.
if "--pretend" not in myopts and myaction not in ("search","info"):
need_superuser = myaction in ('clean', 'depclean', 'deselect',
'prune', 'unmerge') or not \
(fetchonly or \
(buildpkgonly and secpass >= 1) or \
myaction in ("metadata", "regen") or \
(myaction == "sync" and os.access(settings["PORTDIR"], os.W_OK)))
if portage.secpass < 1 or \
need_superuser:
if need_superuser:
access_desc = "superuser"
else:
access_desc = "portage group"
# Always show portage_group_warning() when only portage group
# access is required but the user is not in the portage group.
from portage.data import portage_group_warning
if "--ask" in myopts:
myopts["--pretend"] = True
del myopts["--ask"]
print(("%s access is required... " + \
"adding --pretend to options\n") % access_desc)
if portage.secpass < 1 and not need_superuser:
portage_group_warning()
else:
sys.stderr.write(("emerge: %s access is required\n") \
% access_desc)
if portage.secpass < 1 and not need_superuser:
portage_group_warning()
return 1
disable_emergelog = False
for x in ("--pretend", "--fetchonly", "--fetch-all-uri"):
if x in myopts:
disable_emergelog = True
break
if myaction in ("search", "info"):
disable_emergelog = True
if disable_emergelog:
""" Disable emergelog for everything except build or unmerge
operations. This helps minimize parallel emerge.log entries that can
confuse log parsers. We especially want it disabled during
parallel-fetch, which uses --resume --fetchonly."""
_emerge.emergelog._disable = True
else:
if 'EMERGE_LOG_DIR' in settings:
try:
# At least the parent needs to exist for the lock file.
portage.util.ensure_dirs(settings['EMERGE_LOG_DIR'])
except portage.exception.PortageException as e:
writemsg_level("!!! Error creating directory for " + \
"EMERGE_LOG_DIR='%s':\n!!! %s\n" % \
(settings['EMERGE_LOG_DIR'], e),
noiselevel=-1, level=logging.ERROR)
else:
global _emerge_log_dir
_emerge_log_dir = settings['EMERGE_LOG_DIR']
if not "--pretend" in myopts:
emergelog(xterm_titles, "Started emerge on: "+\
_unicode_decode(
time.strftime("%b %d, %Y %H:%M:%S", time.localtime()),
encoding=_encodings['content'], errors='replace'))
myelogstr=""
if myopts:
myelogstr=" ".join(myopts)
if myaction:
myelogstr+=" "+myaction
if myfiles:
myelogstr += " " + " ".join(oldargs)
emergelog(xterm_titles, " *** emerge " + myelogstr)
del oldargs
def emergeexitsig(signum, frame):
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
portage.util.writemsg("\n\nExiting on signal %(signal)s\n" % {"signal":signum})
sys.exit(100+signum)
signal.signal(signal.SIGINT, emergeexitsig)
signal.signal(signal.SIGTERM, emergeexitsig)
def emergeexit():
"""This gets out final log message in before we quit."""
if "--pretend" not in myopts:
emergelog(xterm_titles, " *** terminating.")
if xterm_titles:
xtermTitleReset()
portage.atexit_register(emergeexit)
if myaction in ("config", "metadata", "regen", "sync"):
if "--pretend" in myopts:
sys.stderr.write(("emerge: The '%s' action does " + \
"not support '--pretend'.\n") % myaction)
return 1
if "sync" == myaction:
return action_sync(settings, trees, mtimedb, myopts, myaction)
elif "metadata" == myaction:
action_metadata(settings, portdb, myopts)
elif myaction=="regen":
validate_ebuild_environment(trees)
return action_regen(settings, portdb, myopts.get("--jobs"),
myopts.get("--load-average"))
# HELP action
elif "config"==myaction:
validate_ebuild_environment(trees)
action_config(settings, trees, myopts, myfiles)
# SEARCH action
elif "search"==myaction:
validate_ebuild_environment(trees)
action_search(trees[settings["ROOT"]]["root_config"],
myopts, myfiles, spinner)
elif myaction in ('clean', 'depclean', 'deselect', 'prune', 'unmerge'):
validate_ebuild_environment(trees)
rval = action_uninstall(settings, trees, mtimedb["ldpath"],
myopts, myaction, myfiles, spinner)
if not (myaction == 'deselect' or buildpkgonly or fetchonly or pretend):
post_emerge(root_config, myopts, mtimedb, rval)
return rval
elif myaction == 'info':
# Ensure atoms are valid before calling unmerge().
vardb = trees[settings["ROOT"]]["vartree"].dbapi
portdb = trees[settings["ROOT"]]["porttree"].dbapi
bindb = trees[settings["ROOT"]]["bintree"].dbapi
valid_atoms = []
for x in myfiles:
if is_valid_package_atom(x):
try:
#look at the installed files first, if there is no match
#look at the ebuilds, since EAPI 4 allows running pkg_info
#on non-installed packages
valid_atom = dep_expand(x, mydb=vardb, settings=settings)
if valid_atom.cp.split("/")[0] == "null":
valid_atom = dep_expand(x, mydb=portdb, settings=settings)
if valid_atom.cp.split("/")[0] == "null" and "--usepkg" in myopts:
valid_atom = dep_expand(x, mydb=bindb, settings=settings)
valid_atoms.append(valid_atom)
except portage.exception.AmbiguousPackageName as e:
msg = "The short ebuild name \"" + x + \
"\" is ambiguous. Please specify " + \
"one of the following " + \
"fully-qualified ebuild names instead:"
for line in textwrap.wrap(msg, 70):
writemsg_level("!!! %s\n" % (line,),
level=logging.ERROR, noiselevel=-1)
for i in e[0]:
writemsg_level(" %s\n" % colorize("INFORM", i),
level=logging.ERROR, noiselevel=-1)
writemsg_level("\n", level=logging.ERROR, noiselevel=-1)
return 1
continue
msg = []
msg.append("'%s' is not a valid package atom." % (x,))
msg.append("Please check ebuild(5) for full details.")
writemsg_level("".join("!!! %s\n" % line for line in msg),
level=logging.ERROR, noiselevel=-1)
return 1
return action_info(settings, trees, myopts, valid_atoms)
# "update", "system", or just process files:
else:
validate_ebuild_environment(trees)
for x in myfiles:
if x.startswith(SETPREFIX) or \
is_valid_package_atom(x):
continue
if x[:1] == os.sep:
continue
try:
os.lstat(x)
continue
except OSError:
pass
msg = []
msg.append("'%s' is not a valid package atom." % (x,))
msg.append("Please check ebuild(5) for full details.")
writemsg_level("".join("!!! %s\n" % line for line in msg),
level=logging.ERROR, noiselevel=-1)
return 1
if "--pretend" not in myopts:
display_news_notification(root_config, myopts)
retval = action_build(settings, trees, mtimedb,
myopts, myaction, myfiles, spinner)
root_config = trees[settings["ROOT"]]["root_config"]
post_emerge(root_config, myopts, mtimedb, retval)
return retval
def GenDependencyTree(self):
"""Get dependency tree info from emerge.
Returns:
A 3-tuple of the dependency tree, the dependency tree info, and the
BDEPEND dependency tree.
The BDEPEND dependency tree, or "bdeps_tree", consists of the packages
that would be installed to the SDK BROOT (at "/") rather than the board
root at "/build/$BOARD". If the ROOT is already equal to "/" the contents
of deps_tree and bdeps_tree will be identical.
"""
start = time.time()
emerge = self.emerge
# Create a list of packages to merge
packages = set(emerge.cmdline_packages[:])
# Tell emerge to be quiet. We print plenty of info ourselves so we don't
# need any extra output from portage.
portage.util.noiselimit = -1
# My favorite feature: The silent spinner. It doesn't spin. Ever.
# I'd disable the colors by default too, but they look kind of cool.
emerge.spinner = stdout_spinner()
emerge.spinner.update = emerge.spinner.update_quiet
if '--quiet' not in emerge.opts:
print('Calculating deps...')
with cros_event.newEvent(task_name='GenerateDepTree'):
self.CreateDepgraph(emerge, packages)
depgraph = emerge.depgraph
# Build our own tree from the emerge digraph.
deps_tree = {}
bdeps_tree = {}
# pylint: disable=protected-access
digraph = depgraph._dynamic_config.digraph
root = emerge.settings['ROOT']
final_db = depgraph._dynamic_config._filtered_trees[root]['graph_db']
for node, node_deps in digraph.nodes.items():
# Calculate dependency packages that need to be installed first. Each
# child on the digraph is a dependency. The "operation" field specifies
# what we're doing (e.g. merge, uninstall, etc.). The "priorities" array
# contains the type of dependency (e.g. build, runtime, runtime_post,
# etc.)
#
# Portage refers to the identifiers for packages as a CPV. This acronym
# stands for Component/Path/Version.
#
# Here's an example CPV: chromeos-base/power_manager-0.0.1-r1
# Split up, this CPV would be:
# C -- Component: chromeos-base
# P -- Path: power_manager
# V -- Version: 0.0.1-r1
#
# We just refer to CPVs as packages here because it's easier.
# Clear the entry for 'child' that persisted from the previous loop
# iteration. In Python 2 loop variables continue to hold their value
# in the enclosing function scope (outside the loop body), so if
# node_deps[0] is empty then 'child' would have whatever value was
# left in it from the previous loop.
child = None
deps = {}
for child, priorities in node_deps[0].items():
if (isinstance(node, Package) and isinstance(child, Package)
and (self.include_bdepend or child.root == node.root)):
cpv = str(child.cpv)
action = str(child.operation)
# If we're uninstalling a package, check whether Portage is
# installing a replacement. If so, just depend on the installation
# of the new package, because the old package will automatically
# be uninstalled at that time.
if action == 'uninstall':
for pkg in final_db.match_pkgs(child.slot_atom):
cpv = str(pkg.cpv)
action = 'merge'
break
deps[cpv] = dict(action=action,
deptypes=[str(x) for x in priorities],
deps={})
# We've built our list of deps, so we can add our package to the tree.
# Some objects in digraph.nodes can be sets instead of packages, but we
# only want to return packages in our dependency graph.
if isinstance(node, Package) and isinstance(child, Package):
# If a package is destined for ROOT, then it is in DEPEND OR RDEPEND
# and we want to include it in the deps tree. If the package is not
# destined for ROOT, then (in the Chrome OS build) we must be building
# for a board and the package is a BDEPEND. We only want to add that
# package to the deps_tree if include_bdepend is set.
if (node.root == root or self.include_bdepend):
deps_tree[str(node.cpv)] = dict(action=str(node.operation),
deps=deps)
# The only packages that will have a distinct root (in the Chrome OS
# build) are BDEPEND packages for a board target. If we are building
# for the host (the SDK) then BDEPEND packages 1. Will have the same
# root as every other package and 2. Are functionally the same as
# DEPEND packages and belong in the deps_tree.
#
# If include_bdepend is passed and we are building for a board target,
# BDEPEND packages will intentionally show up in both deps_tree and
# bdeps_tree.
if node.root != root:
bdeps_tree[str(node.cpv)] = dict(action=str(
node.operation),
deps=deps)
# Ask portage for its install plan, so that we can only throw out
# dependencies that portage throws out.
deps_info = {}
for pkg in depgraph.altlist():
if isinstance(pkg, Package):
# Save off info about the package
deps_info[str(pkg.cpv)] = {'idx': len(deps_info)}
self.package_db[pkg.cpv] = pkg
seconds = time.time() - start
if '--quiet' not in emerge.opts:
print('Deps calculated in %dm%.1fs' %
(seconds // 60, seconds % 60))
return deps_tree, deps_info, bdeps_tree
from portage.output import EOutput
from _emerge.stdout_spinner import stdout_spinner
###############
## Variables ##
###############
quiet = False
force = False
root = '/'
gir_dirs = ['/usr/share/gir-1.0']
girs = {}
girversion = ''
gi = 'gobject-introspection'
settings = portage.settings
eoutput = EOutput()
spinner = stdout_spinner()
###############
## Functions ##
###############
def usage():
print """gir-rebuilder: Rebuilds gobject-introspection typelibs and girs in the following directories:
%s
Usage: %s [--force] [ARGUMENTS TO EMERGE]
Arguments:
--help This text
--force Force rebuilding of *all* girs and typelibs
All other arguments are passed to portage
from portage.output import EOutput
from _emerge.stdout_spinner import stdout_spinner
###############
## Variables ##
###############
quiet = False
force = False
root = '/'
gir_dirs = ['/usr/share/gir-1.0']
girs = {}
girversion = ''
gi = 'gobject-introspection'
settings = portage.settings
eoutput = EOutput()
spinner = stdout_spinner()
###############
## Functions ##
###############
def usage():
print """gir-rebuilder: Rebuilds gobject-introspection typelibs and girs in the following directories:
%s
Usage: %s [--force] [ARGUMENTS TO EMERGE]
Arguments:
--help This text
--force Force rebuilding of *all* girs and typelibs
def _run_builder(self, dirs_cleanup_queue):
"""
This method is called by _run and executes the whole package build
logic, including constraints validation given by argv parameters.
NOTE: negative errors indicate warnings that can be skipped.
"""
if self._packages:
first_package = self._packages[0]
else:
first_package = "_empty_"
log_dir = mkdtemp(prefix="matter_build.",
suffix="." +
first_package.replace("/", "_").lstrip("<>=~"))
dirs_cleanup_queue.append(log_dir)
emerge_settings, emerge_trees, mtimedb = self._emerge_config
# reset settings to original state, variables will be reconfigured
# while others may remain saved due to backup_changes().
emerge_settings.unlock()
emerge_settings.reset()
emerge_settings.lock()
# Setup stable/unstable keywords, must be done on
# emerge_settings bacause the reference is spread everywhere
# in emerge_trees.
# This is not thread-safe, but Portage isn't either, so
# who cares!
# ACCEPT_KEYWORDS is not saved and reset every time by the
# reset() call above.
portdb = emerge_trees[emerge_settings["ROOT"]]["porttree"].dbapi
self._setup_keywords(portdb, emerge_settings)
portdb.freeze()
vardb = emerge_trees[emerge_settings["ROOT"]]["vartree"].dbapi
vardb.settings.unlock()
vardb.settings["PORT_LOGDIR"] = log_dir
vardb.settings.backup_changes("PORT_LOGDIR")
vardb.settings.lock()
# Load the most current variables from /etc/profile.env, which
# has been re-generated by the env-update call in _run()
emerge_settings.unlock()
emerge_settings.reload()
emerge_settings.regenerate()
emerge_settings.lock()
sets = self._get_sets_mod() # can be None
sets_conf = None
if sets is not None:
sets_conf = sets.load_default_config(
emerge_settings, emerge_trees[emerge_settings["ROOT"]])
packages = []
# execute basic, pre-graph generation filters against each
# package dependency in self._packages.
# This is just fast pruning of obvious obviousness.
for package in self._packages:
expanded_pkgs = []
# package sets support
if package.startswith("@") and sets_conf:
try:
set_pkgs = sets_conf.getSetAtoms(package[1:])
expanded_pkgs.extend(sorted(set_pkgs))
except sets.PackageSetNotFound:
# make it fail, add set directly
expanded_pkgs.append(package)
else:
expanded_pkgs.append(package)
for exp_pkg in expanded_pkgs:
accepted = self._pre_graph_filters(exp_pkg, portdb, vardb)
for best_visible in accepted:
packages.append((exp_pkg, best_visible))
if not packages:
print_warning("No remaining packages in queue, aborting.")
return 0
# at this point we can go ahead building packages
print_info("starting to build:")
for package, best_visible in packages:
print_info(": %s -> %s" % (
package,
best_visible,
))
if not getcolor():
portage.output.nocolor()
# non interactive properties, this is not really required
# accept-properties just sets os.environ...
build_args = list(self._setup_build_args(self._params))
build_args += ["=" + best_v for _x, best_v in packages]
myaction, myopts, myfiles = parse_opts(build_args)
adjust_configs(myopts, emerge_trees)
apply_priorities(emerge_settings)
spinner = stdout_spinner()
if "--quiet" in myopts:
spinner.update = spinner.update_basic
elif "--nospinner" in myopts:
spinner.update = spinner.update_basic
if emerge_settings.get("TERM") == "dumb" or not is_stdout_a_tty():
spinner.update = spinner.update_basic
print_info("emerge args: %s" % (" ".join(build_args), ))
params = create_depgraph_params(myopts, myaction)
success, graph, favorites = backtrack_depgraph(emerge_settings,
emerge_trees, myopts,
params, myaction,
myfiles, spinner)
if not success:
# print issues to stdout and give up
print_warning("dependencies calculation failed, aborting")
graph.display_problems()
# try to collect some info about the failure
bt_config = (graph.get_backtrack_infos() or {}).get("config", {})
for k, v in bt_config.items():
if k == "needed_use_config_changes":
for tup in v:
try:
pkg, (new_use, new_changes) = tup
except (ValueError, TypeError):
print_error(
"unsupported needed_use_config_changes: %s" %
(tup, ))
continue
obj = self._missing_use_packages.setdefault(
"%s" % (pkg.cpv, ), {})
obj["cp:slot"] = "%s" % (pkg.slot_atom, )
changes = obj.setdefault("changes", {})
changes.update(copy.deepcopy(new_changes))
elif k == "needed_unstable_keywords":
for pkg in v:
self._needed_unstable_keywords.add("%s" % (pkg.cpv, ))
elif k == "needed_p_mask_changes":
for pkg in v:
self._needed_package_mask_changes.add("%s" %
(pkg.cpv, ))
elif k == "needed_license_changes":
for pkg, lics in v:
obj = self._needed_license_changes.setdefault(
"%s" % (pkg.cpv, ), set())
obj.update(lics)
else:
print_warning("unsupported backtrack info: %s -> %s" % (
k,
v,
))
return 0
print_info("dependency graph generated successfully")
real_queue = self._post_graph_filters(graph, vardb, portdb)
if real_queue is None:
# post-graph filters not passed, giving up
return 0
merge_queue = [x for x in real_queue if x.operation == "merge"]
unmerge_queue = [x for x in real_queue if x.operation == "uninstall"]
if merge_queue:
print_info("about to build the following packages:")
for pkg in merge_queue:
print_info(" %s" % (pkg.cpv, ))
if unmerge_queue:
print_info("about to uninstall the following packages:")
for pkg in unmerge_queue:
print_info(" %s" % (pkg.cpv, ))
if self._pretend:
print_info("portage spawned with --pretend, done!")
return 0
# re-calling action_build(), deps are re-calculated though
validate_ebuild_environment(emerge_trees)
mergetask = Scheduler(emerge_settings,
emerge_trees,
mtimedb,
myopts,
spinner,
favorites=favorites,
graph_config=graph.schedulerGraph())
del graph
self.clear_caches(self._emerge_config)
retval = mergetask.merge()
not_merged = []
real_queue_map = dict((pkg.cpv, pkg) for pkg in real_queue)
failed_package = None
if retval != 0:
merge_list = mtimedb.get("resume", {}).get("mergelist", [])
for _merge_type, _merge_root, merge_atom, _merge_act in merge_list:
merge_atom = "%s" % (merge_atom, )
if failed_package is None:
# we consider the first encountered package the one
# that failed. It makes sense since packages are built
# serially as of today.
# Also, the package object must be available in our
# package queue, so grab it from there.
failed_package = real_queue_map.get(merge_atom)
not_merged.append(merge_atom)
self._not_merged_packages.append(merge_atom)
for pkg in real_queue:
cpv = pkg.cpv
if not cpv:
print_warning("package: %s, has broken cpv: '%s', ignoring" % (
pkg,
cpv,
))
elif cpv not in not_merged:
if pkg.operation == "merge":
# add to build queue
print_info("package: %s, successfully built" % (cpv, ))
self._built_packages.append("%s" % (cpv, ))
else:
# add to uninstall queue
print_info("package: %s, successfully uninstalled" %
(cpv, ))
self._uninstalled_packages.append("%s" % (cpv, ))
post_emerge(myaction, myopts, myfiles, emerge_settings["ROOT"],
emerge_trees, mtimedb, retval)
subprocess.call(["env-update"])
if failed_package is not None:
print_warning("failed package: %s::%s" % (
failed_package.cpv,
failed_package.repo,
))
if self._params["buildfail"] and (failed_package is not None):
std_env = self._build_standard_environment(
repository=self._params["repository"])
std_env["MATTER_PACKAGE_NAMES"] = " ".join(self._packages)
std_env["MATTER_PORTAGE_FAILED_PACKAGE_NAME"] = failed_package.cpv
std_env["MATTER_PORTAGE_REPOSITORY"] = failed_package.repo
# call pkgfail hook if defined
std_env["MATTER_PORTAGE_BUILD_LOG_DIR"] = os.path.join(
log_dir, "build")
buildfail = self._params["buildfail"]
print_info("spawning buildfail: %s" % (buildfail, ))
tmp_fd, tmp_path = mkstemp()
with os.fdopen(tmp_fd, "wb") as tmp_f:
with open(buildfail, "rb") as buildfail_f:
tmp_f.write(buildfail_f.read())
try:
# now execute
os.chmod(tmp_path, 0o700)
exit_st = subprocess.call([tmp_path], env=std_env)
if exit_st != 0:
return exit_st
finally:
os.remove(tmp_path)
print_info("portage spawned, return value: %d" % (retval, ))
return retval