def CreateDepgraph(self, emerge, packages):
"""Create an emerge depgraph object."""
# Setup emerge options.
emerge_opts = emerge.opts.copy()
# Ask portage to build a dependency graph. with the options we specified
# above.
params = create_depgraph_params(emerge_opts, emerge.action)
success, depgraph, favorites = backtrack_depgraph(
emerge.settings, emerge.trees, emerge_opts, params, emerge.action,
packages, emerge.spinner)
emerge.depgraph = depgraph
# Is it impossible to honor the user's request? Bail!
if not success:
depgraph.display_problems()
sys.exit(1)
emerge.depgraph = depgraph
emerge.favorites = favorites
# Prime and flush emerge caches.
root = emerge.settings['ROOT']
vardb = emerge.trees[root]['vartree'].dbapi
if '--pretend' not in emerge.opts:
vardb.counter_tick()
vardb.flush_cache()
def run(self, atoms, options={}, action=None):
options = options.copy()
options["--pretend"] = True
if self.debug:
options["--debug"] = True
if action is None:
if options.get("--depclean"):
action = "depclean"
elif options.get("--prune"):
action = "prune"
if "--usepkgonly" in options:
options["--usepkg"] = True
global_noiselimit = portage.util.noiselimit
global_emergelog_disable = _emerge.emergelog._disable
try:
if not self.debug:
portage.util.noiselimit = -2
_emerge.emergelog._disable = True
if action in ("depclean", "prune"):
depclean_result = _calc_depclean(
self.settings,
self.trees,
None,
options,
action,
InternalPackageSet(initial_atoms=atoms,
allow_wildcard=True),
None,
)
result = ResolverPlaygroundDepcleanResult(
atoms,
depclean_result.returncode,
depclean_result.cleanlist,
depclean_result.ordered,
depclean_result.req_pkg_count,
depclean_result.depgraph,
)
else:
params = create_depgraph_params(options, action)
success, depgraph, favorites = backtrack_depgraph(
self.settings, self.trees, options, params, action, atoms,
None)
depgraph._show_merge_list()
depgraph.display_problems()
result = ResolverPlaygroundResult(atoms, success, depgraph,
favorites)
finally:
portage.util.noiselimit = global_noiselimit
_emerge.emergelog._disable = global_emergelog_disable
return result
def calcdep(config):
settings = config.target_config.settings
trees = config.trees
opts = config.opts
action = config.action
params = create_depgraph_params(opts, action)
files = config.args
spinner = stdout_spinner()
# spinner.update = spinner.update_quiet
return _backtrack_depgraph(settings, trees, opts, params, action, files,
spinner)
def build_mydepgraph(settings, trees, mtimedb, myopts, myparams, myaction, myfiles, spinner, build_dict, session): try: success, mydepgraph, favorites = backtrack_depgraph( settings, trees, myopts, myparams, myaction, myfiles, spinner) except portage.exception.PackageSetNotFound as e: root_config = trees[settings["ROOT"]]["root_config"] display_missing_pkg_set(root_config, e.value) build_dict['type_fail'] = "depgraph fail" build_dict['check_fail'] = True else: if not success: repeat = True repeat_times = 0 while repeat: if mydepgraph._dynamic_config._needed_p_mask_changes: build_dict['type_fail'] = "Mask package or dep" build_dict['check_fail'] = True elif mydepgraph._dynamic_config._needed_use_config_changes: mydepgraph._display_autounmask() build_dict['type_fail'] = "Need use change" build_dict['check_fail'] = True elif mydepgraph._dynamic_config._slot_conflict_handler: build_dict['type_fail'] = "Slot blocking" build_dict['check_fail'] = True elif mydepgraph._dynamic_config._circular_deps_for_display: build_dict['type_fail'] = "Circular Deps" build_dict['check_fail'] = True elif mydepgraph._dynamic_config._unsolvable_blockers: build_dict['type_fail'] = "Blocking packages" build_dict['check_fail'] = True else: build_dict['type_fail'] = "Dep calc fail" build_dict['check_fail'] = True mydepgraph.display_problems() if repeat_times is 2: repeat = False log_fail_queru(session, build_dict, settings) else: repeat_times = repeat_times + 1 settings, trees, mtimedb = load_emerge_config() myparams = create_depgraph_params(myopts, myaction) try: success, mydepgraph, favorites = backtrack_depgraph( settings, trees, myopts, myparams, myaction, myfiles, spinner) except portage.exception.PackageSetNotFound as e: root_config = trees[settings["ROOT"]]["root_config"] display_missing_pkg_set(root_config, e.value) if success: repeat = False return success, settings, trees, mtimedb, mydepgraph
def run(self, atoms, options={}, action=None):
options = options.copy()
options["--pretend"] = True
options["--quiet"] = True
if self.debug:
options["--debug"] = True
if not self.debug:
portage.util.noiselimit = -2
params = create_depgraph_params(options, action)
success, depgraph, favorites = backtrack_depgraph(
self.settings, self.trees, options, params, action, atoms, None)
depgraph.display_problems()
result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
portage.util.noiselimit = 0
return result
def run(self, atoms, options={}, action=None):
options = options.copy()
options["--pretend"] = True
options["--quiet"] = True
if self.debug:
options["--debug"] = True
if not self.debug:
portage.util.noiselimit = -2
params = create_depgraph_params(options, action)
success, depgraph, favorites = backtrack_depgraph(
self.settings, self.trees, options, params, action, atoms, None)
depgraph.display_problems()
result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
portage.util.noiselimit = 0
return result
def run(self, atoms, options={}, action=None):
options = options.copy()
options["--pretend"] = True
if self.debug:
options["--debug"] = True
if action is None:
if options.get("--depclean"):
action = "depclean"
elif options.get("--prune"):
action = "prune"
if "--usepkgonly" in options:
options["--usepkg"] = True
global_noiselimit = portage.util.noiselimit
global_emergelog_disable = _emerge.emergelog._disable
try:
if not self.debug:
portage.util.noiselimit = -2
_emerge.emergelog._disable = True
if action in ("depclean", "prune"):
rval, cleanlist, ordered, req_pkg_count = \
calc_depclean(self.settings, self.trees, None,
options, action, InternalPackageSet(initial_atoms=atoms, allow_wildcard=True), None)
result = ResolverPlaygroundDepcleanResult(
atoms, rval, cleanlist, ordered, req_pkg_count)
else:
params = create_depgraph_params(options, action)
success, depgraph, favorites = backtrack_depgraph(
self.settings, self.trees, options, params, action, atoms, None)
depgraph._show_merge_list()
depgraph.display_problems()
result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
finally:
portage.util.noiselimit = global_noiselimit
_emerge.emergelog._disable = global_emergelog_disable
return result
def build(emerge_config):
settings = emerge_config.target_config.settings
trees = emerge_config.trees
myopts = emerge_config.opts
myaction = emerge_config.action
myparams = create_depgraph_params(myopts, myaction)
myfiles = emerge_config.args
spinner = stdout_spinner()
spinner.update = spinner.update_quiet
try:
success, depgraph, favorites = _backtrack_depgraph(
settings, trees, myopts, myparams, myaction, myfiles,
spinner)
return (success, depgraph, favorites)
finally:
# Call destructors for our portdbapi instances.
for x in emerge_config.trees.values():
if "porttree" in x.lazy_items:
continue
x["porttree"].dbapi.close_caches()
def _run_builder(self, dirs_cleanup_queue):
"""
This method is called by _run and executes the whole package build
logic, including constraints validation given by argv parameters.
NOTE: negative errors indicate warnings that can be skipped.
"""
if self._packages:
first_package = self._packages[0]
else:
first_package = "_empty_"
log_dir = mkdtemp(prefix="matter_build.",
suffix="." + first_package.replace("/", "_").lstrip("<>=~"))
dirs_cleanup_queue.append(log_dir)
emerge_settings, emerge_trees, mtimedb = self._emerge_config
# reset settings to original state, variables will be reconfigured
# while others may remain saved due to backup_changes().
emerge_settings.unlock()
emerge_settings.reset()
emerge_settings.lock()
# Setup stable/unstable keywords, must be done on
# emerge_settings bacause the reference is spread everywhere
# in emerge_trees.
# This is not thread-safe, but Portage isn't either, so
# who cares!
# ACCEPT_KEYWORDS is not saved and reset every time by the
# reset() call above.
portdb = emerge_trees[emerge_settings["ROOT"]]["porttree"].dbapi
self._setup_keywords(portdb, emerge_settings)
portdb.freeze()
vardb = emerge_trees[emerge_settings["ROOT"]]["vartree"].dbapi
vardb.settings.unlock()
vardb.settings["PORT_LOGDIR"] = log_dir
vardb.settings.backup_changes("PORT_LOGDIR")
vardb.settings.lock()
# Load the most current variables from /etc/profile.env, which
# has been re-generated by the env-update call in _run()
emerge_settings.unlock()
emerge_settings.reload()
emerge_settings.regenerate()
emerge_settings.lock()
sets = self._get_sets_mod() # can be None
sets_conf = None
if sets is not None:
sets_conf = sets.load_default_config(
emerge_settings,
emerge_trees[emerge_settings["ROOT"]])
packages = []
# execute basic, pre-graph generation filters against each
# package dependency in self._packages.
# This is just fast pruning of obvious obviousness.
for package in self._packages:
expanded_pkgs = []
# package sets support
if package.startswith("@") and sets_conf:
try:
set_pkgs = sets_conf.getSetAtoms(package[1:])
expanded_pkgs.extend(sorted(set_pkgs))
except sets.PackageSetNotFound:
# make it fail, add set directly
expanded_pkgs.append(package)
else:
expanded_pkgs.append(package)
for exp_pkg in expanded_pkgs:
accepted = self._pre_graph_filters(
exp_pkg, portdb, vardb)
for best_visible in accepted:
packages.append((exp_pkg, best_visible))
if not packages:
print_warning("No remaining packages in queue, aborting.")
return 0
# at this point we can go ahead building packages
print_info("starting to build:")
for package, best_visible in packages:
print_info(": %s -> %s" % (
package, best_visible,))
if not getcolor():
portage.output.nocolor()
# non interactive properties, this is not really required
# accept-properties just sets os.environ...
build_args = list(self._setup_build_args(self._params))
build_args += ["=" + best_v for _x, best_v in packages]
myaction, myopts, myfiles = parse_opts(build_args)
adjust_configs(myopts, emerge_trees)
apply_priorities(emerge_settings)
spinner = stdout_spinner()
if "--quiet" in myopts:
spinner.update = spinner.update_basic
elif "--nospinner" in myopts:
spinner.update = spinner.update_basic
if emerge_settings.get("TERM") == "dumb" or not is_stdout_a_tty():
spinner.update = spinner.update_basic
print_info("emerge args: %s" % (" ".join(build_args),))
params = create_depgraph_params(myopts, myaction)
success, graph, favorites = backtrack_depgraph(emerge_settings,
emerge_trees, myopts, params, myaction, myfiles, spinner)
if not success:
# print issues to stdout and give up
print_warning("dependencies calculation failed, aborting")
graph.display_problems()
# try to collect some info about the failure
bt_config = (graph.get_backtrack_infos() or {}).get("config", {})
for k, v in bt_config.items():
if k == "needed_use_config_changes":
for tup in v:
try:
pkg, (new_use, new_changes) = tup
except (ValueError, TypeError):
print_error(
"unsupported needed_use_config_changes: %s" % (
tup,))
continue
obj = self._missing_use_packages.setdefault(
"%s" % (pkg.cpv,), {})
obj["cp:slot"] = "%s" % (pkg.slot_atom,)
changes = obj.setdefault("changes", {})
changes.update(copy.deepcopy(new_changes))
elif k == "needed_unstable_keywords":
for pkg in v:
self._needed_unstable_keywords.add("%s" % (pkg.cpv,))
elif k == "needed_p_mask_changes":
for pkg in v:
self._needed_package_mask_changes.add(
"%s" % (pkg.cpv,))
elif k == "needed_license_changes":
for pkg, lics in v:
obj = self._needed_license_changes.setdefault(
"%s" % (pkg.cpv,), set())
obj.update(lics)
else:
print_warning("unsupported backtrack info: %s -> %s" % (
k, v,))
return 0
print_info("dependency graph generated successfully")
real_queue = self._post_graph_filters(graph, vardb, portdb)
if real_queue is None:
# post-graph filters not passed, giving up
return 0
merge_queue = [x for x in real_queue if x.operation == "merge"]
unmerge_queue = [x for x in real_queue if x.operation == "uninstall"]
if merge_queue:
print_info("about to build the following packages:")
for pkg in merge_queue:
print_info(" %s" % (pkg.cpv,))
if unmerge_queue:
print_info("about to uninstall the following packages:")
for pkg in unmerge_queue:
print_info(" %s" % (pkg.cpv,))
if self._pretend:
print_info("portage spawned with --pretend, done!")
return 0
# re-calling action_build(), deps are re-calculated though
validate_ebuild_environment(emerge_trees)
mergetask = Scheduler(emerge_settings, emerge_trees, mtimedb,
myopts, spinner, favorites=favorites,
graph_config=graph.schedulerGraph())
del graph
self.clear_caches(self._emerge_config)
retval = mergetask.merge()
not_merged = []
real_queue_map = dict((pkg.cpv, pkg) for pkg in real_queue)
failed_package = None
if retval != 0:
merge_list = mtimedb.get("resume", {}).get("mergelist")
for _merge_type, _merge_root, merge_atom, _merge_act in merge_list:
merge_atom = "%s" % (merge_atom,)
if failed_package is None:
# we consider the first encountered package the one
# that failed. It makes sense since packages are built
# serially as of today.
# Also, the package object must be available in our
# package queue, so grab it from there.
failed_package = real_queue_map.get(merge_atom)
not_merged.append(merge_atom)
self._not_merged_packages.append(merge_atom)
for pkg in real_queue:
cpv = pkg.cpv
if not cpv:
print_warning("package: %s, has broken cpv: '%s', ignoring" % (
pkg, cpv,))
elif cpv not in not_merged:
if pkg.operation == "merge":
# add to build queue
print_info("package: %s, successfully built" % (cpv,))
self._built_packages.append("%s" % (cpv,))
else:
# add to uninstall queue
print_info("package: %s, successfully uninstalled" % (cpv,))
self._uninstalled_packages.append("%s" % (cpv,))
post_emerge(myaction, myopts, myfiles, emerge_settings["ROOT"],
emerge_trees, mtimedb, retval)
subprocess.call(["env-update"])
if failed_package is not None:
print_warning("failed package: %s::%s" % (failed_package.cpv,
failed_package.repo,))
if self._params["buildfail"] and (failed_package is not None):
std_env = self._build_standard_environment(
repository=self._params["repository"])
std_env["MATTER_PACKAGE_NAMES"] = " ".join(self._packages)
std_env["MATTER_PORTAGE_FAILED_PACKAGE_NAME"] = failed_package.cpv
std_env["MATTER_PORTAGE_REPOSITORY"] = failed_package.repo
# call pkgfail hook if defined
std_env["MATTER_PORTAGE_BUILD_LOG_DIR"] = os.path.join(log_dir,
"build")
buildfail = self._params["buildfail"]
print_info("spawning buildfail: %s" % (buildfail,))
tmp_fd, tmp_path = mkstemp()
with os.fdopen(tmp_fd, "wb") as tmp_f:
with open(buildfail, "rb") as buildfail_f:
tmp_f.write(buildfail_f.read())
try:
# now execute
os.chmod(tmp_path, 0o700)
exit_st = subprocess.call([tmp_path], env = std_env)
if exit_st != 0:
return exit_st
finally:
os.remove(tmp_path)
print_info("portage spawned, return value: %d" % (retval,))
return retval
def _run_builder(self, dirs_cleanup_queue):
"""
This method is called by _run and executes the whole package build
logic, including constraints validation given by argv parameters.
NOTE: negative errors indicate warnings that can be skipped.
"""
if self._packages:
first_package = self._packages[0]
else:
first_package = "_empty_"
log_dir = mkdtemp(prefix="matter_build.",
suffix="." +
first_package.replace("/", "_").lstrip("<>=~"))
dirs_cleanup_queue.append(log_dir)
emerge_settings, emerge_trees, mtimedb = self._emerge_config
# reset settings to original state, variables will be reconfigured
# while others may remain saved due to backup_changes().
emerge_settings.unlock()
emerge_settings.reset()
emerge_settings.lock()
# Setup stable/unstable keywords, must be done on
# emerge_settings bacause the reference is spread everywhere
# in emerge_trees.
# This is not thread-safe, but Portage isn't either, so
# who cares!
# ACCEPT_KEYWORDS is not saved and reset every time by the
# reset() call above.
portdb = emerge_trees[emerge_settings["ROOT"]]["porttree"].dbapi
self._setup_keywords(portdb, emerge_settings)
portdb.freeze()
vardb = emerge_trees[emerge_settings["ROOT"]]["vartree"].dbapi
vardb.settings.unlock()
vardb.settings["PORT_LOGDIR"] = log_dir
vardb.settings.backup_changes("PORT_LOGDIR")
vardb.settings.lock()
# Load the most current variables from /etc/profile.env, which
# has been re-generated by the env-update call in _run()
emerge_settings.unlock()
emerge_settings.reload()
emerge_settings.regenerate()
emerge_settings.lock()
sets = self._get_sets_mod() # can be None
sets_conf = None
if sets is not None:
sets_conf = sets.load_default_config(
emerge_settings, emerge_trees[emerge_settings["ROOT"]])
packages = []
# execute basic, pre-graph generation filters against each
# package dependency in self._packages.
# This is just fast pruning of obvious obviousness.
for package in self._packages:
expanded_pkgs = []
# package sets support
if package.startswith("@") and sets_conf:
try:
set_pkgs = sets_conf.getSetAtoms(package[1:])
expanded_pkgs.extend(sorted(set_pkgs))
except sets.PackageSetNotFound:
# make it fail, add set directly
expanded_pkgs.append(package)
else:
expanded_pkgs.append(package)
for exp_pkg in expanded_pkgs:
accepted = self._pre_graph_filters(exp_pkg, portdb, vardb)
for best_visible in accepted:
packages.append((exp_pkg, best_visible))
if not packages:
print_warning("No remaining packages in queue, aborting.")
return 0
# at this point we can go ahead building packages
print_info("starting to build:")
for package, best_visible in packages:
print_info(": %s -> %s" % (
package,
best_visible,
))
if not getcolor():
portage.output.nocolor()
# non interactive properties, this is not really required
# accept-properties just sets os.environ...
build_args = list(self._setup_build_args(self._params))
build_args += ["=" + best_v for _x, best_v in packages]
myaction, myopts, myfiles = parse_opts(build_args)
adjust_configs(myopts, emerge_trees)
apply_priorities(emerge_settings)
spinner = stdout_spinner()
if "--quiet" in myopts:
spinner.update = spinner.update_basic
elif "--nospinner" in myopts:
spinner.update = spinner.update_basic
if emerge_settings.get("TERM") == "dumb" or not is_stdout_a_tty():
spinner.update = spinner.update_basic
print_info("emerge args: %s" % (" ".join(build_args), ))
params = create_depgraph_params(myopts, myaction)
success, graph, favorites = backtrack_depgraph(emerge_settings,
emerge_trees, myopts,
params, myaction,
myfiles, spinner)
if not success:
# print issues to stdout and give up
print_warning("dependencies calculation failed, aborting")
graph.display_problems()
# try to collect some info about the failure
bt_config = (graph.get_backtrack_infos() or {}).get("config", {})
for k, v in bt_config.items():
if k == "needed_use_config_changes":
for tup in v:
try:
pkg, (new_use, new_changes) = tup
except (ValueError, TypeError):
print_error(
"unsupported needed_use_config_changes: %s" %
(tup, ))
continue
obj = self._missing_use_packages.setdefault(
"%s" % (pkg.cpv, ), {})
obj["cp:slot"] = "%s" % (pkg.slot_atom, )
changes = obj.setdefault("changes", {})
changes.update(copy.deepcopy(new_changes))
elif k == "needed_unstable_keywords":
for pkg in v:
self._needed_unstable_keywords.add("%s" % (pkg.cpv, ))
elif k == "needed_p_mask_changes":
for pkg in v:
self._needed_package_mask_changes.add("%s" %
(pkg.cpv, ))
elif k == "needed_license_changes":
for pkg, lics in v:
obj = self._needed_license_changes.setdefault(
"%s" % (pkg.cpv, ), set())
obj.update(lics)
else:
print_warning("unsupported backtrack info: %s -> %s" % (
k,
v,
))
return 0
print_info("dependency graph generated successfully")
real_queue = self._post_graph_filters(graph, vardb, portdb)
if real_queue is None:
# post-graph filters not passed, giving up
return 0
merge_queue = [x for x in real_queue if x.operation == "merge"]
unmerge_queue = [x for x in real_queue if x.operation == "uninstall"]
if merge_queue:
print_info("about to build the following packages:")
for pkg in merge_queue:
print_info(" %s" % (pkg.cpv, ))
if unmerge_queue:
print_info("about to uninstall the following packages:")
for pkg in unmerge_queue:
print_info(" %s" % (pkg.cpv, ))
if self._pretend:
print_info("portage spawned with --pretend, done!")
return 0
# re-calling action_build(), deps are re-calculated though
validate_ebuild_environment(emerge_trees)
mergetask = Scheduler(emerge_settings,
emerge_trees,
mtimedb,
myopts,
spinner,
favorites=favorites,
graph_config=graph.schedulerGraph())
del graph
self.clear_caches(self._emerge_config)
retval = mergetask.merge()
not_merged = []
real_queue_map = dict((pkg.cpv, pkg) for pkg in real_queue)
failed_package = None
if retval != 0:
merge_list = mtimedb.get("resume", {}).get("mergelist", [])
for _merge_type, _merge_root, merge_atom, _merge_act in merge_list:
merge_atom = "%s" % (merge_atom, )
if failed_package is None:
# we consider the first encountered package the one
# that failed. It makes sense since packages are built
# serially as of today.
# Also, the package object must be available in our
# package queue, so grab it from there.
failed_package = real_queue_map.get(merge_atom)
not_merged.append(merge_atom)
self._not_merged_packages.append(merge_atom)
for pkg in real_queue:
cpv = pkg.cpv
if not cpv:
print_warning("package: %s, has broken cpv: '%s', ignoring" % (
pkg,
cpv,
))
elif cpv not in not_merged:
if pkg.operation == "merge":
# add to build queue
print_info("package: %s, successfully built" % (cpv, ))
self._built_packages.append("%s" % (cpv, ))
else:
# add to uninstall queue
print_info("package: %s, successfully uninstalled" %
(cpv, ))
self._uninstalled_packages.append("%s" % (cpv, ))
post_emerge(myaction, myopts, myfiles, emerge_settings["ROOT"],
emerge_trees, mtimedb, retval)
subprocess.call(["env-update"])
if failed_package is not None:
print_warning("failed package: %s::%s" % (
failed_package.cpv,
failed_package.repo,
))
if self._params["buildfail"] and (failed_package is not None):
std_env = self._build_standard_environment(
repository=self._params["repository"])
std_env["MATTER_PACKAGE_NAMES"] = " ".join(self._packages)
std_env["MATTER_PORTAGE_FAILED_PACKAGE_NAME"] = failed_package.cpv
std_env["MATTER_PORTAGE_REPOSITORY"] = failed_package.repo
# call pkgfail hook if defined
std_env["MATTER_PORTAGE_BUILD_LOG_DIR"] = os.path.join(
log_dir, "build")
buildfail = self._params["buildfail"]
print_info("spawning buildfail: %s" % (buildfail, ))
tmp_fd, tmp_path = mkstemp()
with os.fdopen(tmp_fd, "wb") as tmp_f:
with open(buildfail, "rb") as buildfail_f:
tmp_f.write(buildfail_f.read())
try:
# now execute
os.chmod(tmp_path, 0o700)
exit_st = subprocess.call([tmp_path], env=std_env)
if exit_st != 0:
return exit_st
finally:
os.remove(tmp_path)
print_info("portage spawned, return value: %d" % (retval, ))
return retval
