def find_dependency_candidates(requiring_impl_var, dependency):
def meets_restrictions(candidate):
for r in dependency.restrictions:
if not r.meets_restriction(candidate):
#warn("%s rejected due to %s", candidate.get_version(), r)
return False
return True
essential = dependency.importance == model.Dependency.Essential
dep_iface = iface_cache.get_interface(dependency.interface)
dep_union = [sat.neg(requiring_impl_var)
] # Either requiring_impl_var is False, or ...
for candidate in impls_for_iface[dep_iface]:
if (candidate.__class__ is
_DummyImpl) or meets_restrictions(candidate):
if essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
dep_union.append(c_var)
# else we filtered that version out, so ignore it
else:
# Candidate doesn't meet our requirements
# If the dependency is optional add a rule to make sure we don't
# select this candidate.
# (for essential dependencies this isn't necessary because we must
# select a good version and we can't select two)
if not essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
problem.add_clause(dep_union + [sat.neg(c_var)])
# else we filtered that version out, so ignore it
if essential:
problem.add_clause(dep_union)
def find_dependency_candidates(requiring_impl_var, dependency):
def meets_restrictions(candidate):
for r in dependency.restrictions:
if not r.meets_restriction(candidate):
#warn("%s rejected due to %s", candidate.get_version(), r)
return False
return True
essential = dependency.importance == model.Dependency.Essential
dep_iface = iface_cache.get_interface(dependency.interface)
dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ...
for candidate in impls_for_iface[dep_iface]:
if (candidate.__class__ is _DummyImpl) or meets_restrictions(candidate):
if essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
dep_union.append(c_var)
# else we filtered that version out, so ignore it
else:
# Candidate doesn't meet our requirements
# If the dependency is optional add a rule to make sure we don't
# select this candidate.
# (for essential dependencies this isn't necessary because we must
# select a good version and we can't select two)
if not essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
problem.add_clause(dep_union + [sat.neg(c_var)])
# else we filtered that version out, so ignore it
if essential:
problem.add_clause(dep_union)
def add_command_iface(uri, arch, command_name):
"""Add every <command> in interface 'uri' with this name.
Each one depends on the corresponding implementation and only
one can be selected. If closest_match is on, include a dummy
command that can always be selected."""
# Check whether we've already processed this (interface,command) pair
existing = group_clause_for_command.get((uri, command_name), None)
if existing is not None:
return existing.lits
# First ensure that the interface itself has been processed
# We'll reuse the ordering of the implementations to order
# the commands too.
add_iface(uri, arch)
iface = iface_cache.get_interface(uri)
filtered_impls = impls_for_iface[iface]
impl_to_var = iface_to_vars[iface] # Impl -> sat var
var_names = []
for impl in filtered_impls:
command = impl.commands.get(command_name, None)
if not command:
if not isinstance(impl, _DummyImpl):
# Mark implementation as unselectable
problem.add_clause([sat.neg(impl_to_var[impl])])
continue
# We have a candidate <command>. Require that if it's selected
# then we select the corresponding <implementation> too.
command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch))
problem.add_clause([impl_to_var[impl], sat.neg(command_var)])
var_names.append(command_var)
process_dependencies(command_var, command, arch)
# Tell the user why we couldn't use this version
if self.details is not False:
def new_reason(impl, old_reason):
if command_name in impl.commands:
return old_reason
return old_reason or (_('No %s command') % command_name)
self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]]
if closest_match:
dummy_command = problem.add_variable(None)
var_names.append(dummy_command)
if var_names:
# Can't select more than one of them.
assert (uri, command_name) not in group_clause_for_command
group_clause_for_command[(uri, command_name)] = problem.at_most_one(var_names)
return var_names
def testCoverage(self):
# Try to trigger some edge cases...
# An at_most_one clause must be analysed for causing
# a conflict.
solver = sat.SATProblem()
v1 = solver.add_variable("v1")
v2 = solver.add_variable("v2")
v3 = solver.add_variable("v3")
solver.at_most_one([v1, v2])
solver.add_clause([v1, sat.neg(v3)])
solver.add_clause([v2, sat.neg(v3)])
solver.add_clause([v1, v3])
solver.run_solver(lambda: v3)
def testCoverage(self):
# Try to trigger some edge cases...
# An at_most_one clause must be analysed for causing
# a conflict.
solver = sat.SATProblem()
v1 = solver.add_variable("v1")
v2 = solver.add_variable("v2")
v3 = solver.add_variable("v3")
solver.at_most_one([v1, v2])
solver.add_clause([v1, sat.neg(v3)])
solver.add_clause([v2, sat.neg(v3)])
solver.add_clause([v1, v3])
solver.run_solver(lambda: v3)
def process_dependencies(requiring_var, requirer, arch):
for d in deps_in_use(requirer, arch):
#logger.debug(_("Considering command dependency %s"), d)
add_iface(d.interface, arch.child_arch)
for c in d.get_required_commands():
# We depend on a specific command within the implementation.
command_vars = add_command_iface(d.interface, arch.child_arch, c)
# If the parent command/impl is chosen, one of the candidate commands
# must be too. If there aren't any, then this command is unselectable.
problem.add_clause([sat.neg(requiring_var)] + command_vars)
# Must choose one version of d if impl is selected
find_dependency_candidates(requiring_var, d)
def process_dependencies(requiring_var, requirer, arch):
for d in deps_in_use(requirer, arch):
debug(_("Considering command dependency %s"), d)
add_iface(d.interface, arch.child_arch)
for c in d.get_required_commands():
# We depend on a specific command within the implementation.
command_vars = add_command_iface(d.interface,
arch.child_arch, c)
# If the parent command/impl is chosen, one of the candidate commands
# must be too. If there aren't any, then this command is unselectable.
problem.add_clause([sat.neg(requiring_var)] + command_vars)
# Must choose one version of d if impl is selected
find_dependency_candidates(requiring_var, d)
def find_dependency_candidates(requiring_impl_var, dependency):
dep_iface = iface_cache.get_interface(dependency.interface)
dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ...
for candidate in impls_for_iface[dep_iface]:
for r in dependency.restrictions:
if candidate.__class__ is not _DummyImpl and not r.meets_restriction(candidate):
#warn("%s rejected due to %s", candidate.get_version(), r)
if candidate.version is not None:
break
else:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
dep_union.append(c_var)
# else we filtered that version out, so ignore it
assert dep_union
problem.add_clause(dep_union)
def testWatch(self):
solver = sat.SATProblem()
a = solver.add_variable('a')
b = solver.add_variable('b')
c = solver.add_variable('c')
# Add a clause. It starts watching the first two variables (a and b).
# (use the internal function to avoid variable reordering)
solver._add_clause([a, b, c], False, reason = "testing")
# b is False, so it switches to watching a and c
solver.add_clause([sat.neg(b)])
# Try to trigger bug.
solver.add_clause([c])
decisions = [a]
solver.run_solver(lambda: decisions.pop())
assert not decisions # All used up
assert solver.assigns[a].value == True
def testWatch(self):
solver = sat.SATProblem()
a = solver.add_variable('a')
b = solver.add_variable('b')
c = solver.add_variable('c')
# Add a clause. It starts watching the first two variables (a and b).
# (use the internal function to avoid variable reordering)
solver._add_clause([a, b, c], False)
# b is False, so it switches to watching a and c
solver.add_clause([sat.neg(b)])
# Try to trigger bug.
solver.add_clause([c])
decisions = [a]
solver.run_solver(lambda: decisions.pop())
assert not decisions # All used up
assert solver.assigns[a].value == True
def add_command_iface(uri, arch, command_name):
"""Add every <command> in interface 'uri' with this name.
Each one depends on the corresponding implementation and only
one can be selected. If closest_match is on, include a dummy
command that can always be selected."""
# Check whether we've already processed this (interface,command) pair
existing = group_clause_for_command.get((uri, command_name), None)
if existing is not None:
return existing.lits
# First ensure that the interface itself has been processed
# We'll reuse the ordering of the implementations to order
# the commands too.
add_iface(uri, arch)
iface = iface_cache.get_interface(uri)
filtered_impls = impls_for_iface[iface]
var_names = []
for impl in filtered_impls:
command = impl.commands.get(command_name, None)
if not command:
if not isinstance(impl, _DummyImpl):
# Mark implementation as unselectable
problem.add_clause([sat.neg(impl_to_var[impl])])
continue
# We have a candidate <command>. Require that if it's selected
# then we select the corresponding <implementation> too.
command_var = problem.add_variable(
CommandInfo(command_name, command, impl, arch))
problem.add_clause([impl_to_var[impl], sat.neg(command_var)])
var_names.append(command_var)
process_dependencies(command_var, command, arch)
# Tell the user why we couldn't use this version
if self.record_details:
def new_reason(impl, old_reason):
if command_name in impl.commands:
return old_reason
return old_reason or (_('No %s command') % command_name)
self.details[iface] = [(impl, new_reason(impl, reason))
for (impl,
reason) in self.details[iface]]
if closest_match:
dummy_command = problem.add_variable(None)
var_names.append(dummy_command)
if var_names:
# Can't select more than one of them.
assert (uri, command_name) not in group_clause_for_command
group_clause_for_command[(
uri, command_name)] = problem.at_most_one(var_names)
return var_names
def decide(): """This is called by the SAT solver when it cannot simplify the problem further. Our job is to find the most-optimal next selection to try. Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): # Restrictions don't express that we do or don't want the # dependency, so skip them here. if dep.importance == model.Dependency.Restricts: continue for c in dep.get_required_commands(): dep_lit = find_undecided_command(dep.interface, c) if dep_lit is not None: return dep_lit dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) group = group_clause_for.get(uri, None) if group is None: # (can be None if the selected impl has an optional dependency on # a feed with no implementations) return #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in list(group_clause_for.values()) + list(group_clause_for_command.values()) + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination
def solve(self, root_interface, root_arch, command_name = 'run', closest_match = False):
# closest_match is used internally. It adds a lowest-ranked
# by valid implementation to every interface, so we can always
# select something. Useful for diagnostics.
# TODO: We need some way to figure out which feeds to include.
# Currently, we include any feed referenced from anywhere but
# this is probably too much. We could insert a dummy optimial
# implementation in stale/uncached feeds and see whether it
# selects that.
iface_cache = self.config.iface_cache
problem = sat.SATProblem()
impl_to_var = {} # Impl -> sat var
self.feeds_used = set()
self.requires = {}
self.ready = False
self.details = self.record_details and {}
self.selections = None
self._failure_reason = None
ifaces_processed = set()
impls_for_machine_group = {0 : []} # Machine group (e.g. "64") to [impl] in that group
for machine_group in machine_groups.values():
impls_for_machine_group[machine_group] = []
impls_for_iface = {} # Iface -> [impl]
group_clause_for = {} # Iface URI -> AtMostOneClause | bool
group_clause_for_command = {} # (Iface URI, command name) -> AtMostOneClause | bool
# Return the dependencies of impl that we should consider.
# Skips dependencies if the use flag isn't what we need.
# (note: impl may also be a model.Command)
def deps_in_use(impl, arch):
for dep in impl.requires:
use = dep.metadata.get("use", None)
if use not in arch.use:
continue
yield dep
# Must have already done add_iface on dependency.interface.
# If dependency is essential:
# Add a clause so that if requiring_impl_var is True then an implementation
# matching 'dependency' must also be selected.
# If dependency is optional:
# Require that no incompatible version is selected.
def find_dependency_candidates(requiring_impl_var, dependency):
def meets_restrictions(candidate):
for r in dependency.restrictions:
if not r.meets_restriction(candidate):
#warn("%s rejected due to %s", candidate.get_version(), r)
return False
return True
essential = dependency.importance == model.Dependency.Essential
dep_iface = iface_cache.get_interface(dependency.interface)
dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ...
for candidate in impls_for_iface[dep_iface]:
if (candidate.__class__ is _DummyImpl) or meets_restrictions(candidate):
if essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
dep_union.append(c_var)
# else we filtered that version out, so ignore it
else:
# Candidate doesn't meet our requirements
# If the dependency is optional add a rule to make sure we don't
# select this candidate.
# (for essential dependencies this isn't necessary because we must
# select a good version and we can't select two)
if not essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
problem.add_clause(dep_union + [sat.neg(c_var)])
# else we filtered that version out, so ignore it
if essential:
problem.add_clause(dep_union)
def is_unusable(impl, restrictions, arch):
"""@return: whether this implementation is unusable.
@rtype: bool"""
return get_unusable_reason(impl, restrictions, arch) != None
def get_unusable_reason(impl, restrictions, arch):
"""
@param impl: Implementation to test.
@type restrictions: [L{model.Restriction}]
@return: The reason why this impl is unusable, or None if it's OK.
@rtype: str
@note: The restrictions are for the interface being requested, not the feed
of the implementation; they may be different when feeds are being used."""
for r in restrictions:
if not r.meets_restriction(impl):
return _("Incompatible with another selected implementation")
stability = impl.get_stability()
if stability <= model.buggy:
return stability.name
if (self.config.network_use == model.network_offline or not impl.download_sources) and not impl.is_available(self.config.stores):
if not impl.download_sources:
return _("No retrieval methods")
return _("Not cached and we are off-line")
if impl.os not in arch.os_ranks:
return _("Unsupported OS")
if impl.machine not in arch.machine_ranks:
if impl.machine == 'src':
return _("Source code")
return _("Unsupported machine type")
return None
def usable_feeds(iface, arch):
"""Return all feeds for iface that support arch.
@rtype: generator(ZeroInstallFeed)"""
yield iface.uri
for f in iface_cache.get_feed_imports(iface):
# Note: when searching for src, None is not in machine_ranks
if f.os in arch.os_ranks and \
(f.machine is None or f.machine in arch.machine_ranks):
yield f.uri
else:
debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
{'feed': f, 'os': f.os, 'machine': f.machine})
def add_iface(uri, arch):
"""Name implementations from feed and assert that only one can be selected."""
if uri in ifaces_processed: return
ifaces_processed.add(uri)
iface = iface_cache.get_interface(uri)
impls = []
for f in usable_feeds(iface, arch):
self.feeds_used.add(f)
debug(_("Processing feed %s"), f)
try:
feed = iface_cache.get_feed(f)
if feed is None: continue
#if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for:
# info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f})
if feed.implementations:
impls.extend(feed.implementations.values())
distro_feed_url = feed.get_distro_feed()
if distro_feed_url:
self.feeds_used.add(distro_feed_url)
distro_feed = iface_cache.get_feed(distro_feed_url)
if distro_feed.implementations:
impls.extend(distro_feed.implementations.values())
except MissingLocalFeed as ex:
warn(_("Missing local feed; if it's no longer required, remove it with:") +
'\n0install remove-feed ' + iface.uri + ' ' + f,
{'feed': f, 'interface': iface, 'exception': ex})
except Exception as ex:
warn(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f, 'interface': iface, 'exception': ex})
#raise
impls.sort(lambda a, b: self.compare(iface, a, b, arch))
impls_for_iface[iface] = filtered_impls = []
my_extra_restrictions = self.extra_restrictions.get(iface, [])
if self.record_details:
self.details[iface] = [(impl, get_unusable_reason(impl, my_extra_restrictions, arch)) for impl in impls]
rank = 1
var_names = []
for impl in impls:
if is_unusable(impl, my_extra_restrictions, arch):
continue
filtered_impls.append(impl)
assert impl not in impl_to_var
v = problem.add_variable(ImplInfo(iface, impl, arch))
impl_to_var[impl] = v
rank += 1
var_names.append(v)
if impl.machine and impl.machine != 'src':
impls_for_machine_group[machine_groups.get(impl.machine, 0)].append(v)
for d in deps_in_use(impl, arch):
debug(_("Considering dependency %s"), d)
add_iface(d.interface, arch.child_arch)
# Must choose one version of d if impl is selected
find_dependency_candidates(v, d)
if closest_match:
dummy_impl = _DummyImpl()
dummy_var = problem.add_variable(ImplInfo(iface, dummy_impl, arch, dummy = True))
var_names.append(dummy_var)
impl_to_var[dummy_impl] = dummy_var
filtered_impls.append(dummy_impl)
# Only one implementation of this interface can be selected
if uri == root_interface:
if var_names:
clause = problem.at_most_one(var_names)
problem.add_clause(var_names) # at least one
else:
problem.impossible()
clause = False
elif var_names:
clause = problem.at_most_one(var_names)
else:
# Don't need to add to group_clause_for because we should
# never get a possible selection involving this.
return
assert clause is not True
assert clause is not None
if clause is not False:
group_clause_for[uri] = clause
def add_command_iface(uri, arch, command_name):
"""Add every <command> in interface 'uri' with this name.
Each one depends on the corresponding implementation and only
one can be selected. If closest_match is on, include a dummy
command that can always be selected."""
# Check whether we've already processed this (interface,command) pair
existing = group_clause_for_command.get((uri, command_name), None)
if existing is not None:
return existing.lits
# First ensure that the interface itself has been processed
# We'll reuse the ordering of the implementations to order
# the commands too.
add_iface(uri, arch)
iface = iface_cache.get_interface(uri)
filtered_impls = impls_for_iface[iface]
var_names = []
for impl in filtered_impls:
command = impl.commands.get(command_name, None)
if not command:
if not isinstance(impl, _DummyImpl):
# Mark implementation as unselectable
problem.add_clause([sat.neg(impl_to_var[impl])])
continue
# We have a candidate <command>. Require that if it's selected
# then we select the corresponding <implementation> too.
command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch))
problem.add_clause([impl_to_var[impl], sat.neg(command_var)])
var_names.append(command_var)
runner = command.get_runner()
for d in deps_in_use(command, arch):
if d is runner:
# With a <runner>, we depend on a command rather than on an
# implementation. This allows us to support recursive <runner>s, etc.
debug(_("Considering command runner %s"), d)
runner_command_name = _get_command_name(d)
runner_vars = add_command_iface(d.interface, arch.child_arch, runner_command_name)
# If the parent command is chosen, one of the candidate runner commands
# must be too. If there aren't any, then this command is unselectable.
problem.add_clause([sat.neg(command_var)] + runner_vars)
else:
debug(_("Considering command dependency %s"), d)
add_iface(d.interface, arch.child_arch)
# Must choose one version of d if impl is selected
find_dependency_candidates(command_var, d)
# Tell the user why we couldn't use this version
if self.record_details:
def new_reason(impl, old_reason):
if command_name in impl.commands:
return old_reason
return old_reason or (_('No %s command') % command_name)
self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]]
if closest_match:
dummy_command = problem.add_variable(None)
var_names.append(dummy_command)
if var_names:
# Can't select more than one of them.
assert (uri, command_name) not in group_clause_for_command
group_clause_for_command[(uri, command_name)] = problem.at_most_one(var_names)
return var_names
if command_name is None:
add_iface(root_interface, root_arch)
else:
commands = add_command_iface(root_interface, root_arch, command_name)
if len(commands) > int(closest_match):
# (we have at least one non-dummy command)
problem.add_clause(commands) # At least one
else:
# (note: might be because we haven't cached it yet)
info("No %s <command> in %s", command_name, root_interface)
impls = impls_for_iface[iface_cache.get_interface(root_interface)]
if impls == [] or (len(impls) == 1 and isinstance(impls[0], _DummyImpl)):
# There were no candidates at all.
if self.config.network_use == model.network_offline:
self._failure_reason = _("Interface '%s' has no usable implementations in the cache (and 0install is in off-line mode)") % root_interface
else:
self._failure_reason = _("Interface '%s' has no usable implementations") % root_interface
else:
# We had some candidates implementations, but none for the command we need
self._failure_reason = _("Interface '%s' cannot be executed directly; it is just a library "
"to be used by other programs (or missing '%s' command)") % (root_interface, command_name)
problem.impossible()
# Require m<group> to be true if we select an implementation in that group
m_groups = []
for machine_group, impls in impls_for_machine_group.iteritems():
m_group = 'm%d' % machine_group
group_var = problem.add_variable(m_group)
if impls:
for impl in impls:
problem.add_clause([group_var, sat.neg(impl)])
m_groups.append(group_var)
if m_groups:
m_groups_clause = problem.at_most_one(m_groups)
else:
m_groups_clause = None
def decide():
"""Recurse through the current selections until we get to an interface with
no chosen version, then tell the solver to try the best version from that."""
def find_undecided_dep(impl_or_command, arch):
# Check for undecided dependencies of impl_or_command
for dep in deps_in_use(impl_or_command, arch):
if dep.qdom.name == 'runner':
dep_lit = find_undecided_command(dep.interface, _get_command_name(dep))
else:
dep_lit = find_undecided(dep.interface)
if dep_lit is not None:
return dep_lit
return None
seen = set()
def find_undecided(uri):
if uri in seen:
return # Break cycles
seen.add(uri)
group = group_clause_for[uri]
#print "Group for %s = %s" % (uri, group)
lit = group.current
if lit is None:
return group.best_undecided()
# else there was only one choice anyway
# Check for undecided dependencies
lit_info = problem.get_varinfo_for_lit(lit).obj
return find_undecided_dep(lit_info.impl, lit_info.arch)
def find_undecided_command(uri, name):
if name is None: return find_undecided(uri)
group = group_clause_for_command[(uri, name)]
lit = group.current
if lit is None:
return group.best_undecided()
# else we've already chosen which <command> to use
# Check for undecided command-specific dependencies, and then for
# implementation dependencies.
lit_info = problem.get_varinfo_for_lit(lit).obj
if lit_info is None:
assert closest_match
return None # (a dummy command added for better diagnostics; has no dependencies)
return find_undecided_dep(lit_info.command, lit_info.arch) or \
find_undecided_dep(lit_info.impl, lit_info.arch)
best = find_undecided_command(root_interface, command_name)
if best is not None:
return best
# If we're chosen everything we need, we can probably
# set everything else to False.
for group in group_clause_for.values() + group_clause_for_command.values() + [m_groups_clause]:
if group.current is None:
best = group.best_undecided()
if best is not None:
return sat.neg(best)
return None # Failed to find any valid combination
ready = problem.run_solver(decide) is True
if not ready and not closest_match:
# We failed while trying to do a real solve.
# Try a closest match solve to get a better
# error report for the user.
self.solve(root_interface, root_arch, command_name = command_name, closest_match = True)
else:
self.ready = ready and not closest_match
self.selections = selections.Selections(None)
self.selections.interface = root_interface
sels = self.selections.selections
for uri, group in group_clause_for.iteritems():
if group.current is not None:
lit_info = problem.get_varinfo_for_lit(group.current).obj
if lit_info.is_dummy:
sels[lit_info.iface.uri] = None
else:
impl = lit_info.impl
deps = self.requires[lit_info.iface] = []
for dep in deps_in_use(lit_info.impl, lit_info.arch):
deps.append(dep)
sels[lit_info.iface.uri] = selections.ImplSelection(lit_info.iface.uri, impl, deps)
def add_command(iface, name):
sel = sels.get(iface, None)
if sel:
command = sel.impl.commands[name]
self.selections.commands.append(command)
runner = command.get_runner()
if runner:
add_command(runner.metadata['interface'], _get_command_name(runner))
if command_name is not None:
add_command(root_interface, command_name)
def add_command_iface(uri, arch, command_name):
"""Add every <command> in interface 'uri' with this name.
Each one depends on the corresponding implementation and only
one can be selected."""
# First ensure that the interface itself has been processed
# We'll reuse the ordering of the implementations to order
# the commands too.
add_iface(uri, arch)
iface = iface_cache.get_interface(uri)
filtered_impls = impls_for_iface[iface]
var_names = []
for impl in filtered_impls:
command = impl.commands.get(command_name, None)
if not command:
if not isinstance(impl, _DummyImpl):
# Mark implementation as unselectable
problem.add_clause([sat.neg(impl_to_var[impl])])
continue
# We have a candidate <command>. Require that if it's selected
# then we select the corresponding <implementation> too.
command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch))
problem.add_clause([impl_to_var[impl], sat.neg(command_var)])
var_names.append(command_var)
runner = command.get_runner()
for d in deps_in_use(command, arch):
if d is runner:
# With a <runner>, we depend on a command rather than on an
# implementation. This allows us to support recursive <runner>s, etc.
debug(_("Considering command runner %s"), d)
runner_command_name = _get_command_name(d)
runner_vars = add_command_iface(d.interface, arch.child_arch, runner_command_name)
if closest_match:
dummy_command = problem.add_variable(None)
runner_vars.append(dummy_command)
# If the parent command is chosen, one of the candidate runner commands
# must be too. If there aren't any, then this command is unselectable.
problem.add_clause([sat.neg(command_var)] + runner_vars)
if runner_vars:
# Can't select more than one of them.
group_clause_for_command[(d.interface, runner_command_name)] = problem.at_most_one(runner_vars)
else:
debug(_("Considering command dependency %s"), d)
add_iface(d.interface, arch.child_arch)
# Must choose one version of d if impl is selected
find_dependency_candidates(command_var, d)
# Tell the user why we couldn't use this version
if self.record_details:
def new_reason(impl, old_reason):
if command_name in impl.commands:
return old_reason
return old_reason or (_('No %s command') % command_name)
self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]]
return var_names
def decide(): """Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): if dep.qdom.name == 'runner': dep_lit = find_undecided_command(dep.interface, _get_command_name(dep)) else: dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) group = group_clause_for[uri] #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in group_clause_for.values() + group_clause_for_command.values() + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination
def solve(self, root_interface, root_arch, command_name = 'run', closest_match = False):
"""@type root_interface: str
@type root_arch: L{zeroinstall.injector.arch.Architecture}
@type command_name: str | None
@type closest_match: bool"""
# closest_match is used internally. It adds a lowest-ranked
# by valid implementation to every interface, so we can always
# select something. Useful for diagnostics.
# The basic plan is this:
# 1. Scan the root interface and all dependencies recursively, building up a SAT problem.
# 2. Solve the SAT problem. Whenever there are multiple options, try the most preferred one first.
# 3. Create a Selections object from the results.
#
# All three involve recursively walking the tree in a similar way:
# 1) we follow every dependency of every implementation (order not important)
# 2) we follow every dependency of every selected implementation (better versions first)
# 3) we follow every dependency of every selected implementation (order doesn't matter)
#
# In all cases, a dependency may be on an <implementation> or on a specific <command>.
# TODO: We need some way to figure out which feeds to include.
# Currently, we include any feed referenced from anywhere but
# this is probably too much. We could insert a dummy optimial
# implementation in stale/uncached feeds and see whether it
# selects that.
iface_cache = self.config.iface_cache
problem = sat.SATProblem()
# For each (interface, impl) we have a sat variable which, if true, means that we have selected
# that impl as the implementation of the interface. We have to index on interface and impl (not
# just impl) because the same feed can provide implementations for different interfaces. This
# happens, for example, when an interface is renamed and the new interface imports the old feed:
# old versions of a program are likely to use the old interface name while new ones use the new
# name.
iface_to_vars = collections.defaultdict(lambda: {}) # Iface -> (Impl -> sat var)
self.feeds_used = set()
self.requires = {}
self.ready = False
self.details = {} if self.record_details or closest_match else False
self.selections = None
# Only set if there is an error
self._impls_for_iface = None
self._iface_to_vars = None
self._problem = None
ifaces_processed = set()
machine_groups = arch.machine_groups
impls_for_machine_group = {0 : []} # Machine group (e.g. "64") to [impl] in that group
for machine_group in machine_groups.values():
impls_for_machine_group[machine_group] = []
impls_for_iface = {} # Iface -> [impl]
# For each interface, the group clause says we can't select two implementations of it at once.
# We use this map at the end to find out what was actually selected.
group_clause_for = {} # Iface URI -> AtMostOneClause
group_clause_for_command = {} # (Iface URI, command name) -> AtMostOneClause | bool
# Return the dependencies of impl that we should consider.
# Skips dependencies if the use flag isn't what we need.
# (note: impl may also be a model.Command)
def deps_in_use(impl, arch):
for dep in impl.requires:
use = dep.metadata.get("use", None)
if use not in arch.use:
continue
# Ignore dependency if 'os' attribute is present and doesn't match
os = dep.metadata.get("os", None)
if os and os not in arch.os_ranks:
continue
yield dep
def clone_command_for(command, arch):
# This is a bit messy. We need to make a copy of the command, without the
# unnecessary <requires> elements.
all_dep_elems = set(dep.qdom for dep in command.requires)
required_dep_elems = set(dep.qdom for dep in deps_in_use(command, arch))
if all_dep_elems == required_dep_elems:
return command # No change
dep_elems_to_remove = all_dep_elems - required_dep_elems
old_root = command.qdom
new_qdom = qdom.Element(old_root.uri, old_root.name, old_root.attrs)
new_qdom.childNodes = [node for node in command.qdom.childNodes if
node not in dep_elems_to_remove]
return model.Command(new_qdom, command._local_dir)
# Must have already done add_iface on dependency.interface.
# If dependency is essential:
# Add a clause so that if requiring_impl_var is True then an implementation
# matching 'dependency' must also be selected.
# If dependency is optional:
# Require that no incompatible version is selected.
# This ignores any 'command' required. Handle that separately.
def find_dependency_candidates(requiring_impl_var, dependency):
def meets_restrictions(candidate):
for r in dependency.restrictions:
if not r.meets_restriction(candidate):
#warn("%s rejected due to %s", candidate.get_version(), r)
return False
return True
essential = dependency.importance == model.Dependency.Essential
dep_iface = iface_cache.get_interface(dependency.interface)
dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ...
impl_to_var = iface_to_vars[dep_iface] # Impl -> sat var
for candidate in impls_for_iface[dep_iface]:
if (candidate.__class__ is _DummyImpl) or meets_restrictions(candidate):
if essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
dep_union.append(c_var)
# else we filtered that version out, so ignore it
else:
# Candidate doesn't meet our requirements
# If the dependency is optional add a rule to make sure we don't
# select this candidate.
# (for essential dependencies this isn't necessary because we must
# select a good version and we can't select two)
if not essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
problem.add_clause(dep_union + [sat.neg(c_var)])
# else we filtered that version out, so ignore it
if essential:
problem.add_clause(dep_union)
def is_unusable(impl, restrictions, arch):
"""@return: whether this implementation is unusable.
@rtype: bool"""
return get_unusable_reason(impl, restrictions, arch) != None
def get_unusable_reason(impl, restrictions, arch):
"""
@param impl: Implementation to test.
@param restrictions: user-provided restrictions (extra_restrictions)
@type restrictions: [L{model.Restriction}]
@return: The reason why this impl is unusable, or None if it's OK.
@rtype: str
@note: The restrictions are for the interface being requested, not the feed
of the implementation; they may be different when feeds are being used."""
for r in restrictions:
if not r.meets_restriction(impl):
return r.reason
stability = impl.get_stability()
if stability <= model.buggy:
return stability.name
if (self.config.network_use == model.network_offline or not impl.download_sources) and not impl.is_available(self.config.stores):
if not impl.download_sources:
return _("No retrieval methods")
for method in impl.download_sources:
if not method.requires_network:
break
else:
return _("Not cached and we are off-line")
if impl.os not in arch.os_ranks:
return _("Unsupported OS")
if impl.machine not in arch.machine_ranks:
if impl.machine == 'src':
return _("Source code")
elif 'src' in arch.machine_ranks:
return _("Not source code")
return _("Unsupported machine type")
return None
def usable_feeds(iface, arch):
"""Return all feeds for iface that support arch.
@rtype: generator(ZeroInstallFeed)"""
yield iface.uri
for f in iface_cache.get_feed_imports(iface):
# Note: when searching for src, None is not in machine_ranks
if f.os in arch.os_ranks and \
(f.machine is None or f.machine in arch.machine_ranks):
yield f.uri
else:
logger.debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
{'feed': f, 'os': f.os, 'machine': f.machine})
# If requiring_var is True then all of requirer's dependencies must be satisfied.
# requirer can be a <command> or an <implementation>
def process_dependencies(requiring_var, requirer, arch):
for d in deps_in_use(requirer, arch):
#logger.debug(_("Considering command dependency %s"), d)
add_iface(d.interface, arch.child_arch)
for c in d.get_required_commands():
# We depend on a specific command within the implementation.
command_vars = add_command_iface(d.interface, arch.child_arch, c)
# If the parent command/impl is chosen, one of the candidate commands
# must be too. If there aren't any, then this command is unselectable.
problem.add_clause([sat.neg(requiring_var)] + command_vars)
# Must choose one version of d if impl is selected
find_dependency_candidates(requiring_var, d)
replacement_for = {} # Interface -> Replacement Interface
def add_iface(uri, arch):
"""Name implementations from feed and assert that only one can be selected."""
if uri in ifaces_processed: return
ifaces_processed.add(uri)
iface = iface_cache.get_interface(uri)
main_feed = iface_cache.get_feed(uri)
if main_feed:
replacement = main_feed.get_replaced_by()
if replacement is not None:
replacement_for[iface] = iface_cache.get_interface(replacement)
impls = []
for f in usable_feeds(iface, arch):
self.feeds_used.add(f)
logger.debug(_("Processing feed %s"), f)
try:
feed = iface_cache.get_feed(f)
if feed is None: continue
#if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for:
# info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f})
if feed.implementations:
impls.extend(feed.implementations.values())
distro_feed_url = feed.get_distro_feed()
if distro_feed_url:
self.feeds_used.add(distro_feed_url)
distro_feed = iface_cache.get_feed(distro_feed_url)
if distro_feed.implementations:
impls.extend(distro_feed.implementations.values())
except MissingLocalFeed as ex:
logger.warning(_("Missing local feed; if it's no longer required, remove it with:") +
'\n0install remove-feed ' + iface.uri + ' ' + f,
{'feed': f, 'interface': iface, 'exception': ex})
except model.SafeException as ex:
logger.warning(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f, 'interface': iface, 'exception': ex})
#raise
except Exception as ex:
import logging
logger.warning(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f, 'interface': iface, 'exception': ex},
exc_info = True if logger.isEnabledFor(logging.INFO) else None)
impls.sort(key = lambda impl: self.get_rating(iface, impl, arch), reverse = True)
impls_for_iface[iface] = filtered_impls = []
my_extra_restrictions = self.extra_restrictions.get(iface, [])
if self.details is not False:
self.details[iface] = [(impl, get_unusable_reason(impl, my_extra_restrictions, arch)) for impl in impls]
impl_to_var = iface_to_vars[iface] # Impl -> sat var
var_names = []
for impl in impls:
if is_unusable(impl, my_extra_restrictions, arch):
continue
filtered_impls.append(impl)
assert impl not in impl_to_var, impl
v = problem.add_variable(ImplInfo(iface, impl, arch))
impl_to_var[impl] = v
var_names.append(v)
if impl.machine and impl.machine != 'src':
impls_for_machine_group[machine_groups.get(impl.machine, 0)].append(v)
process_dependencies(v, impl, arch)
if closest_match:
dummy_impl = _DummyImpl()
dummy_var = problem.add_variable(ImplInfo(iface, dummy_impl, arch, dummy = True))
var_names.append(dummy_var)
impl_to_var[dummy_impl] = dummy_var
filtered_impls.append(dummy_impl)
# Only one implementation of this interface can be selected
if uri == root_interface:
if var_names:
clause = problem.at_most_one(var_names)
problem.add_clause(var_names) # at least one
else:
problem.impossible()
clause = False
elif var_names:
clause = problem.at_most_one(var_names)
else:
# Don't need to add to group_clause_for because we should
# never get a possible selection involving this.
return
assert clause is not True
assert clause is not None
if clause is not False:
group_clause_for[uri] = clause
def add_command_iface(uri, arch, command_name):
"""Add every <command> in interface 'uri' with this name.
Each one depends on the corresponding implementation and only
one can be selected. If closest_match is on, include a dummy
command that can always be selected."""
# Check whether we've already processed this (interface,command) pair
existing = group_clause_for_command.get((uri, command_name), None)
if existing is not None:
return existing.lits
# First ensure that the interface itself has been processed
# We'll reuse the ordering of the implementations to order
# the commands too.
add_iface(uri, arch)
iface = iface_cache.get_interface(uri)
filtered_impls = impls_for_iface[iface]
impl_to_var = iface_to_vars[iface] # Impl -> sat var
var_names = []
for impl in filtered_impls:
command = impl.commands.get(command_name, None)
if not command:
if not isinstance(impl, _DummyImpl):
# Mark implementation as unselectable
problem.add_clause([sat.neg(impl_to_var[impl])])
continue
# We have a candidate <command>. Require that if it's selected
# then we select the corresponding <implementation> too.
command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch))
problem.add_clause([impl_to_var[impl], sat.neg(command_var)])
var_names.append(command_var)
process_dependencies(command_var, command, arch)
# Tell the user why we couldn't use this version
if self.details is not False:
def new_reason(impl, old_reason):
if command_name in impl.commands:
return old_reason
return old_reason or (_('No %s command') % command_name)
self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]]
if closest_match:
dummy_command = problem.add_variable(None)
var_names.append(dummy_command)
if var_names:
# Can't select more than one of them.
assert (uri, command_name) not in group_clause_for_command
group_clause_for_command[(uri, command_name)] = problem.at_most_one(var_names)
return var_names
if command_name is None:
add_iface(root_interface, root_arch)
else:
commands = add_command_iface(root_interface, root_arch, command_name)
if len(commands) > int(closest_match):
# (we have at least one non-dummy command)
problem.add_clause(commands) # At least one
else:
# (note: might be because we haven't cached it yet)
logger.info("No %s <command> in %s", command_name, root_interface)
problem.impossible()
# Require m<group> to be true if we select an implementation in that group
m_groups = []
for machine_group, impls in impls_for_machine_group.items():
m_group = 'm%d' % machine_group
group_var = problem.add_variable(m_group)
if impls:
for impl in impls:
problem.add_clause([group_var, sat.neg(impl)])
m_groups.append(group_var)
if m_groups:
m_groups_clause = problem.at_most_one(m_groups)
else:
m_groups_clause = None
# Can't select an implementation of an interface and of its replacement
for original, replacement in replacement_for.items():
if original == replacement:
logger.warning("Interface %s replaced-by itself!", original)
continue
rep_impls = iface_to_vars.get(replacement, None)
if rep_impls is None:
# We didn't even look at the replacement interface, so no risk here
continue
# Must select one implementation out of all candidates from both interface.
# Dummy implementations don't conflict, though.
all_impls = []
for impl, var in rep_impls.items():
if not isinstance(impl, _DummyImpl):
all_impls.append(var)
for impl, var in iface_to_vars[original].items():
if not isinstance(impl, _DummyImpl):
all_impls.append(var)
if all_impls:
problem.at_most_one(all_impls)
# else: neither feed has any usable impls anyway
def decide():
"""This is called by the SAT solver when it cannot simplify the problem further.
Our job is to find the most-optimal next selection to try.
Recurse through the current selections until we get to an interface with
no chosen version, then tell the solver to try the best version from that."""
def find_undecided_dep(impl_or_command, arch):
# Check for undecided dependencies of impl_or_command
for dep in deps_in_use(impl_or_command, arch):
# Restrictions don't express that we do or don't want the
# dependency, so skip them here.
if dep.importance == model.Dependency.Restricts: continue
for c in dep.get_required_commands():
dep_lit = find_undecided_command(dep.interface, c)
if dep_lit is not None:
return dep_lit
dep_lit = find_undecided(dep.interface)
if dep_lit is not None:
return dep_lit
return None
seen = set()
def find_undecided(uri):
if uri in seen:
return # Break cycles
seen.add(uri)
group = group_clause_for.get(uri, None)
if group is None:
# (can be None if the selected impl has an optional dependency on
# a feed with no implementations)
return
#print "Group for %s = %s" % (uri, group)
lit = group.current
if lit is None:
return group.best_undecided()
# else there was only one choice anyway
# Check for undecided dependencies
lit_info = problem.get_varinfo_for_lit(lit).obj
return find_undecided_dep(lit_info.impl, lit_info.arch)
def find_undecided_command(uri, name):
if name is None: return find_undecided(uri)
group = group_clause_for_command[(uri, name)]
lit = group.current
if lit is None:
return group.best_undecided()
# else we've already chosen which <command> to use
# Check for undecided command-specific dependencies, and then for
# implementation dependencies.
lit_info = problem.get_varinfo_for_lit(lit).obj
if lit_info is None:
assert closest_match
return None # (a dummy command added for better diagnostics; has no dependencies)
return find_undecided_dep(lit_info.command, lit_info.arch) or \
find_undecided_dep(lit_info.impl, lit_info.arch)
best = find_undecided_command(root_interface, command_name)
if best is not None:
return best
# If we're chosen everything we need, we can probably
# set everything else to False.
for group in list(group_clause_for.values()) + list(group_clause_for_command.values()) + [m_groups_clause]:
if group.current is None:
best = group.best_undecided()
if best is not None:
return sat.neg(best)
return None # Failed to find any valid combination
ready = problem.run_solver(decide) is True
if not ready and not closest_match:
# We failed while trying to do a real solve.
# Try a closest match solve to get a better
# error report for the user.
self.solve(root_interface, root_arch, command_name = command_name, closest_match = True)
else:
self.ready = ready and not closest_match
self.selections = selections.Selections(None)
self.selections.interface = root_interface
if not self.ready:
# Store some things useful for get_failure_reason()
self._impls_for_iface = impls_for_iface
self._iface_to_vars = iface_to_vars
self._problem = problem
sels = self.selections.selections
commands_needed = []
# Populate sels with the selected implementations.
# Also, note down all the commands we need.
for uri, group in group_clause_for.items():
if group.current is not None:
lit_info = problem.get_varinfo_for_lit(group.current).obj
if lit_info.is_dummy:
sels[lit_info.iface.uri] = None
else:
# We selected an implementation for interface 'uri'
impl = lit_info.impl
for b in impl.bindings:
c = b.command
if c is not None:
commands.append((uri, c))
deps = self.requires[lit_info.iface] = []
for dep in deps_in_use(lit_info.impl, lit_info.arch):
deps.append(dep)
for c in dep.get_required_commands():
commands_needed.append((dep.interface, c))
sel = sels[lit_info.iface.uri] = selections.ImplSelection(lit_info.iface.uri, impl, deps)
sel.__arch = lit_info.arch
# Now all all the commands in too.
def add_command(iface, name):
sel = sels.get(iface, None)
if sel:
command = sel.impl.commands[name]
if name in sel._used_commands:
return # Already added
sel._used_commands[name] = clone_command_for(command, sel.__arch)
for dep in command.requires:
for dep_command_name in dep.get_required_commands():
add_command(dep.interface, dep_command_name)
# A <command> can depend on another <command> in the same interface
# (e.g. the tests depending on the main program)
for b in command.bindings:
c = b.command
if c is not None:
add_command(iface, c)
for iface, command in commands_needed:
add_command(iface, command)
if command_name is not None:
self.selections.command = command_name
add_command(root_interface, command_name)
if root_interface not in sels:
sels[root_interface] = None # Useful for get_failure_reason()
def solve(self,
root_interface,
root_arch,
command_name='run',
closest_match=False):
# closest_match is used internally. It adds a lowest-ranked
# by valid implementation to every interface, so we can always
# select something. Useful for diagnostics.
# The basic plan is this:
# 1. Scan the root interface and all dependencies recursively, building up a SAT problem.
# 2. Solve the SAT problem. Whenever there are multiple options, try the most preferred one first.
# 3. Create a Selections object from the results.
#
# All three involve recursively walking the tree in a similar way:
# 1) we follow every dependency of every implementation (order not important)
# 2) we follow every dependency of every selected implementation (better versions first)
# 3) we follow every dependency of every selected implementation (order doesn't matter)
#
# In all cases, a dependency may be on an <implementation> or on a specific <command>.
# TODO: We need some way to figure out which feeds to include.
# Currently, we include any feed referenced from anywhere but
# this is probably too much. We could insert a dummy optimial
# implementation in stale/uncached feeds and see whether it
# selects that.
iface_cache = self.config.iface_cache
problem = sat.SATProblem()
impl_to_var = {} # Impl -> sat var
self.feeds_used = set()
self.requires = {}
self.ready = False
self.details = self.record_details and {}
self.selections = None
self._failure_reason = None
ifaces_processed = set()
impls_for_machine_group = {
0: []
} # Machine group (e.g. "64") to [impl] in that group
for machine_group in machine_groups.values():
impls_for_machine_group[machine_group] = []
impls_for_iface = {} # Iface -> [impl]
group_clause_for = {} # Iface URI -> AtMostOneClause | bool
group_clause_for_command = {
} # (Iface URI, command name) -> AtMostOneClause | bool
# Return the dependencies of impl that we should consider.
# Skips dependencies if the use flag isn't what we need.
# (note: impl may also be a model.Command)
def deps_in_use(impl, arch):
for dep in impl.requires:
use = dep.metadata.get("use", None)
if use not in arch.use:
continue
yield dep
# Must have already done add_iface on dependency.interface.
# If dependency is essential:
# Add a clause so that if requiring_impl_var is True then an implementation
# matching 'dependency' must also be selected.
# If dependency is optional:
# Require that no incompatible version is selected.
# This ignores any 'command' required. Handle that separately.
def find_dependency_candidates(requiring_impl_var, dependency):
def meets_restrictions(candidate):
for r in dependency.restrictions:
if not r.meets_restriction(candidate):
#warn("%s rejected due to %s", candidate.get_version(), r)
return False
return True
essential = dependency.importance == model.Dependency.Essential
dep_iface = iface_cache.get_interface(dependency.interface)
dep_union = [sat.neg(requiring_impl_var)
] # Either requiring_impl_var is False, or ...
for candidate in impls_for_iface[dep_iface]:
if (candidate.__class__ is
_DummyImpl) or meets_restrictions(candidate):
if essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
dep_union.append(c_var)
# else we filtered that version out, so ignore it
else:
# Candidate doesn't meet our requirements
# If the dependency is optional add a rule to make sure we don't
# select this candidate.
# (for essential dependencies this isn't necessary because we must
# select a good version and we can't select two)
if not essential:
c_var = impl_to_var.get(candidate, None)
if c_var is not None:
problem.add_clause(dep_union + [sat.neg(c_var)])
# else we filtered that version out, so ignore it
if essential:
problem.add_clause(dep_union)
def is_unusable(impl, restrictions, arch):
"""@return: whether this implementation is unusable.
@rtype: bool"""
return get_unusable_reason(impl, restrictions, arch) != None
def get_unusable_reason(impl, restrictions, arch):
"""
@param impl: Implementation to test.
@type restrictions: [L{model.Restriction}]
@return: The reason why this impl is unusable, or None if it's OK.
@rtype: str
@note: The restrictions are for the interface being requested, not the feed
of the implementation; they may be different when feeds are being used."""
for r in restrictions:
if not r.meets_restriction(impl):
return _(
"Incompatible with another selected implementation")
stability = impl.get_stability()
if stability <= model.buggy:
return stability.name
if (self.config.network_use == model.network_offline
or not impl.download_sources) and not impl.is_available(
self.config.stores):
if not impl.download_sources:
return _("No retrieval methods")
return _("Not cached and we are off-line")
if impl.os not in arch.os_ranks:
return _("Unsupported OS")
if impl.machine not in arch.machine_ranks:
if impl.machine == 'src':
return _("Source code")
return _("Unsupported machine type")
return None
def usable_feeds(iface, arch):
"""Return all feeds for iface that support arch.
@rtype: generator(ZeroInstallFeed)"""
yield iface.uri
for f in iface_cache.get_feed_imports(iface):
# Note: when searching for src, None is not in machine_ranks
if f.os in arch.os_ranks and \
(f.machine is None or f.machine in arch.machine_ranks):
yield f.uri
else:
debug(
_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"
), {
'feed': f,
'os': f.os,
'machine': f.machine
})
# If requiring_var is True then all of requirer's dependencies must be satisfied.
# requirer can be a <command> or an <implementation>
def process_dependencies(requiring_var, requirer, arch):
for d in deps_in_use(requirer, arch):
debug(_("Considering command dependency %s"), d)
add_iface(d.interface, arch.child_arch)
for c in d.get_required_commands():
# We depend on a specific command within the implementation.
command_vars = add_command_iface(d.interface,
arch.child_arch, c)
# If the parent command/impl is chosen, one of the candidate commands
# must be too. If there aren't any, then this command is unselectable.
problem.add_clause([sat.neg(requiring_var)] + command_vars)
# Must choose one version of d if impl is selected
find_dependency_candidates(requiring_var, d)
def add_iface(uri, arch):
"""Name implementations from feed and assert that only one can be selected."""
if uri in ifaces_processed: return
ifaces_processed.add(uri)
iface = iface_cache.get_interface(uri)
impls = []
for f in usable_feeds(iface, arch):
self.feeds_used.add(f)
debug(_("Processing feed %s"), f)
try:
feed = iface_cache.get_feed(f)
if feed is None: continue
#if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for:
# info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f})
if feed.implementations:
impls.extend(feed.implementations.values())
distro_feed_url = feed.get_distro_feed()
if distro_feed_url:
self.feeds_used.add(distro_feed_url)
distro_feed = iface_cache.get_feed(distro_feed_url)
if distro_feed.implementations:
impls.extend(distro_feed.implementations.values())
except MissingLocalFeed as ex:
warn(
_("Missing local feed; if it's no longer required, remove it with:"
) + '\n0install remove-feed ' + iface.uri + ' ' + f,
{
'feed': f,
'interface': iface,
'exception': ex
})
except Exception as ex:
warn(
_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"
), {
'feed': f,
'interface': iface,
'exception': ex
})
#raise
impls.sort(key=lambda impl: self.get_rating(iface, impl, arch),
reverse=True)
impls_for_iface[iface] = filtered_impls = []
my_extra_restrictions = self.extra_restrictions.get(iface, [])
if self.record_details:
self.details[iface] = [
(impl,
get_unusable_reason(impl, my_extra_restrictions, arch))
for impl in impls
]
var_names = []
for impl in impls:
if is_unusable(impl, my_extra_restrictions, arch):
continue
filtered_impls.append(impl)
if impl in impl_to_var:
# TODO If the same impl comes from original feed and
# the feed with "feed" tag
continue
v = problem.add_variable(ImplInfo(iface, impl, arch))
impl_to_var[impl] = v
var_names.append(v)
if impl.machine and impl.machine != 'src':
impls_for_machine_group[machine_groups.get(
impl.machine, 0)].append(v)
process_dependencies(v, impl, arch)
if closest_match:
dummy_impl = _DummyImpl()
dummy_var = problem.add_variable(
ImplInfo(iface, dummy_impl, arch, dummy=True))
var_names.append(dummy_var)
impl_to_var[dummy_impl] = dummy_var
filtered_impls.append(dummy_impl)
# Only one implementation of this interface can be selected
if uri == root_interface:
if var_names:
clause = problem.at_most_one(var_names)
problem.add_clause(var_names) # at least one
else:
problem.impossible()
clause = False
elif var_names:
clause = problem.at_most_one(var_names)
else:
# Don't need to add to group_clause_for because we should
# never get a possible selection involving this.
return
assert clause is not True
assert clause is not None
if clause is not False:
group_clause_for[uri] = clause
def add_command_iface(uri, arch, command_name):
"""Add every <command> in interface 'uri' with this name.
Each one depends on the corresponding implementation and only
one can be selected. If closest_match is on, include a dummy
command that can always be selected."""
# Check whether we've already processed this (interface,command) pair
existing = group_clause_for_command.get((uri, command_name), None)
if existing is not None:
return existing.lits
# First ensure that the interface itself has been processed
# We'll reuse the ordering of the implementations to order
# the commands too.
add_iface(uri, arch)
iface = iface_cache.get_interface(uri)
filtered_impls = impls_for_iface[iface]
var_names = []
for impl in filtered_impls:
command = impl.commands.get(command_name, None)
if not command:
if not isinstance(impl, _DummyImpl):
# Mark implementation as unselectable
problem.add_clause([sat.neg(impl_to_var[impl])])
continue
# We have a candidate <command>. Require that if it's selected
# then we select the corresponding <implementation> too.
command_var = problem.add_variable(
CommandInfo(command_name, command, impl, arch))
problem.add_clause([impl_to_var[impl], sat.neg(command_var)])
var_names.append(command_var)
process_dependencies(command_var, command, arch)
# Tell the user why we couldn't use this version
if self.record_details:
def new_reason(impl, old_reason):
if command_name in impl.commands:
return old_reason
return old_reason or (_('No %s command') % command_name)
self.details[iface] = [(impl, new_reason(impl, reason))
for (impl,
reason) in self.details[iface]]
if closest_match:
dummy_command = problem.add_variable(None)
var_names.append(dummy_command)
if var_names:
# Can't select more than one of them.
assert (uri, command_name) not in group_clause_for_command
group_clause_for_command[(
uri, command_name)] = problem.at_most_one(var_names)
return var_names
if command_name is None:
add_iface(root_interface, root_arch)
else:
commands = add_command_iface(root_interface, root_arch,
command_name)
if len(commands) > int(closest_match):
# (we have at least one non-dummy command)
problem.add_clause(commands) # At least one
else:
# (note: might be because we haven't cached it yet)
info("No %s <command> in %s", command_name, root_interface)
impls = impls_for_iface[iface_cache.get_interface(
root_interface)]
if impls == [] or (len(impls) == 1
and isinstance(impls[0], _DummyImpl)):
# There were no candidates at all.
if self.config.network_use == model.network_offline:
self._failure_reason = _(
"Interface '%s' has no usable implementations in the cache (and 0install is in off-line mode)"
) % root_interface
else:
self._failure_reason = _(
"Interface '%s' has no usable implementations"
) % root_interface
else:
# We had some candidates implementations, but none for the command we need
self._failure_reason = _(
"Interface '%s' cannot be executed directly; it is just a library "
"to be used by other programs (or missing '%s' command)"
) % (root_interface, command_name)
problem.impossible()
# Require m<group> to be true if we select an implementation in that group
m_groups = []
for machine_group, impls in impls_for_machine_group.iteritems():
m_group = 'm%d' % machine_group
group_var = problem.add_variable(m_group)
if impls:
for impl in impls:
problem.add_clause([group_var, sat.neg(impl)])
m_groups.append(group_var)
if m_groups:
m_groups_clause = problem.at_most_one(m_groups)
else:
m_groups_clause = None
def decide():
"""Recurse through the current selections until we get to an interface with
no chosen version, then tell the solver to try the best version from that."""
def find_undecided_dep(impl_or_command, arch):
# Check for undecided dependencies of impl_or_command
for dep in deps_in_use(impl_or_command, arch):
for c in dep.get_required_commands():
dep_lit = find_undecided_command(dep.interface, c)
if dep_lit is not None:
return dep_lit
dep_lit = find_undecided(dep.interface)
if dep_lit is not None:
return dep_lit
return None
seen = set()
def find_undecided(uri):
if uri in seen:
return # Break cycles
seen.add(uri)
if uri not in group_clause_for:
# TODO Looks like Sweets patch introduced this issue
return
group = group_clause_for[uri]
#print "Group for %s = %s" % (uri, group)
lit = group.current
if lit is None:
return group.best_undecided()
# else there was only one choice anyway
# Check for undecided dependencies
lit_info = problem.get_varinfo_for_lit(lit).obj
return find_undecided_dep(lit_info.impl, lit_info.arch)
def find_undecided_command(uri, name):
if name is None: return find_undecided(uri)
group = group_clause_for_command[(uri, name)]
lit = group.current
if lit is None:
return group.best_undecided()
# else we've already chosen which <command> to use
# Check for undecided command-specific dependencies, and then for
# implementation dependencies.
lit_info = problem.get_varinfo_for_lit(lit).obj
if lit_info is None:
assert closest_match
return None # (a dummy command added for better diagnostics; has no dependencies)
return find_undecided_dep(lit_info.command, lit_info.arch) or \
find_undecided_dep(lit_info.impl, lit_info.arch)
best = find_undecided_command(root_interface, command_name)
if best is not None:
return best
# If we're chosen everything we need, we can probably
# set everything else to False.
for group in group_clause_for.values(
) + group_clause_for_command.values() + [m_groups_clause]:
if group.current is None:
best = group.best_undecided()
if best is not None:
return sat.neg(best)
return None # Failed to find any valid combination
ready = problem.run_solver(decide) is True
if not ready and not closest_match:
# We failed while trying to do a real solve.
# Try a closest match solve to get a better
# error report for the user.
self.solve(root_interface,
root_arch,
command_name=command_name,
closest_match=True)
else:
self.ready = ready and not closest_match
self.selections = selections.Selections(None)
self.selections.interface = root_interface
sels = self.selections.selections
commands_needed = []
# Popular sels with the selected implementations.
# Also, note down all the commands we need.
for uri, group in group_clause_for.iteritems():
if group.current is not None:
lit_info = problem.get_varinfo_for_lit(group.current).obj
if lit_info.is_dummy:
sels[lit_info.iface.uri] = None
else:
# We selected an implementation for interface 'uri'
impl = lit_info.impl
for b in impl.bindings:
c = b.command
if c is not None:
commands.append((uri, c))
deps = self.requires[lit_info.iface] = []
for dep in deps_in_use(lit_info.impl, lit_info.arch):
deps.append(dep)
for c in dep.get_required_commands():
commands_needed.append((dep.interface, c))
sels[lit_info.iface.uri] = selections.ImplSelection(
lit_info.iface.uri, impl, deps)
# Now all all the commands in too.
def add_command(iface, name):
sel = sels.get(iface, None)
if sel:
command = sel.impl.commands[name]
if name in sel._used_commands:
return # Already added
sel._used_commands.add(name)
for dep in command.requires:
for dep_command_name in dep.get_required_commands():
add_command(dep.interface, dep_command_name)
# A <command> can depend on another <command> in the same interface
# (e.g. the tests depending on the main program)
for b in command.bindings:
c = b.command
if c is not None:
add_command(iface, c)
for iface, command in commands_needed:
add_command(iface, command)
if command_name is not None:
self.selections.command = command_name
add_command(root_interface, command_name)
def decide():
"""This is called by the SAT solver when it cannot simplify the problem further.
Our job is to find the most-optimal next selection to try.
Recurse through the current selections until we get to an interface with
no chosen version, then tell the solver to try the best version from that."""
def find_undecided_dep(impl_or_command, arch):
# Check for undecided dependencies of impl_or_command
for dep in deps_in_use(impl_or_command, arch):
# Restrictions don't express that we do or don't want the
# dependency, so skip them here.
if dep.importance == model.Dependency.Restricts: continue
for c in dep.get_required_commands():
dep_lit = find_undecided_command(dep.interface, c)
if dep_lit is not None:
return dep_lit
dep_lit = find_undecided(dep.interface)
if dep_lit is not None:
return dep_lit
return None
seen = set()
def find_undecided(uri):
if uri in seen:
return # Break cycles
seen.add(uri)
group = group_clause_for.get(uri, None)
if group is None:
# (can be None if the selected impl has an optional dependency on
# a feed with no implementations)
return
#print "Group for %s = %s" % (uri, group)
lit = group.current
if lit is None:
return group.best_undecided()
# else there was only one choice anyway
# Check for undecided dependencies
lit_info = problem.get_varinfo_for_lit(lit).obj
return find_undecided_dep(lit_info.impl, lit_info.arch)
def find_undecided_command(uri, name):
if name is None: return find_undecided(uri)
group = group_clause_for_command[(uri, name)]
lit = group.current
if lit is None:
return group.best_undecided()
# else we've already chosen which <command> to use
# Check for undecided command-specific dependencies, and then for
# implementation dependencies.
lit_info = problem.get_varinfo_for_lit(lit).obj
if lit_info is None:
assert closest_match
return None # (a dummy command added for better diagnostics; has no dependencies)
return find_undecided_dep(lit_info.command, lit_info.arch) or \
find_undecided_dep(lit_info.impl, lit_info.arch)
best = find_undecided_command(root_interface, command_name)
if best is not None:
return best
# If we're chosen everything we need, we can probably
# set everything else to False.
for group in list(group_clause_for.values()) + list(
group_clause_for_command.values()) + [m_groups_clause]:
if group.current is None:
best = group.best_undecided()
if best is not None:
return sat.neg(best)
return None # Failed to find any valid combination
def decide():
"""Recurse through the current selections until we get to an interface with
no chosen version, then tell the solver to try the best version from that."""
def find_undecided_dep(impl_or_command, arch):
# Check for undecided dependencies of impl_or_command
for dep in deps_in_use(impl_or_command, arch):
for c in dep.get_required_commands():
dep_lit = find_undecided_command(dep.interface, c)
if dep_lit is not None:
return dep_lit
dep_lit = find_undecided(dep.interface)
if dep_lit is not None:
return dep_lit
return None
seen = set()
def find_undecided(uri):
if uri in seen:
return # Break cycles
seen.add(uri)
if uri not in group_clause_for:
# TODO Looks like Sweets patch introduced this issue
return
group = group_clause_for[uri]
#print "Group for %s = %s" % (uri, group)
lit = group.current
if lit is None:
return group.best_undecided()
# else there was only one choice anyway
# Check for undecided dependencies
lit_info = problem.get_varinfo_for_lit(lit).obj
return find_undecided_dep(lit_info.impl, lit_info.arch)
def find_undecided_command(uri, name):
if name is None: return find_undecided(uri)
group = group_clause_for_command[(uri, name)]
lit = group.current
if lit is None:
return group.best_undecided()
# else we've already chosen which <command> to use
# Check for undecided command-specific dependencies, and then for
# implementation dependencies.
lit_info = problem.get_varinfo_for_lit(lit).obj
if lit_info is None:
assert closest_match
return None # (a dummy command added for better diagnostics; has no dependencies)
return find_undecided_dep(lit_info.command, lit_info.arch) or \
find_undecided_dep(lit_info.impl, lit_info.arch)
best = find_undecided_command(root_interface, command_name)
if best is not None:
return best
# If we're chosen everything we need, we can probably
# set everything else to False.
for group in group_clause_for.values(
) + group_clause_for_command.values() + [m_groups_clause]:
if group.current is None:
best = group.best_undecided()
if best is not None:
return sat.neg(best)
return None # Failed to find any valid combination
def add_command_iface(uri, arch, command_name):
"""Add every <command> in interface 'uri' with this name.
Each one depends on the corresponding implementation and only
one can be selected. If closest_match is on, include a dummy
command that can always be selected."""
# Check whether we've already processed this (interface,command) pair
existing = group_clause_for_command.get((uri, command_name), None)
if existing is not None:
return existing.lits
# First ensure that the interface itself has been processed
# We'll reuse the ordering of the implementations to order
# the commands too.
add_iface(uri, arch)
iface = iface_cache.get_interface(uri)
filtered_impls = impls_for_iface[iface]
var_names = []
for impl in filtered_impls:
command = impl.commands.get(command_name, None)
if not command:
if not isinstance(impl, _DummyImpl):
# Mark implementation as unselectable
problem.add_clause([sat.neg(impl_to_var[impl])])
continue
# We have a candidate <command>. Require that if it's selected
# then we select the corresponding <implementation> too.
command_var = problem.add_variable(
CommandInfo(command_name, command, impl, arch))
problem.add_clause([impl_to_var[impl], sat.neg(command_var)])
var_names.append(command_var)
runner = command.get_runner()
for d in deps_in_use(command, arch):
if d is runner:
# With a <runner>, we depend on a command rather than on an
# implementation. This allows us to support recursive <runner>s, etc.
debug(_("Considering command runner %s"), d)
runner_command_name = _get_command_name(d)
runner_vars = add_command_iface(
d.interface, arch.child_arch, runner_command_name)
# If the parent command is chosen, one of the candidate runner commands
# must be too. If there aren't any, then this command is unselectable.
problem.add_clause([sat.neg(command_var)] +
runner_vars)
else:
debug(_("Considering command dependency %s"), d)
add_iface(d.interface, arch.child_arch)
# Must choose one version of d if impl is selected
find_dependency_candidates(command_var, d)
# Tell the user why we couldn't use this version
if self.record_details:
def new_reason(impl, old_reason):
if command_name in impl.commands:
return old_reason
return old_reason or (_('No %s command') % command_name)
self.details[iface] = [(impl, new_reason(impl, reason))
for (impl,
reason) in self.details[iface]]
if closest_match:
dummy_command = problem.add_variable(None)
var_names.append(dummy_command)
if var_names:
# Can't select more than one of them.
assert (uri, command_name) not in group_clause_for_command
group_clause_for_command[(
uri, command_name)] = problem.at_most_one(var_names)
return var_names