def find_dependency_candidates(requiring_impl_var, dependency): def meets_restrictions(candidate): for r in dependency.restrictions: if not r.meets_restriction(candidate): #warn("%s rejected due to %s", candidate.get_version(), r) return False return True essential = dependency.importance == model.Dependency.Essential dep_iface = iface_cache.get_interface(dependency.interface) dep_union = [sat.neg(requiring_impl_var) ] # Either requiring_impl_var is False, or ... for candidate in impls_for_iface[dep_iface]: if (candidate.__class__ is _DummyImpl) or meets_restrictions(candidate): if essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: dep_union.append(c_var) # else we filtered that version out, so ignore it else: # Candidate doesn't meet our requirements # If the dependency is optional add a rule to make sure we don't # select this candidate. # (for essential dependencies this isn't necessary because we must # select a good version and we can't select two) if not essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: problem.add_clause(dep_union + [sat.neg(c_var)]) # else we filtered that version out, so ignore it if essential: problem.add_clause(dep_union)
def find_dependency_candidates(requiring_impl_var, dependency): def meets_restrictions(candidate): for r in dependency.restrictions: if not r.meets_restriction(candidate): #warn("%s rejected due to %s", candidate.get_version(), r) return False return True essential = dependency.importance == model.Dependency.Essential dep_iface = iface_cache.get_interface(dependency.interface) dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ... for candidate in impls_for_iface[dep_iface]: if (candidate.__class__ is _DummyImpl) or meets_restrictions(candidate): if essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: dep_union.append(c_var) # else we filtered that version out, so ignore it else: # Candidate doesn't meet our requirements # If the dependency is optional add a rule to make sure we don't # select this candidate. # (for essential dependencies this isn't necessary because we must # select a good version and we can't select two) if not essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: problem.add_clause(dep_union + [sat.neg(c_var)]) # else we filtered that version out, so ignore it if essential: problem.add_clause(dep_union)
def add_command_iface(uri, arch, command_name): """Add every <command> in interface 'uri' with this name. Each one depends on the corresponding implementation and only one can be selected. If closest_match is on, include a dummy command that can always be selected.""" # Check whether we've already processed this (interface,command) pair existing = group_clause_for_command.get((uri, command_name), None) if existing is not None: return existing.lits # First ensure that the interface itself has been processed # We'll reuse the ordering of the implementations to order # the commands too. add_iface(uri, arch) iface = iface_cache.get_interface(uri) filtered_impls = impls_for_iface[iface] impl_to_var = iface_to_vars[iface] # Impl -> sat var var_names = [] for impl in filtered_impls: command = impl.commands.get(command_name, None) if not command: if not isinstance(impl, _DummyImpl): # Mark implementation as unselectable problem.add_clause([sat.neg(impl_to_var[impl])]) continue # We have a candidate <command>. Require that if it's selected # then we select the corresponding <implementation> too. command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch)) problem.add_clause([impl_to_var[impl], sat.neg(command_var)]) var_names.append(command_var) process_dependencies(command_var, command, arch) # Tell the user why we couldn't use this version if self.details is not False: def new_reason(impl, old_reason): if command_name in impl.commands: return old_reason return old_reason or (_('No %s command') % command_name) self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]] if closest_match: dummy_command = problem.add_variable(None) var_names.append(dummy_command) if var_names: # Can't select more than one of them. assert (uri, command_name) not in group_clause_for_command group_clause_for_command[(uri, command_name)] = problem.at_most_one(var_names) return var_names
def testCoverage(self): # Try to trigger some edge cases... # An at_most_one clause must be analysed for causing # a conflict. solver = sat.SATProblem() v1 = solver.add_variable("v1") v2 = solver.add_variable("v2") v3 = solver.add_variable("v3") solver.at_most_one([v1, v2]) solver.add_clause([v1, sat.neg(v3)]) solver.add_clause([v2, sat.neg(v3)]) solver.add_clause([v1, v3]) solver.run_solver(lambda: v3)
def process_dependencies(requiring_var, requirer, arch): for d in deps_in_use(requirer, arch): #logger.debug(_("Considering command dependency %s"), d) add_iface(d.interface, arch.child_arch) for c in d.get_required_commands(): # We depend on a specific command within the implementation. command_vars = add_command_iface(d.interface, arch.child_arch, c) # If the parent command/impl is chosen, one of the candidate commands # must be too. If there aren't any, then this command is unselectable. problem.add_clause([sat.neg(requiring_var)] + command_vars) # Must choose one version of d if impl is selected find_dependency_candidates(requiring_var, d)
def process_dependencies(requiring_var, requirer, arch): for d in deps_in_use(requirer, arch): debug(_("Considering command dependency %s"), d) add_iface(d.interface, arch.child_arch) for c in d.get_required_commands(): # We depend on a specific command within the implementation. command_vars = add_command_iface(d.interface, arch.child_arch, c) # If the parent command/impl is chosen, one of the candidate commands # must be too. If there aren't any, then this command is unselectable. problem.add_clause([sat.neg(requiring_var)] + command_vars) # Must choose one version of d if impl is selected find_dependency_candidates(requiring_var, d)
def find_dependency_candidates(requiring_impl_var, dependency): dep_iface = iface_cache.get_interface(dependency.interface) dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ... for candidate in impls_for_iface[dep_iface]: for r in dependency.restrictions: if candidate.__class__ is not _DummyImpl and not r.meets_restriction(candidate): #warn("%s rejected due to %s", candidate.get_version(), r) if candidate.version is not None: break else: c_var = impl_to_var.get(candidate, None) if c_var is not None: dep_union.append(c_var) # else we filtered that version out, so ignore it assert dep_union problem.add_clause(dep_union)
def testWatch(self): solver = sat.SATProblem() a = solver.add_variable('a') b = solver.add_variable('b') c = solver.add_variable('c') # Add a clause. It starts watching the first two variables (a and b). # (use the internal function to avoid variable reordering) solver._add_clause([a, b, c], False, reason = "testing") # b is False, so it switches to watching a and c solver.add_clause([sat.neg(b)]) # Try to trigger bug. solver.add_clause([c]) decisions = [a] solver.run_solver(lambda: decisions.pop()) assert not decisions # All used up assert solver.assigns[a].value == True
def testWatch(self): solver = sat.SATProblem() a = solver.add_variable('a') b = solver.add_variable('b') c = solver.add_variable('c') # Add a clause. It starts watching the first two variables (a and b). # (use the internal function to avoid variable reordering) solver._add_clause([a, b, c], False) # b is False, so it switches to watching a and c solver.add_clause([sat.neg(b)]) # Try to trigger bug. solver.add_clause([c]) decisions = [a] solver.run_solver(lambda: decisions.pop()) assert not decisions # All used up assert solver.assigns[a].value == True
def add_command_iface(uri, arch, command_name): """Add every <command> in interface 'uri' with this name. Each one depends on the corresponding implementation and only one can be selected. If closest_match is on, include a dummy command that can always be selected.""" # Check whether we've already processed this (interface,command) pair existing = group_clause_for_command.get((uri, command_name), None) if existing is not None: return existing.lits # First ensure that the interface itself has been processed # We'll reuse the ordering of the implementations to order # the commands too. add_iface(uri, arch) iface = iface_cache.get_interface(uri) filtered_impls = impls_for_iface[iface] var_names = [] for impl in filtered_impls: command = impl.commands.get(command_name, None) if not command: if not isinstance(impl, _DummyImpl): # Mark implementation as unselectable problem.add_clause([sat.neg(impl_to_var[impl])]) continue # We have a candidate <command>. Require that if it's selected # then we select the corresponding <implementation> too. command_var = problem.add_variable( CommandInfo(command_name, command, impl, arch)) problem.add_clause([impl_to_var[impl], sat.neg(command_var)]) var_names.append(command_var) process_dependencies(command_var, command, arch) # Tell the user why we couldn't use this version if self.record_details: def new_reason(impl, old_reason): if command_name in impl.commands: return old_reason return old_reason or (_('No %s command') % command_name) self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]] if closest_match: dummy_command = problem.add_variable(None) var_names.append(dummy_command) if var_names: # Can't select more than one of them. assert (uri, command_name) not in group_clause_for_command group_clause_for_command[( uri, command_name)] = problem.at_most_one(var_names) return var_names
def decide(): """This is called by the SAT solver when it cannot simplify the problem further. Our job is to find the most-optimal next selection to try. Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): # Restrictions don't express that we do or don't want the # dependency, so skip them here. if dep.importance == model.Dependency.Restricts: continue for c in dep.get_required_commands(): dep_lit = find_undecided_command(dep.interface, c) if dep_lit is not None: return dep_lit dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) group = group_clause_for.get(uri, None) if group is None: # (can be None if the selected impl has an optional dependency on # a feed with no implementations) return #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in list(group_clause_for.values()) + list(group_clause_for_command.values()) + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination
def solve(self, root_interface, root_arch, command_name = 'run', closest_match = False): # closest_match is used internally. It adds a lowest-ranked # by valid implementation to every interface, so we can always # select something. Useful for diagnostics. # TODO: We need some way to figure out which feeds to include. # Currently, we include any feed referenced from anywhere but # this is probably too much. We could insert a dummy optimial # implementation in stale/uncached feeds and see whether it # selects that. iface_cache = self.config.iface_cache problem = sat.SATProblem() impl_to_var = {} # Impl -> sat var self.feeds_used = set() self.requires = {} self.ready = False self.details = self.record_details and {} self.selections = None self._failure_reason = None ifaces_processed = set() impls_for_machine_group = {0 : []} # Machine group (e.g. "64") to [impl] in that group for machine_group in machine_groups.values(): impls_for_machine_group[machine_group] = [] impls_for_iface = {} # Iface -> [impl] group_clause_for = {} # Iface URI -> AtMostOneClause | bool group_clause_for_command = {} # (Iface URI, command name) -> AtMostOneClause | bool # Return the dependencies of impl that we should consider. # Skips dependencies if the use flag isn't what we need. # (note: impl may also be a model.Command) def deps_in_use(impl, arch): for dep in impl.requires: use = dep.metadata.get("use", None) if use not in arch.use: continue yield dep # Must have already done add_iface on dependency.interface. # If dependency is essential: # Add a clause so that if requiring_impl_var is True then an implementation # matching 'dependency' must also be selected. # If dependency is optional: # Require that no incompatible version is selected. def find_dependency_candidates(requiring_impl_var, dependency): def meets_restrictions(candidate): for r in dependency.restrictions: if not r.meets_restriction(candidate): #warn("%s rejected due to %s", candidate.get_version(), r) return False return True essential = dependency.importance == model.Dependency.Essential dep_iface = iface_cache.get_interface(dependency.interface) dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ... for candidate in impls_for_iface[dep_iface]: if (candidate.__class__ is _DummyImpl) or meets_restrictions(candidate): if essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: dep_union.append(c_var) # else we filtered that version out, so ignore it else: # Candidate doesn't meet our requirements # If the dependency is optional add a rule to make sure we don't # select this candidate. # (for essential dependencies this isn't necessary because we must # select a good version and we can't select two) if not essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: problem.add_clause(dep_union + [sat.neg(c_var)]) # else we filtered that version out, so ignore it if essential: problem.add_clause(dep_union) def is_unusable(impl, restrictions, arch): """@return: whether this implementation is unusable. @rtype: bool""" return get_unusable_reason(impl, restrictions, arch) != None def get_unusable_reason(impl, restrictions, arch): """ @param impl: Implementation to test. @type restrictions: [L{model.Restriction}] @return: The reason why this impl is unusable, or None if it's OK. @rtype: str @note: The restrictions are for the interface being requested, not the feed of the implementation; they may be different when feeds are being used.""" for r in restrictions: if not r.meets_restriction(impl): return _("Incompatible with another selected implementation") stability = impl.get_stability() if stability <= model.buggy: return stability.name if (self.config.network_use == model.network_offline or not impl.download_sources) and not impl.is_available(self.config.stores): if not impl.download_sources: return _("No retrieval methods") return _("Not cached and we are off-line") if impl.os not in arch.os_ranks: return _("Unsupported OS") if impl.machine not in arch.machine_ranks: if impl.machine == 'src': return _("Source code") return _("Unsupported machine type") return None def usable_feeds(iface, arch): """Return all feeds for iface that support arch. @rtype: generator(ZeroInstallFeed)""" yield iface.uri for f in iface_cache.get_feed_imports(iface): # Note: when searching for src, None is not in machine_ranks if f.os in arch.os_ranks and \ (f.machine is None or f.machine in arch.machine_ranks): yield f.uri else: debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"), {'feed': f, 'os': f.os, 'machine': f.machine}) def add_iface(uri, arch): """Name implementations from feed and assert that only one can be selected.""" if uri in ifaces_processed: return ifaces_processed.add(uri) iface = iface_cache.get_interface(uri) impls = [] for f in usable_feeds(iface, arch): self.feeds_used.add(f) debug(_("Processing feed %s"), f) try: feed = iface_cache.get_feed(f) if feed is None: continue #if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for: # info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f}) if feed.implementations: impls.extend(feed.implementations.values()) distro_feed_url = feed.get_distro_feed() if distro_feed_url: self.feeds_used.add(distro_feed_url) distro_feed = iface_cache.get_feed(distro_feed_url) if distro_feed.implementations: impls.extend(distro_feed.implementations.values()) except MissingLocalFeed as ex: warn(_("Missing local feed; if it's no longer required, remove it with:") + '\n0install remove-feed ' + iface.uri + ' ' + f, {'feed': f, 'interface': iface, 'exception': ex}) except Exception as ex: warn(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f, 'interface': iface, 'exception': ex}) #raise impls.sort(lambda a, b: self.compare(iface, a, b, arch)) impls_for_iface[iface] = filtered_impls = [] my_extra_restrictions = self.extra_restrictions.get(iface, []) if self.record_details: self.details[iface] = [(impl, get_unusable_reason(impl, my_extra_restrictions, arch)) for impl in impls] rank = 1 var_names = [] for impl in impls: if is_unusable(impl, my_extra_restrictions, arch): continue filtered_impls.append(impl) assert impl not in impl_to_var v = problem.add_variable(ImplInfo(iface, impl, arch)) impl_to_var[impl] = v rank += 1 var_names.append(v) if impl.machine and impl.machine != 'src': impls_for_machine_group[machine_groups.get(impl.machine, 0)].append(v) for d in deps_in_use(impl, arch): debug(_("Considering dependency %s"), d) add_iface(d.interface, arch.child_arch) # Must choose one version of d if impl is selected find_dependency_candidates(v, d) if closest_match: dummy_impl = _DummyImpl() dummy_var = problem.add_variable(ImplInfo(iface, dummy_impl, arch, dummy = True)) var_names.append(dummy_var) impl_to_var[dummy_impl] = dummy_var filtered_impls.append(dummy_impl) # Only one implementation of this interface can be selected if uri == root_interface: if var_names: clause = problem.at_most_one(var_names) problem.add_clause(var_names) # at least one else: problem.impossible() clause = False elif var_names: clause = problem.at_most_one(var_names) else: # Don't need to add to group_clause_for because we should # never get a possible selection involving this. return assert clause is not True assert clause is not None if clause is not False: group_clause_for[uri] = clause def add_command_iface(uri, arch, command_name): """Add every <command> in interface 'uri' with this name. Each one depends on the corresponding implementation and only one can be selected. If closest_match is on, include a dummy command that can always be selected.""" # Check whether we've already processed this (interface,command) pair existing = group_clause_for_command.get((uri, command_name), None) if existing is not None: return existing.lits # First ensure that the interface itself has been processed # We'll reuse the ordering of the implementations to order # the commands too. add_iface(uri, arch) iface = iface_cache.get_interface(uri) filtered_impls = impls_for_iface[iface] var_names = [] for impl in filtered_impls: command = impl.commands.get(command_name, None) if not command: if not isinstance(impl, _DummyImpl): # Mark implementation as unselectable problem.add_clause([sat.neg(impl_to_var[impl])]) continue # We have a candidate <command>. Require that if it's selected # then we select the corresponding <implementation> too. command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch)) problem.add_clause([impl_to_var[impl], sat.neg(command_var)]) var_names.append(command_var) runner = command.get_runner() for d in deps_in_use(command, arch): if d is runner: # With a <runner>, we depend on a command rather than on an # implementation. This allows us to support recursive <runner>s, etc. debug(_("Considering command runner %s"), d) runner_command_name = _get_command_name(d) runner_vars = add_command_iface(d.interface, arch.child_arch, runner_command_name) # If the parent command is chosen, one of the candidate runner commands # must be too. If there aren't any, then this command is unselectable. problem.add_clause([sat.neg(command_var)] + runner_vars) else: debug(_("Considering command dependency %s"), d) add_iface(d.interface, arch.child_arch) # Must choose one version of d if impl is selected find_dependency_candidates(command_var, d) # Tell the user why we couldn't use this version if self.record_details: def new_reason(impl, old_reason): if command_name in impl.commands: return old_reason return old_reason or (_('No %s command') % command_name) self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]] if closest_match: dummy_command = problem.add_variable(None) var_names.append(dummy_command) if var_names: # Can't select more than one of them. assert (uri, command_name) not in group_clause_for_command group_clause_for_command[(uri, command_name)] = problem.at_most_one(var_names) return var_names if command_name is None: add_iface(root_interface, root_arch) else: commands = add_command_iface(root_interface, root_arch, command_name) if len(commands) > int(closest_match): # (we have at least one non-dummy command) problem.add_clause(commands) # At least one else: # (note: might be because we haven't cached it yet) info("No %s <command> in %s", command_name, root_interface) impls = impls_for_iface[iface_cache.get_interface(root_interface)] if impls == [] or (len(impls) == 1 and isinstance(impls[0], _DummyImpl)): # There were no candidates at all. if self.config.network_use == model.network_offline: self._failure_reason = _("Interface '%s' has no usable implementations in the cache (and 0install is in off-line mode)") % root_interface else: self._failure_reason = _("Interface '%s' has no usable implementations") % root_interface else: # We had some candidates implementations, but none for the command we need self._failure_reason = _("Interface '%s' cannot be executed directly; it is just a library " "to be used by other programs (or missing '%s' command)") % (root_interface, command_name) problem.impossible() # Require m<group> to be true if we select an implementation in that group m_groups = [] for machine_group, impls in impls_for_machine_group.iteritems(): m_group = 'm%d' % machine_group group_var = problem.add_variable(m_group) if impls: for impl in impls: problem.add_clause([group_var, sat.neg(impl)]) m_groups.append(group_var) if m_groups: m_groups_clause = problem.at_most_one(m_groups) else: m_groups_clause = None def decide(): """Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): if dep.qdom.name == 'runner': dep_lit = find_undecided_command(dep.interface, _get_command_name(dep)) else: dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) group = group_clause_for[uri] #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in group_clause_for.values() + group_clause_for_command.values() + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination ready = problem.run_solver(decide) is True if not ready and not closest_match: # We failed while trying to do a real solve. # Try a closest match solve to get a better # error report for the user. self.solve(root_interface, root_arch, command_name = command_name, closest_match = True) else: self.ready = ready and not closest_match self.selections = selections.Selections(None) self.selections.interface = root_interface sels = self.selections.selections for uri, group in group_clause_for.iteritems(): if group.current is not None: lit_info = problem.get_varinfo_for_lit(group.current).obj if lit_info.is_dummy: sels[lit_info.iface.uri] = None else: impl = lit_info.impl deps = self.requires[lit_info.iface] = [] for dep in deps_in_use(lit_info.impl, lit_info.arch): deps.append(dep) sels[lit_info.iface.uri] = selections.ImplSelection(lit_info.iface.uri, impl, deps) def add_command(iface, name): sel = sels.get(iface, None) if sel: command = sel.impl.commands[name] self.selections.commands.append(command) runner = command.get_runner() if runner: add_command(runner.metadata['interface'], _get_command_name(runner)) if command_name is not None: add_command(root_interface, command_name)
def add_command_iface(uri, arch, command_name): """Add every <command> in interface 'uri' with this name. Each one depends on the corresponding implementation and only one can be selected.""" # First ensure that the interface itself has been processed # We'll reuse the ordering of the implementations to order # the commands too. add_iface(uri, arch) iface = iface_cache.get_interface(uri) filtered_impls = impls_for_iface[iface] var_names = [] for impl in filtered_impls: command = impl.commands.get(command_name, None) if not command: if not isinstance(impl, _DummyImpl): # Mark implementation as unselectable problem.add_clause([sat.neg(impl_to_var[impl])]) continue # We have a candidate <command>. Require that if it's selected # then we select the corresponding <implementation> too. command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch)) problem.add_clause([impl_to_var[impl], sat.neg(command_var)]) var_names.append(command_var) runner = command.get_runner() for d in deps_in_use(command, arch): if d is runner: # With a <runner>, we depend on a command rather than on an # implementation. This allows us to support recursive <runner>s, etc. debug(_("Considering command runner %s"), d) runner_command_name = _get_command_name(d) runner_vars = add_command_iface(d.interface, arch.child_arch, runner_command_name) if closest_match: dummy_command = problem.add_variable(None) runner_vars.append(dummy_command) # If the parent command is chosen, one of the candidate runner commands # must be too. If there aren't any, then this command is unselectable. problem.add_clause([sat.neg(command_var)] + runner_vars) if runner_vars: # Can't select more than one of them. group_clause_for_command[(d.interface, runner_command_name)] = problem.at_most_one(runner_vars) else: debug(_("Considering command dependency %s"), d) add_iface(d.interface, arch.child_arch) # Must choose one version of d if impl is selected find_dependency_candidates(command_var, d) # Tell the user why we couldn't use this version if self.record_details: def new_reason(impl, old_reason): if command_name in impl.commands: return old_reason return old_reason or (_('No %s command') % command_name) self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]] return var_names
def decide(): """Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): if dep.qdom.name == 'runner': dep_lit = find_undecided_command(dep.interface, _get_command_name(dep)) else: dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) group = group_clause_for[uri] #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in group_clause_for.values() + group_clause_for_command.values() + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination
def solve(self, root_interface, root_arch, command_name = 'run', closest_match = False): """@type root_interface: str @type root_arch: L{zeroinstall.injector.arch.Architecture} @type command_name: str | None @type closest_match: bool""" # closest_match is used internally. It adds a lowest-ranked # by valid implementation to every interface, so we can always # select something. Useful for diagnostics. # The basic plan is this: # 1. Scan the root interface and all dependencies recursively, building up a SAT problem. # 2. Solve the SAT problem. Whenever there are multiple options, try the most preferred one first. # 3. Create a Selections object from the results. # # All three involve recursively walking the tree in a similar way: # 1) we follow every dependency of every implementation (order not important) # 2) we follow every dependency of every selected implementation (better versions first) # 3) we follow every dependency of every selected implementation (order doesn't matter) # # In all cases, a dependency may be on an <implementation> or on a specific <command>. # TODO: We need some way to figure out which feeds to include. # Currently, we include any feed referenced from anywhere but # this is probably too much. We could insert a dummy optimial # implementation in stale/uncached feeds and see whether it # selects that. iface_cache = self.config.iface_cache problem = sat.SATProblem() # For each (interface, impl) we have a sat variable which, if true, means that we have selected # that impl as the implementation of the interface. We have to index on interface and impl (not # just impl) because the same feed can provide implementations for different interfaces. This # happens, for example, when an interface is renamed and the new interface imports the old feed: # old versions of a program are likely to use the old interface name while new ones use the new # name. iface_to_vars = collections.defaultdict(lambda: {}) # Iface -> (Impl -> sat var) self.feeds_used = set() self.requires = {} self.ready = False self.details = {} if self.record_details or closest_match else False self.selections = None # Only set if there is an error self._impls_for_iface = None self._iface_to_vars = None self._problem = None ifaces_processed = set() machine_groups = arch.machine_groups impls_for_machine_group = {0 : []} # Machine group (e.g. "64") to [impl] in that group for machine_group in machine_groups.values(): impls_for_machine_group[machine_group] = [] impls_for_iface = {} # Iface -> [impl] # For each interface, the group clause says we can't select two implementations of it at once. # We use this map at the end to find out what was actually selected. group_clause_for = {} # Iface URI -> AtMostOneClause group_clause_for_command = {} # (Iface URI, command name) -> AtMostOneClause | bool # Return the dependencies of impl that we should consider. # Skips dependencies if the use flag isn't what we need. # (note: impl may also be a model.Command) def deps_in_use(impl, arch): for dep in impl.requires: use = dep.metadata.get("use", None) if use not in arch.use: continue # Ignore dependency if 'os' attribute is present and doesn't match os = dep.metadata.get("os", None) if os and os not in arch.os_ranks: continue yield dep def clone_command_for(command, arch): # This is a bit messy. We need to make a copy of the command, without the # unnecessary <requires> elements. all_dep_elems = set(dep.qdom for dep in command.requires) required_dep_elems = set(dep.qdom for dep in deps_in_use(command, arch)) if all_dep_elems == required_dep_elems: return command # No change dep_elems_to_remove = all_dep_elems - required_dep_elems old_root = command.qdom new_qdom = qdom.Element(old_root.uri, old_root.name, old_root.attrs) new_qdom.childNodes = [node for node in command.qdom.childNodes if node not in dep_elems_to_remove] return model.Command(new_qdom, command._local_dir) # Must have already done add_iface on dependency.interface. # If dependency is essential: # Add a clause so that if requiring_impl_var is True then an implementation # matching 'dependency' must also be selected. # If dependency is optional: # Require that no incompatible version is selected. # This ignores any 'command' required. Handle that separately. def find_dependency_candidates(requiring_impl_var, dependency): def meets_restrictions(candidate): for r in dependency.restrictions: if not r.meets_restriction(candidate): #warn("%s rejected due to %s", candidate.get_version(), r) return False return True essential = dependency.importance == model.Dependency.Essential dep_iface = iface_cache.get_interface(dependency.interface) dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ... impl_to_var = iface_to_vars[dep_iface] # Impl -> sat var for candidate in impls_for_iface[dep_iface]: if (candidate.__class__ is _DummyImpl) or meets_restrictions(candidate): if essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: dep_union.append(c_var) # else we filtered that version out, so ignore it else: # Candidate doesn't meet our requirements # If the dependency is optional add a rule to make sure we don't # select this candidate. # (for essential dependencies this isn't necessary because we must # select a good version and we can't select two) if not essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: problem.add_clause(dep_union + [sat.neg(c_var)]) # else we filtered that version out, so ignore it if essential: problem.add_clause(dep_union) def is_unusable(impl, restrictions, arch): """@return: whether this implementation is unusable. @rtype: bool""" return get_unusable_reason(impl, restrictions, arch) != None def get_unusable_reason(impl, restrictions, arch): """ @param impl: Implementation to test. @param restrictions: user-provided restrictions (extra_restrictions) @type restrictions: [L{model.Restriction}] @return: The reason why this impl is unusable, or None if it's OK. @rtype: str @note: The restrictions are for the interface being requested, not the feed of the implementation; they may be different when feeds are being used.""" for r in restrictions: if not r.meets_restriction(impl): return r.reason stability = impl.get_stability() if stability <= model.buggy: return stability.name if (self.config.network_use == model.network_offline or not impl.download_sources) and not impl.is_available(self.config.stores): if not impl.download_sources: return _("No retrieval methods") for method in impl.download_sources: if not method.requires_network: break else: return _("Not cached and we are off-line") if impl.os not in arch.os_ranks: return _("Unsupported OS") if impl.machine not in arch.machine_ranks: if impl.machine == 'src': return _("Source code") elif 'src' in arch.machine_ranks: return _("Not source code") return _("Unsupported machine type") return None def usable_feeds(iface, arch): """Return all feeds for iface that support arch. @rtype: generator(ZeroInstallFeed)""" yield iface.uri for f in iface_cache.get_feed_imports(iface): # Note: when searching for src, None is not in machine_ranks if f.os in arch.os_ranks and \ (f.machine is None or f.machine in arch.machine_ranks): yield f.uri else: logger.debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"), {'feed': f, 'os': f.os, 'machine': f.machine}) # If requiring_var is True then all of requirer's dependencies must be satisfied. # requirer can be a <command> or an <implementation> def process_dependencies(requiring_var, requirer, arch): for d in deps_in_use(requirer, arch): #logger.debug(_("Considering command dependency %s"), d) add_iface(d.interface, arch.child_arch) for c in d.get_required_commands(): # We depend on a specific command within the implementation. command_vars = add_command_iface(d.interface, arch.child_arch, c) # If the parent command/impl is chosen, one of the candidate commands # must be too. If there aren't any, then this command is unselectable. problem.add_clause([sat.neg(requiring_var)] + command_vars) # Must choose one version of d if impl is selected find_dependency_candidates(requiring_var, d) replacement_for = {} # Interface -> Replacement Interface def add_iface(uri, arch): """Name implementations from feed and assert that only one can be selected.""" if uri in ifaces_processed: return ifaces_processed.add(uri) iface = iface_cache.get_interface(uri) main_feed = iface_cache.get_feed(uri) if main_feed: replacement = main_feed.get_replaced_by() if replacement is not None: replacement_for[iface] = iface_cache.get_interface(replacement) impls = [] for f in usable_feeds(iface, arch): self.feeds_used.add(f) logger.debug(_("Processing feed %s"), f) try: feed = iface_cache.get_feed(f) if feed is None: continue #if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for: # info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f}) if feed.implementations: impls.extend(feed.implementations.values()) distro_feed_url = feed.get_distro_feed() if distro_feed_url: self.feeds_used.add(distro_feed_url) distro_feed = iface_cache.get_feed(distro_feed_url) if distro_feed.implementations: impls.extend(distro_feed.implementations.values()) except MissingLocalFeed as ex: logger.warning(_("Missing local feed; if it's no longer required, remove it with:") + '\n0install remove-feed ' + iface.uri + ' ' + f, {'feed': f, 'interface': iface, 'exception': ex}) except model.SafeException as ex: logger.warning(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f, 'interface': iface, 'exception': ex}) #raise except Exception as ex: import logging logger.warning(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f, 'interface': iface, 'exception': ex}, exc_info = True if logger.isEnabledFor(logging.INFO) else None) impls.sort(key = lambda impl: self.get_rating(iface, impl, arch), reverse = True) impls_for_iface[iface] = filtered_impls = [] my_extra_restrictions = self.extra_restrictions.get(iface, []) if self.details is not False: self.details[iface] = [(impl, get_unusable_reason(impl, my_extra_restrictions, arch)) for impl in impls] impl_to_var = iface_to_vars[iface] # Impl -> sat var var_names = [] for impl in impls: if is_unusable(impl, my_extra_restrictions, arch): continue filtered_impls.append(impl) assert impl not in impl_to_var, impl v = problem.add_variable(ImplInfo(iface, impl, arch)) impl_to_var[impl] = v var_names.append(v) if impl.machine and impl.machine != 'src': impls_for_machine_group[machine_groups.get(impl.machine, 0)].append(v) process_dependencies(v, impl, arch) if closest_match: dummy_impl = _DummyImpl() dummy_var = problem.add_variable(ImplInfo(iface, dummy_impl, arch, dummy = True)) var_names.append(dummy_var) impl_to_var[dummy_impl] = dummy_var filtered_impls.append(dummy_impl) # Only one implementation of this interface can be selected if uri == root_interface: if var_names: clause = problem.at_most_one(var_names) problem.add_clause(var_names) # at least one else: problem.impossible() clause = False elif var_names: clause = problem.at_most_one(var_names) else: # Don't need to add to group_clause_for because we should # never get a possible selection involving this. return assert clause is not True assert clause is not None if clause is not False: group_clause_for[uri] = clause def add_command_iface(uri, arch, command_name): """Add every <command> in interface 'uri' with this name. Each one depends on the corresponding implementation and only one can be selected. If closest_match is on, include a dummy command that can always be selected.""" # Check whether we've already processed this (interface,command) pair existing = group_clause_for_command.get((uri, command_name), None) if existing is not None: return existing.lits # First ensure that the interface itself has been processed # We'll reuse the ordering of the implementations to order # the commands too. add_iface(uri, arch) iface = iface_cache.get_interface(uri) filtered_impls = impls_for_iface[iface] impl_to_var = iface_to_vars[iface] # Impl -> sat var var_names = [] for impl in filtered_impls: command = impl.commands.get(command_name, None) if not command: if not isinstance(impl, _DummyImpl): # Mark implementation as unselectable problem.add_clause([sat.neg(impl_to_var[impl])]) continue # We have a candidate <command>. Require that if it's selected # then we select the corresponding <implementation> too. command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch)) problem.add_clause([impl_to_var[impl], sat.neg(command_var)]) var_names.append(command_var) process_dependencies(command_var, command, arch) # Tell the user why we couldn't use this version if self.details is not False: def new_reason(impl, old_reason): if command_name in impl.commands: return old_reason return old_reason or (_('No %s command') % command_name) self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]] if closest_match: dummy_command = problem.add_variable(None) var_names.append(dummy_command) if var_names: # Can't select more than one of them. assert (uri, command_name) not in group_clause_for_command group_clause_for_command[(uri, command_name)] = problem.at_most_one(var_names) return var_names if command_name is None: add_iface(root_interface, root_arch) else: commands = add_command_iface(root_interface, root_arch, command_name) if len(commands) > int(closest_match): # (we have at least one non-dummy command) problem.add_clause(commands) # At least one else: # (note: might be because we haven't cached it yet) logger.info("No %s <command> in %s", command_name, root_interface) problem.impossible() # Require m<group> to be true if we select an implementation in that group m_groups = [] for machine_group, impls in impls_for_machine_group.items(): m_group = 'm%d' % machine_group group_var = problem.add_variable(m_group) if impls: for impl in impls: problem.add_clause([group_var, sat.neg(impl)]) m_groups.append(group_var) if m_groups: m_groups_clause = problem.at_most_one(m_groups) else: m_groups_clause = None # Can't select an implementation of an interface and of its replacement for original, replacement in replacement_for.items(): if original == replacement: logger.warning("Interface %s replaced-by itself!", original) continue rep_impls = iface_to_vars.get(replacement, None) if rep_impls is None: # We didn't even look at the replacement interface, so no risk here continue # Must select one implementation out of all candidates from both interface. # Dummy implementations don't conflict, though. all_impls = [] for impl, var in rep_impls.items(): if not isinstance(impl, _DummyImpl): all_impls.append(var) for impl, var in iface_to_vars[original].items(): if not isinstance(impl, _DummyImpl): all_impls.append(var) if all_impls: problem.at_most_one(all_impls) # else: neither feed has any usable impls anyway def decide(): """This is called by the SAT solver when it cannot simplify the problem further. Our job is to find the most-optimal next selection to try. Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): # Restrictions don't express that we do or don't want the # dependency, so skip them here. if dep.importance == model.Dependency.Restricts: continue for c in dep.get_required_commands(): dep_lit = find_undecided_command(dep.interface, c) if dep_lit is not None: return dep_lit dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) group = group_clause_for.get(uri, None) if group is None: # (can be None if the selected impl has an optional dependency on # a feed with no implementations) return #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in list(group_clause_for.values()) + list(group_clause_for_command.values()) + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination ready = problem.run_solver(decide) is True if not ready and not closest_match: # We failed while trying to do a real solve. # Try a closest match solve to get a better # error report for the user. self.solve(root_interface, root_arch, command_name = command_name, closest_match = True) else: self.ready = ready and not closest_match self.selections = selections.Selections(None) self.selections.interface = root_interface if not self.ready: # Store some things useful for get_failure_reason() self._impls_for_iface = impls_for_iface self._iface_to_vars = iface_to_vars self._problem = problem sels = self.selections.selections commands_needed = [] # Populate sels with the selected implementations. # Also, note down all the commands we need. for uri, group in group_clause_for.items(): if group.current is not None: lit_info = problem.get_varinfo_for_lit(group.current).obj if lit_info.is_dummy: sels[lit_info.iface.uri] = None else: # We selected an implementation for interface 'uri' impl = lit_info.impl for b in impl.bindings: c = b.command if c is not None: commands.append((uri, c)) deps = self.requires[lit_info.iface] = [] for dep in deps_in_use(lit_info.impl, lit_info.arch): deps.append(dep) for c in dep.get_required_commands(): commands_needed.append((dep.interface, c)) sel = sels[lit_info.iface.uri] = selections.ImplSelection(lit_info.iface.uri, impl, deps) sel.__arch = lit_info.arch # Now all all the commands in too. def add_command(iface, name): sel = sels.get(iface, None) if sel: command = sel.impl.commands[name] if name in sel._used_commands: return # Already added sel._used_commands[name] = clone_command_for(command, sel.__arch) for dep in command.requires: for dep_command_name in dep.get_required_commands(): add_command(dep.interface, dep_command_name) # A <command> can depend on another <command> in the same interface # (e.g. the tests depending on the main program) for b in command.bindings: c = b.command if c is not None: add_command(iface, c) for iface, command in commands_needed: add_command(iface, command) if command_name is not None: self.selections.command = command_name add_command(root_interface, command_name) if root_interface not in sels: sels[root_interface] = None # Useful for get_failure_reason()
def solve(self, root_interface, root_arch, command_name='run', closest_match=False): # closest_match is used internally. It adds a lowest-ranked # by valid implementation to every interface, so we can always # select something. Useful for diagnostics. # The basic plan is this: # 1. Scan the root interface and all dependencies recursively, building up a SAT problem. # 2. Solve the SAT problem. Whenever there are multiple options, try the most preferred one first. # 3. Create a Selections object from the results. # # All three involve recursively walking the tree in a similar way: # 1) we follow every dependency of every implementation (order not important) # 2) we follow every dependency of every selected implementation (better versions first) # 3) we follow every dependency of every selected implementation (order doesn't matter) # # In all cases, a dependency may be on an <implementation> or on a specific <command>. # TODO: We need some way to figure out which feeds to include. # Currently, we include any feed referenced from anywhere but # this is probably too much. We could insert a dummy optimial # implementation in stale/uncached feeds and see whether it # selects that. iface_cache = self.config.iface_cache problem = sat.SATProblem() impl_to_var = {} # Impl -> sat var self.feeds_used = set() self.requires = {} self.ready = False self.details = self.record_details and {} self.selections = None self._failure_reason = None ifaces_processed = set() impls_for_machine_group = { 0: [] } # Machine group (e.g. "64") to [impl] in that group for machine_group in machine_groups.values(): impls_for_machine_group[machine_group] = [] impls_for_iface = {} # Iface -> [impl] group_clause_for = {} # Iface URI -> AtMostOneClause | bool group_clause_for_command = { } # (Iface URI, command name) -> AtMostOneClause | bool # Return the dependencies of impl that we should consider. # Skips dependencies if the use flag isn't what we need. # (note: impl may also be a model.Command) def deps_in_use(impl, arch): for dep in impl.requires: use = dep.metadata.get("use", None) if use not in arch.use: continue yield dep # Must have already done add_iface on dependency.interface. # If dependency is essential: # Add a clause so that if requiring_impl_var is True then an implementation # matching 'dependency' must also be selected. # If dependency is optional: # Require that no incompatible version is selected. # This ignores any 'command' required. Handle that separately. def find_dependency_candidates(requiring_impl_var, dependency): def meets_restrictions(candidate): for r in dependency.restrictions: if not r.meets_restriction(candidate): #warn("%s rejected due to %s", candidate.get_version(), r) return False return True essential = dependency.importance == model.Dependency.Essential dep_iface = iface_cache.get_interface(dependency.interface) dep_union = [sat.neg(requiring_impl_var) ] # Either requiring_impl_var is False, or ... for candidate in impls_for_iface[dep_iface]: if (candidate.__class__ is _DummyImpl) or meets_restrictions(candidate): if essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: dep_union.append(c_var) # else we filtered that version out, so ignore it else: # Candidate doesn't meet our requirements # If the dependency is optional add a rule to make sure we don't # select this candidate. # (for essential dependencies this isn't necessary because we must # select a good version and we can't select two) if not essential: c_var = impl_to_var.get(candidate, None) if c_var is not None: problem.add_clause(dep_union + [sat.neg(c_var)]) # else we filtered that version out, so ignore it if essential: problem.add_clause(dep_union) def is_unusable(impl, restrictions, arch): """@return: whether this implementation is unusable. @rtype: bool""" return get_unusable_reason(impl, restrictions, arch) != None def get_unusable_reason(impl, restrictions, arch): """ @param impl: Implementation to test. @type restrictions: [L{model.Restriction}] @return: The reason why this impl is unusable, or None if it's OK. @rtype: str @note: The restrictions are for the interface being requested, not the feed of the implementation; they may be different when feeds are being used.""" for r in restrictions: if not r.meets_restriction(impl): return _( "Incompatible with another selected implementation") stability = impl.get_stability() if stability <= model.buggy: return stability.name if (self.config.network_use == model.network_offline or not impl.download_sources) and not impl.is_available( self.config.stores): if not impl.download_sources: return _("No retrieval methods") return _("Not cached and we are off-line") if impl.os not in arch.os_ranks: return _("Unsupported OS") if impl.machine not in arch.machine_ranks: if impl.machine == 'src': return _("Source code") return _("Unsupported machine type") return None def usable_feeds(iface, arch): """Return all feeds for iface that support arch. @rtype: generator(ZeroInstallFeed)""" yield iface.uri for f in iface_cache.get_feed_imports(iface): # Note: when searching for src, None is not in machine_ranks if f.os in arch.os_ranks and \ (f.machine is None or f.machine in arch.machine_ranks): yield f.uri else: debug( _("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s" ), { 'feed': f, 'os': f.os, 'machine': f.machine }) # If requiring_var is True then all of requirer's dependencies must be satisfied. # requirer can be a <command> or an <implementation> def process_dependencies(requiring_var, requirer, arch): for d in deps_in_use(requirer, arch): debug(_("Considering command dependency %s"), d) add_iface(d.interface, arch.child_arch) for c in d.get_required_commands(): # We depend on a specific command within the implementation. command_vars = add_command_iface(d.interface, arch.child_arch, c) # If the parent command/impl is chosen, one of the candidate commands # must be too. If there aren't any, then this command is unselectable. problem.add_clause([sat.neg(requiring_var)] + command_vars) # Must choose one version of d if impl is selected find_dependency_candidates(requiring_var, d) def add_iface(uri, arch): """Name implementations from feed and assert that only one can be selected.""" if uri in ifaces_processed: return ifaces_processed.add(uri) iface = iface_cache.get_interface(uri) impls = [] for f in usable_feeds(iface, arch): self.feeds_used.add(f) debug(_("Processing feed %s"), f) try: feed = iface_cache.get_feed(f) if feed is None: continue #if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for: # info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f}) if feed.implementations: impls.extend(feed.implementations.values()) distro_feed_url = feed.get_distro_feed() if distro_feed_url: self.feeds_used.add(distro_feed_url) distro_feed = iface_cache.get_feed(distro_feed_url) if distro_feed.implementations: impls.extend(distro_feed.implementations.values()) except MissingLocalFeed as ex: warn( _("Missing local feed; if it's no longer required, remove it with:" ) + '\n0install remove-feed ' + iface.uri + ' ' + f, { 'feed': f, 'interface': iface, 'exception': ex }) except Exception as ex: warn( _("Failed to load feed %(feed)s for %(interface)s: %(exception)s" ), { 'feed': f, 'interface': iface, 'exception': ex }) #raise impls.sort(key=lambda impl: self.get_rating(iface, impl, arch), reverse=True) impls_for_iface[iface] = filtered_impls = [] my_extra_restrictions = self.extra_restrictions.get(iface, []) if self.record_details: self.details[iface] = [ (impl, get_unusable_reason(impl, my_extra_restrictions, arch)) for impl in impls ] var_names = [] for impl in impls: if is_unusable(impl, my_extra_restrictions, arch): continue filtered_impls.append(impl) if impl in impl_to_var: # TODO If the same impl comes from original feed and # the feed with "feed" tag continue v = problem.add_variable(ImplInfo(iface, impl, arch)) impl_to_var[impl] = v var_names.append(v) if impl.machine and impl.machine != 'src': impls_for_machine_group[machine_groups.get( impl.machine, 0)].append(v) process_dependencies(v, impl, arch) if closest_match: dummy_impl = _DummyImpl() dummy_var = problem.add_variable( ImplInfo(iface, dummy_impl, arch, dummy=True)) var_names.append(dummy_var) impl_to_var[dummy_impl] = dummy_var filtered_impls.append(dummy_impl) # Only one implementation of this interface can be selected if uri == root_interface: if var_names: clause = problem.at_most_one(var_names) problem.add_clause(var_names) # at least one else: problem.impossible() clause = False elif var_names: clause = problem.at_most_one(var_names) else: # Don't need to add to group_clause_for because we should # never get a possible selection involving this. return assert clause is not True assert clause is not None if clause is not False: group_clause_for[uri] = clause def add_command_iface(uri, arch, command_name): """Add every <command> in interface 'uri' with this name. Each one depends on the corresponding implementation and only one can be selected. If closest_match is on, include a dummy command that can always be selected.""" # Check whether we've already processed this (interface,command) pair existing = group_clause_for_command.get((uri, command_name), None) if existing is not None: return existing.lits # First ensure that the interface itself has been processed # We'll reuse the ordering of the implementations to order # the commands too. add_iface(uri, arch) iface = iface_cache.get_interface(uri) filtered_impls = impls_for_iface[iface] var_names = [] for impl in filtered_impls: command = impl.commands.get(command_name, None) if not command: if not isinstance(impl, _DummyImpl): # Mark implementation as unselectable problem.add_clause([sat.neg(impl_to_var[impl])]) continue # We have a candidate <command>. Require that if it's selected # then we select the corresponding <implementation> too. command_var = problem.add_variable( CommandInfo(command_name, command, impl, arch)) problem.add_clause([impl_to_var[impl], sat.neg(command_var)]) var_names.append(command_var) process_dependencies(command_var, command, arch) # Tell the user why we couldn't use this version if self.record_details: def new_reason(impl, old_reason): if command_name in impl.commands: return old_reason return old_reason or (_('No %s command') % command_name) self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]] if closest_match: dummy_command = problem.add_variable(None) var_names.append(dummy_command) if var_names: # Can't select more than one of them. assert (uri, command_name) not in group_clause_for_command group_clause_for_command[( uri, command_name)] = problem.at_most_one(var_names) return var_names if command_name is None: add_iface(root_interface, root_arch) else: commands = add_command_iface(root_interface, root_arch, command_name) if len(commands) > int(closest_match): # (we have at least one non-dummy command) problem.add_clause(commands) # At least one else: # (note: might be because we haven't cached it yet) info("No %s <command> in %s", command_name, root_interface) impls = impls_for_iface[iface_cache.get_interface( root_interface)] if impls == [] or (len(impls) == 1 and isinstance(impls[0], _DummyImpl)): # There were no candidates at all. if self.config.network_use == model.network_offline: self._failure_reason = _( "Interface '%s' has no usable implementations in the cache (and 0install is in off-line mode)" ) % root_interface else: self._failure_reason = _( "Interface '%s' has no usable implementations" ) % root_interface else: # We had some candidates implementations, but none for the command we need self._failure_reason = _( "Interface '%s' cannot be executed directly; it is just a library " "to be used by other programs (or missing '%s' command)" ) % (root_interface, command_name) problem.impossible() # Require m<group> to be true if we select an implementation in that group m_groups = [] for machine_group, impls in impls_for_machine_group.iteritems(): m_group = 'm%d' % machine_group group_var = problem.add_variable(m_group) if impls: for impl in impls: problem.add_clause([group_var, sat.neg(impl)]) m_groups.append(group_var) if m_groups: m_groups_clause = problem.at_most_one(m_groups) else: m_groups_clause = None def decide(): """Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): for c in dep.get_required_commands(): dep_lit = find_undecided_command(dep.interface, c) if dep_lit is not None: return dep_lit dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) if uri not in group_clause_for: # TODO Looks like Sweets patch introduced this issue return group = group_clause_for[uri] #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in group_clause_for.values( ) + group_clause_for_command.values() + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination ready = problem.run_solver(decide) is True if not ready and not closest_match: # We failed while trying to do a real solve. # Try a closest match solve to get a better # error report for the user. self.solve(root_interface, root_arch, command_name=command_name, closest_match=True) else: self.ready = ready and not closest_match self.selections = selections.Selections(None) self.selections.interface = root_interface sels = self.selections.selections commands_needed = [] # Popular sels with the selected implementations. # Also, note down all the commands we need. for uri, group in group_clause_for.iteritems(): if group.current is not None: lit_info = problem.get_varinfo_for_lit(group.current).obj if lit_info.is_dummy: sels[lit_info.iface.uri] = None else: # We selected an implementation for interface 'uri' impl = lit_info.impl for b in impl.bindings: c = b.command if c is not None: commands.append((uri, c)) deps = self.requires[lit_info.iface] = [] for dep in deps_in_use(lit_info.impl, lit_info.arch): deps.append(dep) for c in dep.get_required_commands(): commands_needed.append((dep.interface, c)) sels[lit_info.iface.uri] = selections.ImplSelection( lit_info.iface.uri, impl, deps) # Now all all the commands in too. def add_command(iface, name): sel = sels.get(iface, None) if sel: command = sel.impl.commands[name] if name in sel._used_commands: return # Already added sel._used_commands.add(name) for dep in command.requires: for dep_command_name in dep.get_required_commands(): add_command(dep.interface, dep_command_name) # A <command> can depend on another <command> in the same interface # (e.g. the tests depending on the main program) for b in command.bindings: c = b.command if c is not None: add_command(iface, c) for iface, command in commands_needed: add_command(iface, command) if command_name is not None: self.selections.command = command_name add_command(root_interface, command_name)
def decide(): """This is called by the SAT solver when it cannot simplify the problem further. Our job is to find the most-optimal next selection to try. Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): # Restrictions don't express that we do or don't want the # dependency, so skip them here. if dep.importance == model.Dependency.Restricts: continue for c in dep.get_required_commands(): dep_lit = find_undecided_command(dep.interface, c) if dep_lit is not None: return dep_lit dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) group = group_clause_for.get(uri, None) if group is None: # (can be None if the selected impl has an optional dependency on # a feed with no implementations) return #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in list(group_clause_for.values()) + list( group_clause_for_command.values()) + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination
def decide(): """Recurse through the current selections until we get to an interface with no chosen version, then tell the solver to try the best version from that.""" def find_undecided_dep(impl_or_command, arch): # Check for undecided dependencies of impl_or_command for dep in deps_in_use(impl_or_command, arch): for c in dep.get_required_commands(): dep_lit = find_undecided_command(dep.interface, c) if dep_lit is not None: return dep_lit dep_lit = find_undecided(dep.interface) if dep_lit is not None: return dep_lit return None seen = set() def find_undecided(uri): if uri in seen: return # Break cycles seen.add(uri) if uri not in group_clause_for: # TODO Looks like Sweets patch introduced this issue return group = group_clause_for[uri] #print "Group for %s = %s" % (uri, group) lit = group.current if lit is None: return group.best_undecided() # else there was only one choice anyway # Check for undecided dependencies lit_info = problem.get_varinfo_for_lit(lit).obj return find_undecided_dep(lit_info.impl, lit_info.arch) def find_undecided_command(uri, name): if name is None: return find_undecided(uri) group = group_clause_for_command[(uri, name)] lit = group.current if lit is None: return group.best_undecided() # else we've already chosen which <command> to use # Check for undecided command-specific dependencies, and then for # implementation dependencies. lit_info = problem.get_varinfo_for_lit(lit).obj if lit_info is None: assert closest_match return None # (a dummy command added for better diagnostics; has no dependencies) return find_undecided_dep(lit_info.command, lit_info.arch) or \ find_undecided_dep(lit_info.impl, lit_info.arch) best = find_undecided_command(root_interface, command_name) if best is not None: return best # If we're chosen everything we need, we can probably # set everything else to False. for group in group_clause_for.values( ) + group_clause_for_command.values() + [m_groups_clause]: if group.current is None: best = group.best_undecided() if best is not None: return sat.neg(best) return None # Failed to find any valid combination
def add_command_iface(uri, arch, command_name): """Add every <command> in interface 'uri' with this name. Each one depends on the corresponding implementation and only one can be selected. If closest_match is on, include a dummy command that can always be selected.""" # Check whether we've already processed this (interface,command) pair existing = group_clause_for_command.get((uri, command_name), None) if existing is not None: return existing.lits # First ensure that the interface itself has been processed # We'll reuse the ordering of the implementations to order # the commands too. add_iface(uri, arch) iface = iface_cache.get_interface(uri) filtered_impls = impls_for_iface[iface] var_names = [] for impl in filtered_impls: command = impl.commands.get(command_name, None) if not command: if not isinstance(impl, _DummyImpl): # Mark implementation as unselectable problem.add_clause([sat.neg(impl_to_var[impl])]) continue # We have a candidate <command>. Require that if it's selected # then we select the corresponding <implementation> too. command_var = problem.add_variable( CommandInfo(command_name, command, impl, arch)) problem.add_clause([impl_to_var[impl], sat.neg(command_var)]) var_names.append(command_var) runner = command.get_runner() for d in deps_in_use(command, arch): if d is runner: # With a <runner>, we depend on a command rather than on an # implementation. This allows us to support recursive <runner>s, etc. debug(_("Considering command runner %s"), d) runner_command_name = _get_command_name(d) runner_vars = add_command_iface( d.interface, arch.child_arch, runner_command_name) # If the parent command is chosen, one of the candidate runner commands # must be too. If there aren't any, then this command is unselectable. problem.add_clause([sat.neg(command_var)] + runner_vars) else: debug(_("Considering command dependency %s"), d) add_iface(d.interface, arch.child_arch) # Must choose one version of d if impl is selected find_dependency_candidates(command_var, d) # Tell the user why we couldn't use this version if self.record_details: def new_reason(impl, old_reason): if command_name in impl.commands: return old_reason return old_reason or (_('No %s command') % command_name) self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]] if closest_match: dummy_command = problem.add_variable(None) var_names.append(dummy_command) if var_names: # Can't select more than one of them. assert (uri, command_name) not in group_clause_for_command group_clause_for_command[( uri, command_name)] = problem.at_most_one(var_names) return var_names