def _expand_depedencies(self, local_repo, repo, targets, exclude=set()):
missing = set()
resolved = set(targets)
queue = list(targets)
seen = set()
while queue:
package = queue.pop()
if package in seen:
continue
seen.add(package)
# We don't want to expand dependencies for a package that is
# already installed. The user might have intentionally ignored
# dependencies when they installed the package, so we are just
# going to assume they know what they are doing.
if package.is_local:
continue
for dep_name in package.dependecies:
q = Query(dep_name)
dep_package = self._find_satisfier(local_repo,
repo,
q,
targets,
exclude=exclude)
# We couldn't find a satisfier
if dep_package is None:
missing.add((package, q))
continue
resolved.add(dep_package)
queue.append(dep_package)
return (resolved, missing)
def visualize():
import networkx as nx
# How about we don't fire up a JVM just to start my script?
from asciinet import graph_to_ascii
G = nx.DiGraph()
root = "Root"
G.add_node(root)
# @ENHANCEMENT
# Right now we just visualize it as a stright up dependency graph. We might
# want to show when we use provides instead in the future. This would
# involve adding a fake node when we look for a provides and let that
# depend on the actual implementors
for package in local_repo.get_all_packages():
G.add_node(package)
if package.reason == InstallReason.REQ:
G.add_edge(root, package)
for dep_str in package.dependecies:
q = Query(dep_str)
dep = local_repo.find_package(q)
if dep is None:
print(package, q)
G.add_edge(package, dep)
print(graph_to_ascii(G))
def _dependencies_filled_by_other(self, package):
for dep_string in package.dependecies:
dep_packages = self.local_repo.find_packages(Query(dep_string),
exclude=self.targets)
if not dep_packages:
return False
return True
def details(package_q):
q = Query(package_q)
package = local_repo.find_package(q)
if package is None:
print("No package named {Style.BRIGHT}{}{Style.RESET_ALL}".format(
q, Style=Style))
exit(1)
print_local_package(local_repo, package)
def _find_bridge_conflicts(self):
conflicts = set()
for package in self.removes:
for (b1, b2) in package.bridges:
p1 = self.local_repo.find_package(Query(b1))
p2 = self.local_repo.find_package(Query(b2))
if not p1 or not p2:
# One of the sides were not installed, which means it's
# fine
continue
if p1 in self.removes or p2 in self.removes:
# We are about to remove one of the sides. It's fine
continue
conflict = self._check_conflict(p1, p2)
if conflict:
conflicts.add((package, conflict))
return conflicts
def _rate_satisfier(self, local_repo, package):
if not package.dependecies:
return 0
cnt = 0
for dep_str in package.dependecies:
q = Query(dep_str)
if self.local_repo.find_package(q) is not None:
continue
cnt += 1
return cnt / len(package.dependecies)
def _find_package_upgrade(self, package):
# Ignore any package that is already in the targets array, regardless
# of version
if package in self.targets:
return
# Just search for the name and do the version comparison manually
q = Query(package.name)
remote_package = self.repo.find_literal(q)
if remote_package.version > package.version:
self.targets.add(remote_package)
def _find_upgrade_required(self, targets):
upgrade = set()
for package in targets:
assert (package.is_local)
q = Query(package.name)
remote_package = self.repo.find_literal(q)
if remote_package is None:
print("No remote package for {}".format(package))
continue
if remote_package.version > package.version:
upgrade.add((package, remote_package))
return upgrade
def _does_bridge(self, local_repo, targets, p1, p2):
# Do we have a bridge already installed?
installed_bridges = local_repo.find_bridges(p1, p2)
if installed_bridges:
return True
# Do any of the packages we are about to install offer to bridge this?
for package in targets:
for bridge in package.bridges:
b1 = Query(bridge[0])
b2 = Query(bridge[1])
if b1.matches(p1) and b2.matches(p2):
return True
elif b2.matches(p1) and b1.matches(p2):
return True
return False
def check(packages):
init()
if packages:
qs = [Query(p) for p in packages]
packages = []
for q in qs:
package = repo.find_package(q)
if package is None:
print("No package named "
"{Style.BRIGHT}{}{Style.RESET_ALL}".format(q,
Style=Style))
return
packages.append(package)
else:
# A bit hacky, but this is a debug command, so i'm not too worried.
packages = [repo._load_package(p) for p in repo._all_packages()]
owners = {}
fetches = set()
for package in packages:
print("Collecting sources from {}".format(package.name))
for s in package.sources:
# We need the owners to report who requested a download later
if s.uri not in owners:
owners[s.uri] = []
owners[s.uri].append(package)
fetches.add((s.uri))
# Actually do the check, this should use some fast method in the individual
# handlers
checked = downloader.check(fetches)
anyFail = False
for (uri, result) in checked.items():
# Only report if something went wrong
if not result:
print("Failed checking uri "
"{Style.BRIGHT}{Fore.RED}{}{Style.RESET_ALL} "
"requested by:".format(uri, Style=Style, Fore=Fore))
# Print the packages that depend on the download, helps find the
# culprit in large checks
for package in owners[uri]:
print(f"\t{package.name}")
anyFail = True
# If everything went well let's just print something to instill some
# confidence
if not anyFail:
print("Everything was good")
def packages_to_graph(self, targets):
G = nx.DiGraph()
for package in targets:
G.add_node(package)
for dep_name in package.dependecies:
q = Query(dep_name)
dep_package = self._find_satisfier_in_set(targets, q)
if dep_package is None:
# Swallow the error, since this is expected in some cases
continue
# raise Exception(
# "Tried to make a graph out of packages that "
# "have unresolved dependencies: " + str(q)
# )
G.add_edge(package, dep_package)
return G
def _find_conflicts(self, targets, local_repo):
# Find conflicts inside targets
conflicts = set()
for t in targets:
for conflict in t.conflicts:
cq = Query(conflict)
for tc in targets:
# Packages don't conflict with themselves
if t == tc:
continue
if (cq.matches(tc) and
not self._does_bridge(local_repo, targets, t, tc)):
conflicts.add((t, tc))
local_packages = local_repo.get_all_packages()
# Find conflicts from targets to local_repo
for t in targets:
for conflict in t.conflicts:
cq = Query(conflict)
for lp in local_packages:
# Packages don't conflict with themselves
if t == lp:
continue
if (cq.matches(lp) and
not self._does_bridge(local_repo, targets, t, lp)):
conflicts.add((t, lp))
# Find conflicts from local_repo to targets
for lp in local_packages:
for conflict in lp.conflicts:
cq = Query(conflict)
for tc in targets:
# Packages don't conflict with themselves
if lp == tc:
continue
if (cq.matches(tc) and not self._does_bridge(
local_repo, targets, lp, tc)): # NOQA
conflicts.add((lp, tc))
return conflicts
def _get_owned(self):
owned = set(self.targets)
# Here we want to process the same package multiple times
queue = set(self.targets)
while queue:
package = queue.pop()
if package.reason == InstallReason.DEP:
for depa in self.local_repo.find_dependants(package):
if depa not in owned:
break
else:
owned.add(package)
for dep_str in package.dependecies:
deps = self.local_repo.find_packages(Query(dep_str))
for dep in deps:
queue.add(dep)
return owned
def _check_conflict(self, p1, p2):
# Does p1 conflict with p2
for conflict in p1.conflicts:
cq = Query(conflict)
if cq.matches(p2) and not self._does_bridge_after(p1, p2):
return (p1, p2)
# Does p2 conflict with p1
for conflict in p2.conflicts:
cq = Query(conflict)
if cq.matches(p1) and not self._does_bridge_after(p2, p1):
return p2, p1
def search(term, reverse):
candidates = repo.search(" ".join(term))
if reverse:
candidates = list(reversed(candidates))
lst = PackageList(candidates)
for package in lst:
# Set all the tags
q = Query(package.name)
local_pkg = local_repo.find_package(q)
# Having a local package means it's currently installed. Fill in the
# tag with the version
if local_pkg:
lst.add_tag(package, InstalledTag(local_pkg.version))
lst.present()
def expand(self):
assert (self.state == TransactionState.INIT)
(expanded, missing) = self._expand_depedencies(self.local_repo,
self.repo,
self.targets,
exclude=self.removes)
if len(missing) > 0:
raise MissingDependencyError(missing)
self.depend_G = super().packages_to_graph(expanded)
try:
cycle = next(nx.simple_cycles(self.depend_G))
except StopIteration:
pass
else:
raise TransactionCycleError(cycle)
self._sort_deps_to_targets()
# Find all the packages that are upgrades rather than installs
for package in expanded:
if not package.is_local:
# Search for just the name to check if we have any version
# locally
q = Query(package.name)
local_package = self.local_repo.find_package(q)
# If we have something we want to remove it first, we call that
# an "upgrade"
if local_package is None:
continue
self.removes.append(local_package)
conflicts = super()._find_conflicts(self.installs, self.local_repo)
if len(conflicts) > 0:
# There were at least some conflicts
raise ConflictError(conflicts)
self.state = TransactionState.EXPANDED
def make_profile(self, local_repo):
modlist_path = self.cfg.profile_dir / "modlist.txt"
with open(modlist_path, "w") as f:
G = nx.DiGraph()
for package in local_repo.get_all_packages():
G.add_node(package)
for dep_name in package.dependecies:
q = Query(dep_name)
dep = local_repo.find_package(q)
if dep is None:
# Skip mising dependecies. If we have an installed
# package which has a not installed dependency, then we
# just want to skip it. It's up to the user to make
# sure everything is resolved, of course assisted by
# the tool. @COMPLETE it might be useful give the user
# some way of doing a full dependency verification of
# the local repo
continue
G.add_edge(package, dep)
for package in nx.lexicographical_topological_sort(
G,
key=lambda x: x.priority):
print("+" + package.name, file=f)
print("*Unmanaged: Dawnguard", file=f)
print("*Unmanaged: Dragonborn", file=f)
print("*Unmanaged: HearthFires", file=f)
print("*Unmanaged: HighResTexturePack01", file=f)
print("*Unmanaged: HighResTexturePack02", file=f)
print("*Unmanaged: HighResTexturePack03", file=f)
print("*Unmanaged: Unofficial Dawnguard Patch", file=f)
print("*Unmanaged: Unofficial Dragonborn Patch", file=f)
print("*Unmanaged: Unofficial Hearthfire Patch", file=f)
print("*Unmanaged: Unofficial High Resolution Patch", file=f)
print("*Unmanaged: Unofficial Skyrim Patch", file=f)
def remove(packages, no_dep):
# Skipping dependency checking can be dangerous. Lets print a nice big
# warning to make sure the user knows what they are doing.
if no_dep:
print("{Fore.YELLOW}{Style.BRIGHT}Warning:{Style.RESET_ALL}"
" depedency checking disabled".format(Style=Style, Fore=Fore))
qs = [Query(p) for p in packages]
t = RemoveTransaction(local_repo, repo, downloader, no_dep)
for q in qs:
package = local_repo.find_package(q)
if package is None:
print(
"No installed package named {Style.BRIGHT}{}{Style.RESET_ALL}".
format(q, Style=Style))
return
t.add(package)
try:
t.expand()
except ConflictError as e:
# Removing these packages would mean creating these new conflicts
# This is possible because we have bridges
print("{Fore.RED}Conflicting packages{Fore.RESET}".format(Fore=Fore))
for (bridge, conflict) in e.conflicts:
print("Removing {Style.BRIGHT}{}{Style.RESET_ALL} would break "
"bridge of {Style.BRIGHT}{}{Style.RESET_ALL} and "
"{Style.BRIGHT}{}{Style.RESET_ALL}".format(bridge,
*conflict,
Style=Style))
bridges = repo.find_bridges(*conflict, exclude=t.targets)
for bridge in bridges:
print("\t{Style.BRIGHT}{}{Style.RESET_ALL} is an "
"alternative bridge".format(bridge, Style=Style))
if not bridges:
print("\t {Fore.RED}Unfortunately{Fore.RESET} I don't know of "
"any alternative".format(Fore=Fore))
exit(1)
except DependencyBreakError as e:
print("")
print(
"Removal would {Style.BRIGHT}break{Style.RESET_ALL} dependencies: "
.format(Style=Style))
for p in e.dependencies:
print("\t{Style.BRIGHT}{}{Style.RESET_ALL} depends on "
"{Style.BRIGHT}{}{Style.RESET_ALL}".format(*p, Style=Style))
exit(1)
print("")
print("This will remove the following packages: ")
print("PACKAGES: {Style.BRIGHT}{}".format(", ".join(map(str, t.removes)),
Style=Style))
print("")
Q.yes_no("Are you sure?")
try:
t.prepare()
# @DEAD
except DependencyBreakError as e:
print("")
print(
"Removal would {Style.BRIGHT}break{Style.RESET_ALL} dependencies: "
.format(Style=Style))
for p in e.dependencies:
print("\t{Style.BRIGHT}{}{Style.RESET_ALL} depends on "
"{Style.BRIGHT}{}{Style.RESET_ALL}".format(*p, Style=Style))
exit(1)
t.commit()
organizer.make_profile(local_repo)
def install(packages, explicit, upgrade, reason):
# Create the queries from the strings
qs = [Query(p) for p in packages]
if upgrade:
t = UpgradeTransaction(local_repo, repo, downloader)
else:
t = AddTransaction(local_repo, repo, downloader, reason, src_cache)
# Support for loading out of tree package specifications
for e_path in explicit:
# @HACK this should be a config option and also maybe not specified
# here?
pkgsrc = cfg.source.dir
e = Path(e_path)
if not e.exists():
print("Explicit package at {Style.BRIGHT}{}{Style.RESET_ALL} "
"not found".format(e, Style=Style, Fore=Fore))
exit(1)
package = load_package(e, pkgsrc, organizer.getModsDir())
t.add(package)
for q in qs:
package = repo.find_package(q)
if package is None:
print("No package named {Style.BRIGHT}{}{Style.RESET_ALL}".format(
q, Style=Style))
return
t.add(package)
try:
t.expand()
except TransactionCycleError as e:
# Some form of dependency cycle
# @ENHANCEMENT We should really run visualize on the graph that failed.
# That would be way more helpful
# -- We can't use asciinet since installing it is a pain
print("ERROR Cycle detected: ")
print("\tCycle consists of: {}".format(", ".join(
(p.name for p in e.cycle))))
exit(1)
except ConflictError as e:
# We found some conflicts which means we need to look for some package
# to bridge them that aren't part of the transaction
print("{Fore.RED}Conflicting packages{Fore.RESET}".format(Fore=Fore))
for conflict in e.conflicts:
print("{Style.BRIGHT}{}{Style.RESET_ALL} conflicts with "
"{Style.BRIGHT}{}{Style.RESET_ALL}".format(*conflict,
Style=Style))
bridges = repo.find_bridges(*conflict)
for bridge in bridges:
print("\t{Style.BRIGHT}{}{Style.RESET_ALL} could bridge "
"that conflict".format(bridge, Style=Style))
exit(1)
except MissingDependencyError as e:
print(
"{Fore.RED}Unresolved dependencies{Fore.RESET}".format(Fore=Fore))
for missing in e.dependencies:
print(
"{Style.BRIGHT}{}{Style.RESET_ALL} requires "
"{Style.BRIGHT}{}{Style.RESET_ALL} which wasn't found".format(
*missing, Style=Style))
exit(1)
if not t.targets:
print("Nothing to do".format(Style=Style, Fore=Fore))
exit(0)
print("")
print("This will install the following packages: ")
print("-> " + Style.BRIGHT + ", ".join(map(str, t.installs)))
print("")
Q.yes_no("Are you sure?")
t.prepare()
t.commit()
organizer.make_profile(local_repo)
def expand(self):
# If theres no targets, expand to all local packages
if not self.targets:
self.targets = self.local_repo.get_all_packages()
upgrades = self._find_upgrade_required(self.targets)
self.removes = [u[0] for u in upgrades]
self.installs = [u[1] for u in upgrades]
self.targets = [
u[1] for u in upgrades if u[0].reason == InstallReason.REQ
]
# So the dependencies might have changed. I don't think we want to
# remove unused dependencies, but we do want to pull in new ones. We
# might also stop providing something, which could invalidate some
# other part of the local database.
# Basically we will want to do the following:
# - Check if all dependencies are filled
# - Fill new ones (maybe asking?)
# - Check that all the people depending on us still have their
# dependencies filled after the transaction
# During all of these steps we need to make sure that we aren't
# matching packages that this transaction is going to remove, but
# include packages we are about to install.
# Find all packages touched by something we are removing
new_missing = set()
for package in self.removes:
dependants = self.local_repo.find_dependants(package)
for dependant in dependants:
# If the dependant is queued for removal we don't really care
# if we are going to break dependencies
if dependant in self.removes:
continue
for dep_str in dependant.dependecies:
q = Query(dep_str)
# If it didn't match before then we don't care
# This is important because we don't want to pop up and
# error now if the user purposefully broke some
# dependencies at some point, but if we are breaking
# something we want to report that
if not q.matches(package):
continue
# Are we going to install something that fixes it?
dep_sat = super()._find_satisfier_in_set(self.installs, q)
if dep_sat is not None:
continue
# Do we have something else that fixes it?
dep_sat = self.local_repo.find_package(
q, exclude=self.removes)
if dep_sat is not None:
continue
# This upgrade is going to be a problem
# @COMPLETE We should dump the package here as well.
new_missing.add((dependant, q))
if len(new_missing) > 0:
raise MissingDependencyError(new_missing)
# @HACK: Reset removes, since the add transaction finds all packages
# that are actually upgrades
self.removes = []
self.targets = self.installs
super().expand()