def resolve_requirement(self, req):
best = self.best.get(req.key, None)
log.debug(" -- previous best is %r " % (best, ))
if best is not None:
dist, prev_req = best
# FIXME: In theory, if the previously found requirement, and the requirement
# we are processing are the same, we should be able to skip the
# checks for merging requirements process them as normal.
# In practice, merging a requirement with itself takes you down
# a differnet resolution route as it picks up stuff from the environment
# that we otherwise wouldn't have seen. Leaving this as-is for now
# until the egg-cache changes are in.
# if prev_req.hashCmp == req.hashCmp:
# log.debug(" -- already seen {0}".format(req))
# else:
dist, new_req = self.check_previous_requirement(
req, dist, prev_req)
return dist, prev_req, new_req
# Find the best distribution and add it to the map.
dist = self.ws.by_key.get(req.key)
if dist is not None:
log.debug(' -- dist in working set: %r' % dist)
# We get here when the dist was already installed.
return dist, None, None
try:
dist = self.env.best_match(req, self.ws)
except pkg_resources.VersionConflict as err:
raise VersionConflict(err, self.ws, None, self.best,
self.req_graph)
log.debug(" -- env best match is %r " % (dist))
if (dist is not None
and not (dist.location in self.installer._site_packages
and not self.installer.allow_site_package_egg(
dist.project_name))):
# We get here when things are in the egg cache, or
# deactivated in site-packages. Need to add to
# the working set or they don't get setup properly.
log.debug(' -- dist in environ: %r' % dist)
return dist, None, None
# If we didn't find a distribution in the environment, or what we found
# is from site-packages and not allowed to be there, try again.
log.debug(' -- %s required %r',
'getting' if self.installer._dest else 'adding', str(req))
easy_install._log_requirement(self.ws, req)
return self._get_dist(req), None, None
def resolve_requirement(self, req):
best = self.best.get(req.key, None)
log.debug(" -- previous best is %r " % (best,))
if best is not None:
dist, prev_req = best
# FIXME: In theory, if the previously found requirement, and the requirement
# we are processing are the same, we should be able to skip the
# checks for merging requirements process them as normal.
# In practice, merging a requirement with itself takes you down
# a differnet resolution route as it picks up stuff from the environment
# that we otherwise wouldn't have seen. Leaving this as-is for now
# until the egg-cache changes are in.
# if prev_req.hashCmp == req.hashCmp:
# log.debug(" -- already seen {0}".format(req))
# else:
dist, new_req = self.check_previous_requirement(req, dist, prev_req)
return dist, prev_req, new_req
# Find the best distribution and add it to the map.
dist = self.ws.by_key.get(req.key)
if dist is not None:
log.debug(' -- dist in working set: %r' % dist)
# We get here when the dist was already installed.
return dist, None, None
try:
dist = self.env.best_match(req, self.ws)
except pkg_resources.VersionConflict as err:
raise VersionConflict(err, self.ws, None, self.best, self.req_graph)
log.debug(" -- env best match is %r " % (dist))
if (dist is not None and
not (dist.location in self.installer._site_packages and
not self.installer.allow_site_package_egg(dist.project_name))):
# We get here when things are in the egg cache, or
# deactivated in site-packages. Need to add to
# the working set or they don't get setup properly.
log.debug(' -- dist in environ: %r' % dist)
return dist, None, None
# If we didn't find a distribution in the environment, or what we found
# is from site-packages and not allowed to be there, try again.
log.debug(' -- %s required %r',
'getting' if self.installer._dest else 'adding', str(req))
easy_install._log_requirement(self.ws, req)
return self._get_dist(req), None, None
def install(self, specs, working_set=None, use_existing=False,
draw_graph=False):
""" We've overridden the install function here to make the following
changes:
1) Add the ability to prefer already-installed packages over
newer packages from the server, if the requirement is already
satisfied (use_existing flag) dependency
2) fix behavior wrt dependency resolution. The original version
does not handle this case:
A B Where A and B depend on C, A doesnt care
\* /==2 which version of C it is and B is pinned to
C a specific version.
In the original if there is a later version of C available it will
pick that first and then conflict with B's pinned requirement.
In our version we attempt to override the version picked by A, and
replace it with the pinned version in B as it is a more specific
requirement - ie, order-by-specivicity.
"""
import networkx
# TODO: break this method down into manageable chunks.
log.debug('Installing requirements: %s', repr(specs)[1:-1])
# This is a set of processed requirements.
processed = {}
# This is the graph of requirements
req_graph = networkx.DiGraph()
# This is the list of stuff we've installed, to hand off to the
# postinstall steps like egg-link etc
setup_dists = pkg_resources.WorkingSet([])
path = self._path
destination = self._dest
if destination is not None and destination not in path:
path.insert(0, destination)
requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
for spec in specs]
if working_set is None:
ws = pkg_resources.WorkingSet([])
else:
# Make a copy, we don't want to mess up the global w/s if this is
# what's been passed in.
ws = pkg_resources.WorkingSet(working_set.entries)
# First we need to get a map of requirements for what is currently
# installed. This is so we can play them off against new requirements.
# For simplicity's sake, we merge all requirements matching installed
# packages into a single requirement. This also mimics how the packages
# would have been installed in the first place.
# This is a mapping of key -> (dist, originating req) which is our best
# found so far.
best = dependency.get_requirements_from_ws(ws, req_graph)
log.debug("Baseline working set: (merged req, dist)")
for dist, req in best.values():
log.debug(" %25s: %r" % (req, dist))
if draw_graph:
graph.draw_networkx_with_pydot(req_graph, True)
# Set up the stack, so we're popping from the front
requirements.reverse()
# This is our 'baseline' set of packages. Anything we've picked that
# isn't in here, hasn't yet been fully installed.
baseline = copy.copy(ws.entries)
env = pkg_resources.Environment(baseline)
def purge_req(req):
""" Purge a requirement from all our indexes, used for
backtracking
"""
if req.key in best:
del best[req.key]
[dependency.remove_from_ws(w, req._chosen_dist)
for w in (ws, setup_dists) if req._chosen_dist in w]
while requirements:
# Process dependencies breadth-first.
req = self._constrain(requirements.pop(0))
if req in processed:
# Ignore cyclic or redundant dependencies.
continue
# Add the req to the graph
req_graph.add_node(req)
log.debug('Processing %r' % req)
for r in req_graph.predecessors(req):
log.debug(' -- downstream: %r' % r)
dist, prev_req = best.get(req.key, (None, None))
log.debug(" previous best is %r (%r) " % (dist, prev_req))
if dist is None:
# Find the best distribution and add it to the map.
dist = ws.by_key.get(req.key)
if dist is None:
try:
dist = env.best_match(req, ws)
except pkg_resources.VersionConflict, err:
raise easy_install.VersionConflict(err, ws)
log.debug(" env best match is %r " % (dist))
if dist is None or (
dist.location in self._site_packages and not
self.allow_site_package_egg(dist.project_name)):
# If we didn't find a distribution in the
# environment, or what we found is from site
# packages and not allowed to be there, try
# again.
if destination:
log.debug(' getting required %r', str(req))
else:
log.debug(' adding required %r', str(req))
easy_install._log_requirement(ws, req)
for dist in self._get_dist(req,
ws, self._always_unzip):
ws.add(dist)
log.debug(' adding dist to target installs: %r',
dist)
setup_dists.add(dist)
else:
# We get here when things are in the egg cache, or
# deactivated in site-packages. Need to add to
# the working set or they don't get setup properly.
log.debug(' dist in environ: %r' % dist)
ws.add(dist)
setup_dists.add(dist)
log.debug(' adding dist to target installs: %r', dist)
best[req.key] = (dist, req)
log.debug(" best is now (%s): %r" % (req, dist))
else:
log.debug(' dist in working set: %r' % dist)
# We get here when the dist was already installed.
# TODO: check we don't need this
#setup_dists.add(dist)
else:
log.debug(' already have dist: %r' % dist)
if prev_req and prev_req.hashCmp != req.hashCmp:
log.debug("--- checking previously requirements: %s vs %s" %
(prev_req, req))
# Here is where we can possibly backtrack in our graph walking.
# We need to check if we can merge the new requirement with
# ones that we found previously. This merging is done on the
# rules of specivicity - ie, creating a new requirement that is
# bounded by the most specific specs from both old and new.
try:
merged_req = dependency.merge_requirements(prev_req, req)
log.debug("--- merged requirement: %s" % merged_req)
if dist in merged_req:
# The dist we've already picked matches the more new
# req, just update the 'best' index to the new one
if prev_req.hashCmp != merged_req.hashCmp:
log.debug("--- upgrading to more specific "
"requirement %s -> %s" % (prev_req,
merged_req))
best[req.key] = (dist, merged_req)
req = merged_req
# Add a new node in our graph for the merged
# requirement.
req_graph.add_node(req)
upstream = req_graph.successors(prev_req)
if upstream:
log.debug("---- adding edges from %s to %s" %
(req, upstream))
[req_graph.add_edge(req, i) for i in upstream]
else:
log.debug("--- skipping %s, it's more general than"
" %s" % (req, prev_req))
processed[req] = True
continue
# TODO: look @ req.extras?
else:
# The more specific version is different to what we've
# already found, we need to override it.
log.debug("**** overriding requirement %r with %r" %
(prev_req, req))
# Now we need to purge the old package and everything
# it brought in, so that there's no chance of conflicts
# with the new version we're about to install
log.debug("**** resolving possible backtrack "
"targets")
upstream_reqs = dependency.get_all_upstream(req_graph,
prev_req)
for upstream_req in upstream_reqs:
if not hasattr(upstream_req, '_chosen_dist'):
continue
upstream_dist = upstream_req._chosen_dist
# TODO : find a way to warn users here that makes sense, doing this for
# every package is misleading, as a lot of them will get re-chosen
# by the new requirement
#if target_dist.location in baseline:
# log.debug("**** target in baseline, we may be changing the environment")
if upstream_dist in ws or upstream_dist in setup_dists:
log.debug("**** pulling out backtrack target: %r" % upstream_dist)
# XXX this isn't working properly yet
# We need to check if there was more than one downstream
# source for this target, so that we're only pulling out the minimal
# set of packages from the graph.
#downstream_diffs = dependency.get_downstream_difference(
# req_graph, upstream_req, prev_req)
#if downstream_diffs:
# log.debug("**** %r has other downstream parents: %r" %
# (upstream_dist, downstream_diffs))
#else:
# ... do the purging
purge_req(upstream_req)
# Now purge the requirement we're replacing
purge_req(prev_req)
# Push the updated req back to the front of the queue
requirements.insert(0, merged_req)
continue
except dependency.CannotMergeError:
log.debug("--- cannot merge requirements")
pass
if dist not in req:
# Oops, the "best" so far conflicts with a dependency.
raise easy_install.VersionConflict(
pkg_resources.VersionConflict(dist, req), ws)
# If we get to this point, we're happy with this requirement and the distribution
# that has been found for it. Store a reference to this mapping, so we can get back
# to it if we need to backtrack.
req._chosen_dist = dist
for new_req in dist.requires(req.extras)[::-1]:
if not self._constrain(new_req) in processed.keys() + requirements:
log.debug(' new requirement: %s' % new_req)
requirements.append(new_req)
# Add the new requirements into the graph
req_graph.add_node(new_req)
# And an edge for the new req
req_graph.add_edge(req, new_req)
processed[req] = True
if dist.location in self._site_packages:
log.debug(' egg from site-packages: %s', dist)
log.debug(' finished processing %s' % req)
