コード例 #1
0
ファイル: buildout.py プロジェクト: pombredanne/pkglib
    def resolve_requirement(self, req):
        best = self.best.get(req.key, None)
        log.debug(" -- previous best is %r " % (best, ))
        if best is not None:
            dist, prev_req = best

            # FIXME: In theory, if the previously found requirement, and the requirement
            #        we are processing are the same, we should be able to skip the
            #        checks for merging requirements process them as normal.
            #        In practice, merging a requirement with itself takes you down
            #        a differnet resolution route as it picks up stuff from the environment
            #        that we otherwise wouldn't have seen.  Leaving this as-is for now
            #        until the egg-cache changes are in.

            # if prev_req.hashCmp == req.hashCmp:
            #    log.debug(" -- already seen {0}".format(req))
            # else:

            dist, new_req = self.check_previous_requirement(
                req, dist, prev_req)
            return dist, prev_req, new_req

        # Find the best distribution and add it to the map.
        dist = self.ws.by_key.get(req.key)
        if dist is not None:
            log.debug(' -- dist in working set: %r' % dist)
            # We get here when the dist was already installed.
            return dist, None, None

        try:
            dist = self.env.best_match(req, self.ws)
        except pkg_resources.VersionConflict as err:
            raise VersionConflict(err, self.ws, None, self.best,
                                  self.req_graph)
        log.debug(" -- env best match is %r " % (dist))
        if (dist is not None
                and not (dist.location in self.installer._site_packages
                         and not self.installer.allow_site_package_egg(
                             dist.project_name))):
            # We get here when things are in the egg cache, or
            # deactivated in site-packages. Need to add to
            # the working set or they don't get setup properly.
            log.debug(' -- dist in environ: %r' % dist)
            return dist, None, None

        # If we didn't find a distribution in the environment, or what we found
        # is from site-packages and not allowed to be there, try again.
        log.debug(' -- %s required %r',
                  'getting' if self.installer._dest else 'adding', str(req))
        easy_install._log_requirement(self.ws, req)
        return self._get_dist(req), None, None
コード例 #2
0
ファイル: buildout.py プロジェクト: agiledata/pkglib
    def resolve_requirement(self, req):
        best = self.best.get(req.key, None)
        log.debug(" -- previous best is %r " % (best,))
        if best is not None:
            dist, prev_req = best

            # FIXME: In theory, if the previously found requirement, and the requirement
            #        we are processing are the same, we should be able to skip the
            #        checks for merging requirements process them as normal.
            #        In practice, merging a requirement with itself takes you down
            #        a differnet resolution route as it picks up stuff from the environment
            #        that we otherwise wouldn't have seen.  Leaving this as-is for now
            #        until the egg-cache changes are in.

            # if prev_req.hashCmp == req.hashCmp:
            #    log.debug(" -- already seen {0}".format(req))
            # else:

            dist, new_req = self.check_previous_requirement(req, dist, prev_req)
            return dist, prev_req, new_req

        # Find the best distribution and add it to the map.
        dist = self.ws.by_key.get(req.key)
        if dist is not None:
            log.debug(' -- dist in working set: %r' % dist)
            # We get here when the dist was already installed.
            return dist, None, None

        try:
            dist = self.env.best_match(req, self.ws)
        except pkg_resources.VersionConflict as err:
            raise VersionConflict(err, self.ws, None, self.best, self.req_graph)
        log.debug(" -- env best match is %r " % (dist))
        if (dist is not None and
            not (dist.location in self.installer._site_packages and
                 not self.installer.allow_site_package_egg(dist.project_name))):
            # We get here when things are in the egg cache, or
            # deactivated in site-packages. Need to add to
            # the working set or they don't get setup properly.
            log.debug(' -- dist in environ: %r' % dist)
            return dist, None, None

        # If we didn't find a distribution in the environment, or what we found
        # is from site-packages and not allowed to be there, try again.
        log.debug(' -- %s required %r',
                  'getting' if self.installer._dest else 'adding', str(req))
        easy_install._log_requirement(self.ws, req)
        return self._get_dist(req), None, None
コード例 #3
0
ファイル: buildout.py プロジェクト: drkjam/pkglib
    def install(self, specs, working_set=None, use_existing=False,
                draw_graph=False):
        """ We've overridden the install function here to make the following
            changes:

            1) Add the ability to prefer already-installed packages over
               newer packages from the server, if the requirement is already
               satisfied (use_existing flag) dependency
            2) fix behavior wrt dependency resolution. The original version
               does not handle this case:
               A   B        Where A and B depend on C, A doesnt care
               \* /==2      which version of C it is and B is pinned to
                C           a specific version.

            In the original if there is a later version of C available it will
            pick that first and then conflict with B's pinned requirement.
            In our version we attempt to override the version picked by A, and
            replace it with the pinned version in B as it is a more specific
            requirement - ie, order-by-specivicity.

        """
        import networkx
        # TODO: break this method down into manageable chunks.
        log.debug('Installing requirements: %s', repr(specs)[1:-1])

        # This is a set of processed requirements.
        processed = {}

        # This is the graph of requirements
        req_graph = networkx.DiGraph()

        # This is the list of stuff we've installed, to hand off to the
        # postinstall steps like egg-link etc
        setup_dists = pkg_resources.WorkingSet([])

        path = self._path
        destination = self._dest
        if destination is not None and destination not in path:
            path.insert(0, destination)

        requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
                        for spec in specs]

        if working_set is None:
            ws = pkg_resources.WorkingSet([])
        else:
            # Make a copy, we don't want to mess up the global w/s if this is
            # what's been passed in.
            ws = pkg_resources.WorkingSet(working_set.entries)

        # First we need to get a map of requirements for what is currently
        # installed. This is so we can play them off against new requirements.

        # For simplicity's sake, we merge all requirements matching installed
        # packages into a single requirement. This also mimics how the packages
        # would have been installed in the first place.

        # This is a mapping of key -> (dist, originating req) which is our best
        # found so far.
        best = dependency.get_requirements_from_ws(ws, req_graph)

        log.debug("Baseline working set: (merged req, dist)")
        for dist, req in best.values():
            log.debug("   %25s: %r" % (req, dist))

        if draw_graph:
            graph.draw_networkx_with_pydot(req_graph, True)

        # Set up the stack, so we're popping from the front
        requirements.reverse()

        # This is our 'baseline' set of packages. Anything we've picked that
        # isn't in here, hasn't yet been fully installed.
        baseline = copy.copy(ws.entries)
        env = pkg_resources.Environment(baseline)

        def purge_req(req):
            """ Purge a requirement from all our indexes, used for
                backtracking
            """
            if req.key in best:
                del best[req.key]
            [dependency.remove_from_ws(w, req._chosen_dist)
             for w in (ws, setup_dists) if req._chosen_dist in w]

        while requirements:
            # Process dependencies breadth-first.
            req = self._constrain(requirements.pop(0))
            if req in processed:
                # Ignore cyclic or redundant dependencies.
                continue

            # Add the req to the graph
            req_graph.add_node(req)

            log.debug('Processing %r' % req)
            for r in req_graph.predecessors(req):
                log.debug(' -- downstream: %r' % r)

            dist, prev_req = best.get(req.key, (None, None))
            log.debug("  previous best is %r (%r) " % (dist, prev_req))

            if dist is None:
                # Find the best distribution and add it to the map.
                dist = ws.by_key.get(req.key)
                if dist is None:
                    try:
                        dist = env.best_match(req, ws)
                    except pkg_resources.VersionConflict, err:
                        raise easy_install.VersionConflict(err, ws)

                    log.debug("  env best match is %r " % (dist))
                    if dist is None or (
                        dist.location in self._site_packages and not
                        self.allow_site_package_egg(dist.project_name)):
                        # If we didn't find a distribution in the
                        # environment, or what we found is from site
                        # packages and not allowed to be there, try
                        # again.
                        if destination:
                            log.debug('  getting required %r', str(req))
                        else:
                            log.debug('  adding required %r', str(req))
                        easy_install._log_requirement(ws, req)
                        for dist in self._get_dist(req,
                                                   ws, self._always_unzip):
                            ws.add(dist)
                            log.debug('  adding dist to target installs: %r',
                                      dist)
                            setup_dists.add(dist)
                    else:
                        # We get here when things are in the egg cache, or
                        # deactivated in site-packages. Need to add to
                        # the working set or they don't get setup properly.
                        log.debug('  dist in environ: %r' % dist)
                        ws.add(dist)
                        setup_dists.add(dist)
                        log.debug('  adding dist to target installs: %r', dist)

                    best[req.key] = (dist, req)
                    log.debug("   best is now (%s): %r" % (req, dist))
                else:
                    log.debug('  dist in working set: %r' % dist)
                    # We get here when the dist was already installed.
                    # TODO: check we don't need this
                    #setup_dists.add(dist)

            else:
                log.debug('  already have dist: %r' % dist)

            if prev_req and prev_req.hashCmp != req.hashCmp:
                log.debug("--- checking previously requirements: %s vs %s" %
                          (prev_req, req))
                # Here is where we can possibly backtrack in our graph walking.

                # We need to check if we can merge the new requirement with
                # ones  that we found previously. This merging is done on the
                # rules of specivicity - ie, creating a new requirement that is
                # bounded by the most specific specs from both old and new.
                try:
                    merged_req = dependency.merge_requirements(prev_req, req)
                    log.debug("--- merged requirement: %s" % merged_req)

                    if dist in merged_req:
                        # The dist we've already picked matches the more new
                        # req, just update the 'best' index to the new one
                        if prev_req.hashCmp != merged_req.hashCmp:
                            log.debug("--- upgrading to more specific "
                                      "requirement %s -> %s" % (prev_req,
                                                                merged_req))
                            best[req.key] = (dist, merged_req)
                            req = merged_req

                            # Add a new node in our graph for the merged 
                            # requirement.
                            req_graph.add_node(req)
                            upstream = req_graph.successors(prev_req)
                            if upstream:
                                log.debug("---- adding edges from %s to %s" %
                                          (req, upstream))
                                [req_graph.add_edge(req, i) for i in upstream]
                        else:
                            log.debug("--- skipping %s, it's more general than"
                                      " %s" % (req, prev_req))
                            processed[req] = True
                            continue
                            # TODO: look @ req.extras?
                    else:
                        # The more specific version is different to what we've
                        # already found, we need to override it.
                        log.debug("**** overriding requirement %r with %r" %
                                  (prev_req, req))

                        # Now we need to purge the old package and everything
                        # it brought in, so that there's no chance of conflicts
                        # with the new version we're about to install

                        log.debug("****   resolving possible backtrack "
                                  "targets")

                        upstream_reqs = dependency.get_all_upstream(req_graph,
                                                                    prev_req)

                        for upstream_req in upstream_reqs:
                            if not hasattr(upstream_req, '_chosen_dist'):
                                continue
                            upstream_dist = upstream_req._chosen_dist

                            # TODO : find a way to warn users here that makes sense, doing this for
                            #        every package is misleading, as a lot of them will get re-chosen
                            #        by the new requirement

                            #if target_dist.location in baseline:
                            #    log.debug("**** target in baseline, we may be changing the environment")

                            if upstream_dist in ws or upstream_dist in setup_dists:
                                log.debug("**** pulling out backtrack target: %r" % upstream_dist)

                                # XXX this isn't working properly yet
                                # We need to check if there was more than one downstream
                                # source for this target, so that we're only pulling out the minimal
                                # set of packages from the graph.
                                #downstream_diffs = dependency.get_downstream_difference(
                                #            req_graph, upstream_req, prev_req)

                                #if downstream_diffs:
                                #    log.debug("**** %r has other downstream parents: %r" %
                                #              (upstream_dist, downstream_diffs))
                                #else:
                                # ...  do the purging

                            purge_req(upstream_req)

                        # Now purge the requirement we're replacing
                        purge_req(prev_req)

                        # Push the updated req back to the front of the queue
                        requirements.insert(0, merged_req)
                        continue

                except dependency.CannotMergeError:
                    log.debug("--- cannot merge requirements")
                    pass

            if dist not in req:
                # Oops, the "best" so far conflicts with a dependency.
                raise easy_install.VersionConflict(
                    pkg_resources.VersionConflict(dist, req), ws)

            # If we get to this point, we're happy with this requirement and the distribution
            # that has been found for it. Store a reference to this mapping, so we can get back
            # to it if we need to backtrack.
            req._chosen_dist = dist

            for new_req in dist.requires(req.extras)[::-1]:
                if not self._constrain(new_req) in processed.keys() + requirements:
                    log.debug('  new requirement: %s' % new_req)
                    requirements.append(new_req)

                    # Add the new requirements into the graph
                    req_graph.add_node(new_req)

                    # And an edge for the new req
                    req_graph.add_edge(req, new_req)

            processed[req] = True
            if dist.location in self._site_packages:
                log.debug('  egg from site-packages: %s', dist)
            log.debug('  finished processing %s' % req)