Beispiel #1
0
    def test_cycle(self):
        data = {'a':'b', 'b':'a'}

        with self.assertRaises(ValueError):
            toposort(data, False)

        results = toposort(data)
        # Results do not have an guaranteed order
        self.assertEqual(set(results), {'b', 'a'})
Beispiel #2
0
    def test_cycle(self):
        data = {'a': 'b', 'b': 'a'}

        with self.assertRaises(ValueError):
            toposort(data, False)

        results = toposort(data)
        # Results do not have an guaranteed order
        self.assertEqual(set(results), {'b', 'a'})
Beispiel #3
0
    def graph_sort(self, must_have):
        def lookup(value):
            index_data = self.index.get('%s.tar.bz2' % value, {})
            return {
                item.split(' ', 1)[0]
                for item in index_data.get('depends', [])
            }

        digraph = {}

        for key, value in must_have.items():
            depends = lookup(value)
            digraph[key] = depends

        sorted_keys = toposort(digraph)

        must_have = must_have.copy()
        # Take all of the items in the sorted keys
        # Don't fail if the key does not exist
        result = [
            must_have.pop(key) for key in sorted_keys if key in must_have
        ]

        # Take any key that were not sorted
        result.extend(must_have.values())

        return result
Beispiel #4
0
def implicated_packages(specs: List[str], r: Resolve) -> List[str]:
    """Get a list of all packages implicated as possible direct or indirect
    depdencies of ``specs``.

    Example
    -------
    >>> r = Resolve(index)
    >>> specs = ('python 3.5*', 'numpy 1.9*', 'statsmodels')
    >>> implicated_packages(specs, r)
    ['msvc_runtime', 'python', 'distribute', 'numpy', 'pytz', 'setuptools',
     'six', 'wheel', 'dateutil', 'patsy', 'pip', 'python-dateutil', 'scipy',
     'pandas', 'statsmodels']
    """
    depgraph = defaultdict(lambda: set())  # type: Dict[str, Set[str]]

    def add_package(spec: str) -> None:
        ms = MatchSpec(spec)
        name = ms.name

        if name in depgraph:
            return

        depnames = {
            d.name
            for fn in r.find_matches(ms) for d in r.ms_depends(fn)
        }
        for depname in depnames:
            depgraph[name].add(depname)
            add_package(depname)

    for spec in specs:
        add_package(spec)
    return toposort(depgraph)
Beispiel #5
0
def implicated_packages(specs: List[str], r: Resolve) -> List[str]:
    """Get a list of all packages implicated as possible direct or indirect
    depdencies of ``specs``.

    Example
    -------
    >>> r = Resolve(index)
    >>> specs = ('python 3.5*', 'numpy 1.9*', 'statsmodels')
    >>> implicated_packages(specs, r)
    ['msvc_runtime', 'python', 'distribute', 'numpy', 'pytz', 'setuptools',
     'six', 'wheel', 'dateutil', 'patsy', 'pip', 'python-dateutil', 'scipy',
     'pandas', 'statsmodels']
    """
    depgraph = defaultdict(lambda: set())  # type: Dict[str, Set[str]]

    def add_package(spec: str) -> None:
        ms = MatchSpec(spec)
        name = ms.name

        if name in depgraph:
            return

        depnames = {d.name for fn in r.find_matches(ms) for d in r.ms_depends(fn)}
        for depname in depnames:
            depgraph[name].add(depname)
            add_package(depname)

    for spec in specs:
        add_package(spec)
    return toposort(depgraph)
Beispiel #6
0
    def test_cycle_best_effort(self):
        data = {'a':'bc', 'b':'c', '1':'2', '2':'1'}

        results = toposort(data)
        self.assertEqual(results[:3], ['c', 'b', 'a'])

        # Cycles come last
        # Results do not have an guaranteed order
        self.assertEqual(set(results[3:]), {'1', '2'})
Beispiel #7
0
    def test_cycle_best_effort(self):
        data = {'a': 'bc', 'b': 'c', '1': '2', '2': '1'}

        results = toposort(data)
        self.assertEqual(results[:3], ['c', 'b', 'a'])

        # Cycles come last
        # Results do not have an guaranteed order
        self.assertEqual(set(results[3:]), {'1', '2'})
def dependencies_for_env(prefix):
    """
    Return an OrderedDict of all packages in the packages in the
    given environment and each one's list of dependencies.
    The returned items are in toposort order. 
    """
    # Get all package strings as 'name-version-build'
    installed = conda.install.linked(prefix)
    exitcode, packages = list_packages(prefix,
                                       installed,
                                       regex=None,
                                       format='canonical',
                                       show_channel_urls=False)

    # If present, remove channel prefix (e.g. from 'ilastik::boost=1.55.0=5')
    packages = map(lambda p: p.split('::')[-1], packages)

    # Replace last two '-' with '='
    packages = map(lambda p: p[::-1].replace('-', '=', 2)[::-1], packages)

    # Load dependencies into a dict
    index = conda.api.get_index()
    r = Resolve(index)

    deps_dict = {}
    for package in packages:
        try:
            versions = r.get_pkgs(conda.cli.common.arg2spec(package))
        except NoPackagesFound:
            print("Skipping " + package, file=sys.stderr)
        else:
            for pkg in sorted(versions):
                deps_dict[pkg.name] = []
                for dep in pkg.info['depends']:
                    deps_dict[pkg.name].append(dep.split(' ')[0])

    # If a package's dependencies have been updated recently on the server,
    # there may be entries in the deps list that aren't actually present in our environment.
    # In that case, we just omit that dependency.
    for pkg_name, deps in deps_dict.items():
        to_remove = []
        for dep in deps:
            if dep not in deps_dict:
                to_remove.append(dep)
        for dep in to_remove:
            deps_dict[pkg_name].remove(dep)

    # Convenience: Return dict with keys in topologically sorted order
    sorted_keys = toposort(deps_dict)
    #print('\n'.join(sorted_keys))

    deps_dict = collections.OrderedDict(
        map(lambda k: (k, deps_dict[k]), sorted_keys))
    return deps_dict
Beispiel #9
0
 def dependency_sort(self, must_have):
     def lookup(value):
         return set(ms.name for ms in self.ms_depends(value + '.tar.bz2'))
     digraph = {}
     for key, value in iteritems(must_have):
         depends = lookup(value)
         digraph[key] = depends
     sorted_keys = toposort(digraph)
     must_have = must_have.copy()
     # Take all of the items in the sorted keys
     # Don't fail if the key does not exist
     result = [must_have.pop(key) for key in sorted_keys if key in must_have]
     # Take any key that were not sorted
     result.extend(must_have.values())
     return result
Beispiel #10
0
 def dependency_sort(self, must_have):
     def lookup(value):
         return set(ms.name for ms in self.ms_depends(value + '.tar.bz2'))
     digraph = {}
     for key, value in iteritems(must_have):
         depends = lookup(value)
         digraph[key] = depends
     sorted_keys = toposort(digraph)
     must_have = must_have.copy()
     # Take all of the items in the sorted keys
     # Don't fail if the key does not exist
     result = [must_have.pop(key) for key in sorted_keys if key in must_have]
     # Take any key that were not sorted
     result.extend(must_have.values())
     return result
Beispiel #11
0
 def dependency_sort(self, must_have):
     if not isinstance(must_have, dict):
         must_have = {self.package_name(dist): dist for dist in must_have}
     digraph = {}
     for key, value in iteritems(must_have):
         fn = value + '.tar.bz2'
         if fn in self.index:
             depends = set(ms.name for ms in self.ms_depends(fn))
             digraph[key] = depends
     sorted_keys = toposort(digraph)
     must_have = must_have.copy()
     # Take all of the items in the sorted keys
     # Don't fail if the key does not exist
     result = [must_have.pop(key) for key in sorted_keys if key in must_have]
     # Take any key that were not sorted
     result.extend(must_have.values())
     return result
Beispiel #12
0
 def dependency_sort(self, must_have):
     if not isinstance(must_have, dict):
         must_have = {self.package_name(dist): dist for dist in must_have}
     digraph = {}
     for key, value in iteritems(must_have):
         fn = value + '.tar.bz2'
         if fn in self.index:
             depends = set(ms.name for ms in self.ms_depends(fn))
             digraph[key] = depends
     sorted_keys = toposort(digraph)
     must_have = must_have.copy()
     # Take all of the items in the sorted keys
     # Don't fail if the key does not exist
     result = [must_have.pop(key) for key in sorted_keys if key in must_have]
     # Take any key that were not sorted
     result.extend(must_have.values())
     return result
Beispiel #13
0
    def test_python_is_prioritized(self):
        """
        This test checks a special invariant related to 'python' specifically.
        Python is part of a cycle (pip <--> python), which can cause it to be
        installed *after* packages that need python (possibly in 
        post-install.sh).
        
        A special case in toposort() breaks the cycle, to ensure that python
        isn't installed too late.  Here, we verify that it works.
        """
        # This is the actual dependency graph for python (as of the time of this writing, anyway)
        data = {
            'python':
            ['pip', 'openssl', 'readline', 'sqlite', 'tk', 'xz', 'zlib'],
            'pip': ['python', 'setuptools', 'wheel'],
            'setuptools': ['python'],
            'wheel': ['python'],
            'openssl': [],
            'readline': [],
            'sqlite': [],
            'tk': [],
            'xz': [],
            'zlib': []
        }

        # Here are some extra pure-python libs, just for good measure.
        data.update({
            'psutil': ['python'],
            'greenlet': ['python'],
            'futures': ['python'],
            'six': ['python']
        })

        results = toposort(data)

        # Python always comes before things that need it!
        self.assertLess(results.index('python'), results.index('setuptools'))
        self.assertLess(results.index('python'), results.index('wheel'))
        self.assertLess(results.index('python'), results.index('pip'))
        self.assertLess(results.index('python'), results.index('psutil'))
        self.assertLess(results.index('python'), results.index('greenlet'))
        self.assertLess(results.index('python'), results.index('futures'))
        self.assertLess(results.index('python'), results.index('six'))
Beispiel #14
0
    def graph_sort(self, must_have):
        def lookup(value):
            index_data = self.index.get("%s.tar.bz2" % value, {})
            return {item.split(" ", 1)[0] for item in index_data.get("depends", [])}

        digraph = {}

        for key, value in must_have.items():
            depends = lookup(value)
            digraph[key] = depends

        sorted_keys = toposort(digraph)

        must_have = must_have.copy()
        # Take all of the items in the sorted keys
        # Don't fail if the key does not exist
        result = [must_have.pop(key) for key in sorted_keys if key in must_have]

        # Take any key that were not sorted
        result.extend(must_have.values())

        return result
Beispiel #15
0
    def test_python_is_prioritized(self):
        """
        This test checks a special invariant related to 'python' specifically.
        Python is part of a cycle (pip <--> python), which can cause it to be
        installed *after* packages that need python (possibly in 
        post-install.sh).
        
        A special case in toposort() breaks the cycle, to ensure that python
        isn't installed too late.  Here, we verify that it works.
        """
        # This is the actual dependency graph for python (as of the time of this writing, anyway)
        data = {'python' : ['pip', 'openssl', 'readline', 'sqlite', 'tk', 'xz', 'zlib'],
                'pip': ['python', 'setuptools', 'wheel'],
                'setuptools' : ['python'],
                'wheel' : ['python'],
                'openssl' : [],
                'readline' : [],
                'sqlite' : [],
                'tk' : [],
                'xz' : [],
                'zlib' : []}

        # Here are some extra pure-python libs, just for good measure.
        data.update({'psutil' : ['python'],
                     'greenlet' : ['python'],
                     'futures' : ['python'],
                     'six' : ['python']})

        results = toposort(data)

        # Python always comes before things that need it!
        self.assertLess(results.index('python'), results.index('setuptools'))
        self.assertLess(results.index('python'), results.index('wheel'))
        self.assertLess(results.index('python'), results.index('pip'))
        self.assertLess(results.index('python'), results.index('psutil'))
        self.assertLess(results.index('python'), results.index('greenlet'))
        self.assertLess(results.index('python'), results.index('futures'))
        self.assertLess(results.index('python'), results.index('six'))
def dependencies_for_env( prefix ):
    """
    Return an OrderedDict of all packages in the packages in the
    given environment and each one's list of dependencies.
    The returned items are in toposort order. 
    """
    # Get all package strings as 'name-version-build'
    installed = conda.install.linked(prefix)
    exitcode, packages = list_packages(prefix, installed, regex=None, format='canonical', show_channel_urls=False)

    # If present, remove channel prefix (e.g. from 'ilastik::boost=1.55.0=5')
    packages = map( lambda p: p.split('::')[-1], packages )

    # Replace last two '-' with '='
    packages = map(lambda p: p[::-1].replace('-', '=', 2)[::-1], packages)

    # Load dependencies into a dict
    index = conda.api.get_index()
    r = Resolve(index)
    
    deps_dict = {}
    for package in packages:
        try:
            versions = r.get_pkgs(conda.cli.common.arg2spec(package))
        except NoPackagesFound:
            print("Skipping " + package, file=sys.stderr)
        else:
            for pkg in sorted(versions):
                deps_dict[pkg.name] = []
                for dep in pkg.info['depends']:
                    deps_dict[pkg.name].append(dep.split(' ')[0])

    # Convenience: Return dict with keys in topologically sorted order
    sorted_keys = toposort( deps_dict )
    deps_dict = collections.OrderedDict( map(lambda k: (k, deps_dict[k]), sorted_keys ) )
    return deps_dict
Beispiel #17
0
 def test_simple(self):
     data = {'a':'bc', 'b':'c'}
     results = toposort(data, safe=True)
     self.assertEqual(results, ['c', 'b', 'a'])
     results = toposort(data, safe=False)
     self.assertEqual(results, ['c', 'b', 'a'])
Beispiel #18
0
 def test_simple(self):
     data = {'a': 'bc', 'b': 'c'}
     results = toposort(data, safe=True)
     self.assertEqual(results, ['c', 'b', 'a'])
     results = toposort(data, safe=False)
     self.assertEqual(results, ['c', 'b', 'a'])