def test_cycle(self): data = {'a':'b', 'b':'a'} with self.assertRaises(ValueError): toposort(data, False) results = toposort(data) # Results do not have an guaranteed order self.assertEqual(set(results), {'b', 'a'})
def test_cycle(self): data = {'a': 'b', 'b': 'a'} with self.assertRaises(ValueError): toposort(data, False) results = toposort(data) # Results do not have an guaranteed order self.assertEqual(set(results), {'b', 'a'})
def graph_sort(self, must_have): def lookup(value): index_data = self.index.get('%s.tar.bz2' % value, {}) return { item.split(' ', 1)[0] for item in index_data.get('depends', []) } digraph = {} for key, value in must_have.items(): depends = lookup(value) digraph[key] = depends sorted_keys = toposort(digraph) must_have = must_have.copy() # Take all of the items in the sorted keys # Don't fail if the key does not exist result = [ must_have.pop(key) for key in sorted_keys if key in must_have ] # Take any key that were not sorted result.extend(must_have.values()) return result
def implicated_packages(specs: List[str], r: Resolve) -> List[str]: """Get a list of all packages implicated as possible direct or indirect depdencies of ``specs``. Example ------- >>> r = Resolve(index) >>> specs = ('python 3.5*', 'numpy 1.9*', 'statsmodels') >>> implicated_packages(specs, r) ['msvc_runtime', 'python', 'distribute', 'numpy', 'pytz', 'setuptools', 'six', 'wheel', 'dateutil', 'patsy', 'pip', 'python-dateutil', 'scipy', 'pandas', 'statsmodels'] """ depgraph = defaultdict(lambda: set()) # type: Dict[str, Set[str]] def add_package(spec: str) -> None: ms = MatchSpec(spec) name = ms.name if name in depgraph: return depnames = { d.name for fn in r.find_matches(ms) for d in r.ms_depends(fn) } for depname in depnames: depgraph[name].add(depname) add_package(depname) for spec in specs: add_package(spec) return toposort(depgraph)
def implicated_packages(specs: List[str], r: Resolve) -> List[str]: """Get a list of all packages implicated as possible direct or indirect depdencies of ``specs``. Example ------- >>> r = Resolve(index) >>> specs = ('python 3.5*', 'numpy 1.9*', 'statsmodels') >>> implicated_packages(specs, r) ['msvc_runtime', 'python', 'distribute', 'numpy', 'pytz', 'setuptools', 'six', 'wheel', 'dateutil', 'patsy', 'pip', 'python-dateutil', 'scipy', 'pandas', 'statsmodels'] """ depgraph = defaultdict(lambda: set()) # type: Dict[str, Set[str]] def add_package(spec: str) -> None: ms = MatchSpec(spec) name = ms.name if name in depgraph: return depnames = {d.name for fn in r.find_matches(ms) for d in r.ms_depends(fn)} for depname in depnames: depgraph[name].add(depname) add_package(depname) for spec in specs: add_package(spec) return toposort(depgraph)
def test_cycle_best_effort(self): data = {'a':'bc', 'b':'c', '1':'2', '2':'1'} results = toposort(data) self.assertEqual(results[:3], ['c', 'b', 'a']) # Cycles come last # Results do not have an guaranteed order self.assertEqual(set(results[3:]), {'1', '2'})
def test_cycle_best_effort(self): data = {'a': 'bc', 'b': 'c', '1': '2', '2': '1'} results = toposort(data) self.assertEqual(results[:3], ['c', 'b', 'a']) # Cycles come last # Results do not have an guaranteed order self.assertEqual(set(results[3:]), {'1', '2'})
def dependencies_for_env(prefix): """ Return an OrderedDict of all packages in the packages in the given environment and each one's list of dependencies. The returned items are in toposort order. """ # Get all package strings as 'name-version-build' installed = conda.install.linked(prefix) exitcode, packages = list_packages(prefix, installed, regex=None, format='canonical', show_channel_urls=False) # If present, remove channel prefix (e.g. from 'ilastik::boost=1.55.0=5') packages = map(lambda p: p.split('::')[-1], packages) # Replace last two '-' with '=' packages = map(lambda p: p[::-1].replace('-', '=', 2)[::-1], packages) # Load dependencies into a dict index = conda.api.get_index() r = Resolve(index) deps_dict = {} for package in packages: try: versions = r.get_pkgs(conda.cli.common.arg2spec(package)) except NoPackagesFound: print("Skipping " + package, file=sys.stderr) else: for pkg in sorted(versions): deps_dict[pkg.name] = [] for dep in pkg.info['depends']: deps_dict[pkg.name].append(dep.split(' ')[0]) # If a package's dependencies have been updated recently on the server, # there may be entries in the deps list that aren't actually present in our environment. # In that case, we just omit that dependency. for pkg_name, deps in deps_dict.items(): to_remove = [] for dep in deps: if dep not in deps_dict: to_remove.append(dep) for dep in to_remove: deps_dict[pkg_name].remove(dep) # Convenience: Return dict with keys in topologically sorted order sorted_keys = toposort(deps_dict) #print('\n'.join(sorted_keys)) deps_dict = collections.OrderedDict( map(lambda k: (k, deps_dict[k]), sorted_keys)) return deps_dict
def dependency_sort(self, must_have): def lookup(value): return set(ms.name for ms in self.ms_depends(value + '.tar.bz2')) digraph = {} for key, value in iteritems(must_have): depends = lookup(value) digraph[key] = depends sorted_keys = toposort(digraph) must_have = must_have.copy() # Take all of the items in the sorted keys # Don't fail if the key does not exist result = [must_have.pop(key) for key in sorted_keys if key in must_have] # Take any key that were not sorted result.extend(must_have.values()) return result
def dependency_sort(self, must_have): if not isinstance(must_have, dict): must_have = {self.package_name(dist): dist for dist in must_have} digraph = {} for key, value in iteritems(must_have): fn = value + '.tar.bz2' if fn in self.index: depends = set(ms.name for ms in self.ms_depends(fn)) digraph[key] = depends sorted_keys = toposort(digraph) must_have = must_have.copy() # Take all of the items in the sorted keys # Don't fail if the key does not exist result = [must_have.pop(key) for key in sorted_keys if key in must_have] # Take any key that were not sorted result.extend(must_have.values()) return result
def test_python_is_prioritized(self): """ This test checks a special invariant related to 'python' specifically. Python is part of a cycle (pip <--> python), which can cause it to be installed *after* packages that need python (possibly in post-install.sh). A special case in toposort() breaks the cycle, to ensure that python isn't installed too late. Here, we verify that it works. """ # This is the actual dependency graph for python (as of the time of this writing, anyway) data = { 'python': ['pip', 'openssl', 'readline', 'sqlite', 'tk', 'xz', 'zlib'], 'pip': ['python', 'setuptools', 'wheel'], 'setuptools': ['python'], 'wheel': ['python'], 'openssl': [], 'readline': [], 'sqlite': [], 'tk': [], 'xz': [], 'zlib': [] } # Here are some extra pure-python libs, just for good measure. data.update({ 'psutil': ['python'], 'greenlet': ['python'], 'futures': ['python'], 'six': ['python'] }) results = toposort(data) # Python always comes before things that need it! self.assertLess(results.index('python'), results.index('setuptools')) self.assertLess(results.index('python'), results.index('wheel')) self.assertLess(results.index('python'), results.index('pip')) self.assertLess(results.index('python'), results.index('psutil')) self.assertLess(results.index('python'), results.index('greenlet')) self.assertLess(results.index('python'), results.index('futures')) self.assertLess(results.index('python'), results.index('six'))
def graph_sort(self, must_have): def lookup(value): index_data = self.index.get("%s.tar.bz2" % value, {}) return {item.split(" ", 1)[0] for item in index_data.get("depends", [])} digraph = {} for key, value in must_have.items(): depends = lookup(value) digraph[key] = depends sorted_keys = toposort(digraph) must_have = must_have.copy() # Take all of the items in the sorted keys # Don't fail if the key does not exist result = [must_have.pop(key) for key in sorted_keys if key in must_have] # Take any key that were not sorted result.extend(must_have.values()) return result
def test_python_is_prioritized(self): """ This test checks a special invariant related to 'python' specifically. Python is part of a cycle (pip <--> python), which can cause it to be installed *after* packages that need python (possibly in post-install.sh). A special case in toposort() breaks the cycle, to ensure that python isn't installed too late. Here, we verify that it works. """ # This is the actual dependency graph for python (as of the time of this writing, anyway) data = {'python' : ['pip', 'openssl', 'readline', 'sqlite', 'tk', 'xz', 'zlib'], 'pip': ['python', 'setuptools', 'wheel'], 'setuptools' : ['python'], 'wheel' : ['python'], 'openssl' : [], 'readline' : [], 'sqlite' : [], 'tk' : [], 'xz' : [], 'zlib' : []} # Here are some extra pure-python libs, just for good measure. data.update({'psutil' : ['python'], 'greenlet' : ['python'], 'futures' : ['python'], 'six' : ['python']}) results = toposort(data) # Python always comes before things that need it! self.assertLess(results.index('python'), results.index('setuptools')) self.assertLess(results.index('python'), results.index('wheel')) self.assertLess(results.index('python'), results.index('pip')) self.assertLess(results.index('python'), results.index('psutil')) self.assertLess(results.index('python'), results.index('greenlet')) self.assertLess(results.index('python'), results.index('futures')) self.assertLess(results.index('python'), results.index('six'))
def dependencies_for_env( prefix ): """ Return an OrderedDict of all packages in the packages in the given environment and each one's list of dependencies. The returned items are in toposort order. """ # Get all package strings as 'name-version-build' installed = conda.install.linked(prefix) exitcode, packages = list_packages(prefix, installed, regex=None, format='canonical', show_channel_urls=False) # If present, remove channel prefix (e.g. from 'ilastik::boost=1.55.0=5') packages = map( lambda p: p.split('::')[-1], packages ) # Replace last two '-' with '=' packages = map(lambda p: p[::-1].replace('-', '=', 2)[::-1], packages) # Load dependencies into a dict index = conda.api.get_index() r = Resolve(index) deps_dict = {} for package in packages: try: versions = r.get_pkgs(conda.cli.common.arg2spec(package)) except NoPackagesFound: print("Skipping " + package, file=sys.stderr) else: for pkg in sorted(versions): deps_dict[pkg.name] = [] for dep in pkg.info['depends']: deps_dict[pkg.name].append(dep.split(' ')[0]) # Convenience: Return dict with keys in topologically sorted order sorted_keys = toposort( deps_dict ) deps_dict = collections.OrderedDict( map(lambda k: (k, deps_dict[k]), sorted_keys ) ) return deps_dict
def test_simple(self): data = {'a':'bc', 'b':'c'} results = toposort(data, safe=True) self.assertEqual(results, ['c', 'b', 'a']) results = toposort(data, safe=False) self.assertEqual(results, ['c', 'b', 'a'])
def test_simple(self): data = {'a': 'bc', 'b': 'c'} results = toposort(data, safe=True) self.assertEqual(results, ['c', 'b', 'a']) results = toposort(data, safe=False) self.assertEqual(results, ['c', 'b', 'a'])