def test_cycle(self):
data = {'a':'b', 'b':'a'}
with self.assertRaises(ValueError):
toposort(data, False)
results = toposort(data)
# Results do not have an guaranteed order
self.assertEqual(set(results), {'b', 'a'})
def test_cycle(self):
data = {'a': 'b', 'b': 'a'}
with self.assertRaises(ValueError):
toposort(data, False)
results = toposort(data)
# Results do not have an guaranteed order
self.assertEqual(set(results), {'b', 'a'})
def graph_sort(self, must_have):
def lookup(value):
index_data = self.index.get('%s.tar.bz2' % value, {})
return {
item.split(' ', 1)[0]
for item in index_data.get('depends', [])
}
digraph = {}
for key, value in must_have.items():
depends = lookup(value)
digraph[key] = depends
sorted_keys = toposort(digraph)
must_have = must_have.copy()
# Take all of the items in the sorted keys
# Don't fail if the key does not exist
result = [
must_have.pop(key) for key in sorted_keys if key in must_have
]
# Take any key that were not sorted
result.extend(must_have.values())
return result
def implicated_packages(specs: List[str], r: Resolve) -> List[str]:
"""Get a list of all packages implicated as possible direct or indirect
depdencies of ``specs``.
Example
-------
>>> r = Resolve(index)
>>> specs = ('python 3.5*', 'numpy 1.9*', 'statsmodels')
>>> implicated_packages(specs, r)
['msvc_runtime', 'python', 'distribute', 'numpy', 'pytz', 'setuptools',
'six', 'wheel', 'dateutil', 'patsy', 'pip', 'python-dateutil', 'scipy',
'pandas', 'statsmodels']
"""
depgraph = defaultdict(lambda: set()) # type: Dict[str, Set[str]]
def add_package(spec: str) -> None:
ms = MatchSpec(spec)
name = ms.name
if name in depgraph:
return
depnames = {
d.name
for fn in r.find_matches(ms) for d in r.ms_depends(fn)
}
for depname in depnames:
depgraph[name].add(depname)
add_package(depname)
for spec in specs:
add_package(spec)
return toposort(depgraph)
def implicated_packages(specs: List[str], r: Resolve) -> List[str]:
"""Get a list of all packages implicated as possible direct or indirect
depdencies of ``specs``.
Example
-------
>>> r = Resolve(index)
>>> specs = ('python 3.5*', 'numpy 1.9*', 'statsmodels')
>>> implicated_packages(specs, r)
['msvc_runtime', 'python', 'distribute', 'numpy', 'pytz', 'setuptools',
'six', 'wheel', 'dateutil', 'patsy', 'pip', 'python-dateutil', 'scipy',
'pandas', 'statsmodels']
"""
depgraph = defaultdict(lambda: set()) # type: Dict[str, Set[str]]
def add_package(spec: str) -> None:
ms = MatchSpec(spec)
name = ms.name
if name in depgraph:
return
depnames = {d.name for fn in r.find_matches(ms) for d in r.ms_depends(fn)}
for depname in depnames:
depgraph[name].add(depname)
add_package(depname)
for spec in specs:
add_package(spec)
return toposort(depgraph)
def test_cycle_best_effort(self):
data = {'a':'bc', 'b':'c', '1':'2', '2':'1'}
results = toposort(data)
self.assertEqual(results[:3], ['c', 'b', 'a'])
# Cycles come last
# Results do not have an guaranteed order
self.assertEqual(set(results[3:]), {'1', '2'})
def test_cycle_best_effort(self):
data = {'a': 'bc', 'b': 'c', '1': '2', '2': '1'}
results = toposort(data)
self.assertEqual(results[:3], ['c', 'b', 'a'])
# Cycles come last
# Results do not have an guaranteed order
self.assertEqual(set(results[3:]), {'1', '2'})
def dependencies_for_env(prefix):
"""
Return an OrderedDict of all packages in the packages in the
given environment and each one's list of dependencies.
The returned items are in toposort order.
"""
# Get all package strings as 'name-version-build'
installed = conda.install.linked(prefix)
exitcode, packages = list_packages(prefix,
installed,
regex=None,
format='canonical',
show_channel_urls=False)
# If present, remove channel prefix (e.g. from 'ilastik::boost=1.55.0=5')
packages = map(lambda p: p.split('::')[-1], packages)
# Replace last two '-' with '='
packages = map(lambda p: p[::-1].replace('-', '=', 2)[::-1], packages)
# Load dependencies into a dict
index = conda.api.get_index()
r = Resolve(index)
deps_dict = {}
for package in packages:
try:
versions = r.get_pkgs(conda.cli.common.arg2spec(package))
except NoPackagesFound:
print("Skipping " + package, file=sys.stderr)
else:
for pkg in sorted(versions):
deps_dict[pkg.name] = []
for dep in pkg.info['depends']:
deps_dict[pkg.name].append(dep.split(' ')[0])
# If a package's dependencies have been updated recently on the server,
# there may be entries in the deps list that aren't actually present in our environment.
# In that case, we just omit that dependency.
for pkg_name, deps in deps_dict.items():
to_remove = []
for dep in deps:
if dep not in deps_dict:
to_remove.append(dep)
for dep in to_remove:
deps_dict[pkg_name].remove(dep)
# Convenience: Return dict with keys in topologically sorted order
sorted_keys = toposort(deps_dict)
#print('\n'.join(sorted_keys))
deps_dict = collections.OrderedDict(
map(lambda k: (k, deps_dict[k]), sorted_keys))
return deps_dict
def dependency_sort(self, must_have):
def lookup(value):
return set(ms.name for ms in self.ms_depends(value + '.tar.bz2'))
digraph = {}
for key, value in iteritems(must_have):
depends = lookup(value)
digraph[key] = depends
sorted_keys = toposort(digraph)
must_have = must_have.copy()
# Take all of the items in the sorted keys
# Don't fail if the key does not exist
result = [must_have.pop(key) for key in sorted_keys if key in must_have]
# Take any key that were not sorted
result.extend(must_have.values())
return result
def dependency_sort(self, must_have):
def lookup(value):
return set(ms.name for ms in self.ms_depends(value + '.tar.bz2'))
digraph = {}
for key, value in iteritems(must_have):
depends = lookup(value)
digraph[key] = depends
sorted_keys = toposort(digraph)
must_have = must_have.copy()
# Take all of the items in the sorted keys
# Don't fail if the key does not exist
result = [must_have.pop(key) for key in sorted_keys if key in must_have]
# Take any key that were not sorted
result.extend(must_have.values())
return result
def dependency_sort(self, must_have):
if not isinstance(must_have, dict):
must_have = {self.package_name(dist): dist for dist in must_have}
digraph = {}
for key, value in iteritems(must_have):
fn = value + '.tar.bz2'
if fn in self.index:
depends = set(ms.name for ms in self.ms_depends(fn))
digraph[key] = depends
sorted_keys = toposort(digraph)
must_have = must_have.copy()
# Take all of the items in the sorted keys
# Don't fail if the key does not exist
result = [must_have.pop(key) for key in sorted_keys if key in must_have]
# Take any key that were not sorted
result.extend(must_have.values())
return result
def dependency_sort(self, must_have):
if not isinstance(must_have, dict):
must_have = {self.package_name(dist): dist for dist in must_have}
digraph = {}
for key, value in iteritems(must_have):
fn = value + '.tar.bz2'
if fn in self.index:
depends = set(ms.name for ms in self.ms_depends(fn))
digraph[key] = depends
sorted_keys = toposort(digraph)
must_have = must_have.copy()
# Take all of the items in the sorted keys
# Don't fail if the key does not exist
result = [must_have.pop(key) for key in sorted_keys if key in must_have]
# Take any key that were not sorted
result.extend(must_have.values())
return result
def test_python_is_prioritized(self):
"""
This test checks a special invariant related to 'python' specifically.
Python is part of a cycle (pip <--> python), which can cause it to be
installed *after* packages that need python (possibly in
post-install.sh).
A special case in toposort() breaks the cycle, to ensure that python
isn't installed too late. Here, we verify that it works.
"""
# This is the actual dependency graph for python (as of the time of this writing, anyway)
data = {
'python':
['pip', 'openssl', 'readline', 'sqlite', 'tk', 'xz', 'zlib'],
'pip': ['python', 'setuptools', 'wheel'],
'setuptools': ['python'],
'wheel': ['python'],
'openssl': [],
'readline': [],
'sqlite': [],
'tk': [],
'xz': [],
'zlib': []
}
# Here are some extra pure-python libs, just for good measure.
data.update({
'psutil': ['python'],
'greenlet': ['python'],
'futures': ['python'],
'six': ['python']
})
results = toposort(data)
# Python always comes before things that need it!
self.assertLess(results.index('python'), results.index('setuptools'))
self.assertLess(results.index('python'), results.index('wheel'))
self.assertLess(results.index('python'), results.index('pip'))
self.assertLess(results.index('python'), results.index('psutil'))
self.assertLess(results.index('python'), results.index('greenlet'))
self.assertLess(results.index('python'), results.index('futures'))
self.assertLess(results.index('python'), results.index('six'))
def graph_sort(self, must_have):
def lookup(value):
index_data = self.index.get("%s.tar.bz2" % value, {})
return {item.split(" ", 1)[0] for item in index_data.get("depends", [])}
digraph = {}
for key, value in must_have.items():
depends = lookup(value)
digraph[key] = depends
sorted_keys = toposort(digraph)
must_have = must_have.copy()
# Take all of the items in the sorted keys
# Don't fail if the key does not exist
result = [must_have.pop(key) for key in sorted_keys if key in must_have]
# Take any key that were not sorted
result.extend(must_have.values())
return result
def test_python_is_prioritized(self):
"""
This test checks a special invariant related to 'python' specifically.
Python is part of a cycle (pip <--> python), which can cause it to be
installed *after* packages that need python (possibly in
post-install.sh).
A special case in toposort() breaks the cycle, to ensure that python
isn't installed too late. Here, we verify that it works.
"""
# This is the actual dependency graph for python (as of the time of this writing, anyway)
data = {'python' : ['pip', 'openssl', 'readline', 'sqlite', 'tk', 'xz', 'zlib'],
'pip': ['python', 'setuptools', 'wheel'],
'setuptools' : ['python'],
'wheel' : ['python'],
'openssl' : [],
'readline' : [],
'sqlite' : [],
'tk' : [],
'xz' : [],
'zlib' : []}
# Here are some extra pure-python libs, just for good measure.
data.update({'psutil' : ['python'],
'greenlet' : ['python'],
'futures' : ['python'],
'six' : ['python']})
results = toposort(data)
# Python always comes before things that need it!
self.assertLess(results.index('python'), results.index('setuptools'))
self.assertLess(results.index('python'), results.index('wheel'))
self.assertLess(results.index('python'), results.index('pip'))
self.assertLess(results.index('python'), results.index('psutil'))
self.assertLess(results.index('python'), results.index('greenlet'))
self.assertLess(results.index('python'), results.index('futures'))
self.assertLess(results.index('python'), results.index('six'))
def dependencies_for_env( prefix ):
"""
Return an OrderedDict of all packages in the packages in the
given environment and each one's list of dependencies.
The returned items are in toposort order.
"""
# Get all package strings as 'name-version-build'
installed = conda.install.linked(prefix)
exitcode, packages = list_packages(prefix, installed, regex=None, format='canonical', show_channel_urls=False)
# If present, remove channel prefix (e.g. from 'ilastik::boost=1.55.0=5')
packages = map( lambda p: p.split('::')[-1], packages )
# Replace last two '-' with '='
packages = map(lambda p: p[::-1].replace('-', '=', 2)[::-1], packages)
# Load dependencies into a dict
index = conda.api.get_index()
r = Resolve(index)
deps_dict = {}
for package in packages:
try:
versions = r.get_pkgs(conda.cli.common.arg2spec(package))
except NoPackagesFound:
print("Skipping " + package, file=sys.stderr)
else:
for pkg in sorted(versions):
deps_dict[pkg.name] = []
for dep in pkg.info['depends']:
deps_dict[pkg.name].append(dep.split(' ')[0])
# Convenience: Return dict with keys in topologically sorted order
sorted_keys = toposort( deps_dict )
deps_dict = collections.OrderedDict( map(lambda k: (k, deps_dict[k]), sorted_keys ) )
return deps_dict
def test_simple(self):
data = {'a':'bc', 'b':'c'}
results = toposort(data, safe=True)
self.assertEqual(results, ['c', 'b', 'a'])
results = toposort(data, safe=False)
self.assertEqual(results, ['c', 'b', 'a'])
def test_simple(self):
data = {'a': 'bc', 'b': 'c'}
results = toposort(data, safe=True)
self.assertEqual(results, ['c', 'b', 'a'])
results = toposort(data, safe=False)
self.assertEqual(results, ['c', 'b', 'a'])
