def fetch_index(channel_urls, use_cache=False, unknown=False):
log.debug('channel_urls=' + repr(channel_urls))
# pool = ThreadPool(5)
index = {}
stdoutlog.info("Fetching package metadata ...")
if not isinstance(channel_urls, dict):
channel_urls = {url: pri + 1 for pri, url in enumerate(channel_urls)}
for url in iterkeys(channel_urls):
if config.allowed_channels and url not in config.allowed_channels:
sys.exit("""
Error: URL '%s' not in allowed channels.
Allowed channels are:
- %s
""" % (url, '\n - '.join(config.allowed_channels)))
try:
import concurrent.futures
executor = concurrent.futures.ThreadPoolExecutor(10)
except (ImportError, RuntimeError):
# concurrent.futures is only available in Python >= 3.2 or if futures is installed
# RuntimeError is thrown if number of threads are limited by OS
session = CondaSession()
repodatas = [(url,
fetch_repodata(url, use_cache=use_cache,
session=session))
for url in iterkeys(channel_urls)]
else:
try:
urls = tuple(channel_urls)
futures = tuple(
executor.submit(fetch_repodata,
url,
use_cache=use_cache,
session=CondaSession()) for url in urls)
repodatas = [(u, f.result()) for u, f in zip(urls, futures)]
finally:
executor.shutdown(wait=True)
for channel, repodata in repodatas:
if repodata is None:
continue
new_index = repodata['packages']
url_s, priority = channel_urls[channel]
for fn, info in iteritems(new_index):
info['fn'] = fn
info['schannel'] = url_s
info['channel'] = channel
info['priority'] = priority
info['url'] = channel + fn
key = url_s + '::' + fn if url_s else fn
index[key] = info
stdoutlog.info('\n')
if unknown:
add_unknown(index, channel_urls)
if config.add_pip_as_python_dependency:
add_pip_dependency(index)
return index
def fetch_index(channel_urls, use_cache=False, unknown=False, index=None):
log.debug('channel_urls=' + repr(channel_urls))
# pool = ThreadPool(5)
if index is None:
index = {}
stdoutlog.info("Fetching package metadata ...")
if not isinstance(channel_urls, dict):
channel_urls = {url: pri+1 for pri, url in enumerate(channel_urls)}
for url in iterkeys(channel_urls):
if allowed_channels and url not in allowed_channels:
sys.exit("""
Error: URL '%s' not in allowed channels.
Allowed channels are:
- %s
""" % (url, '\n - '.join(allowed_channels)))
try:
import concurrent.futures
executor = concurrent.futures.ThreadPoolExecutor(10)
except (ImportError, RuntimeError):
# concurrent.futures is only available in Python >= 3.2 or if futures is installed
# RuntimeError is thrown if number of threads are limited by OS
session = CondaSession()
repodatas = [(url, fetch_repodata(url, use_cache=use_cache, session=session))
for url in iterkeys(channel_urls)]
else:
try:
urls = tuple(channel_urls)
futures = tuple(executor.submit(fetch_repodata, url, use_cache=use_cache,
session=CondaSession()) for url in urls)
repodatas = [(u, f.result()) for u, f in zip(urls, futures)]
finally:
executor.shutdown(wait=True)
for channel, repodata in repodatas:
if repodata is None:
continue
new_index = repodata['packages']
url_s, priority = channel_urls[channel]
channel = channel.rstrip('/')
for fn, info in iteritems(new_index):
info['fn'] = fn
info['schannel'] = url_s
info['channel'] = channel
info['priority'] = priority
info['url'] = channel + '/' + fn
key = url_s + '::' + fn if url_s != 'defaults' else fn
index[key] = info
stdoutlog.info('\n')
if unknown:
add_unknown(index, channel_urls)
if add_pip_as_python_dependency:
add_pip_dependency(index)
return index
def fetch_index(channel_urls, use_cache=False, unknown=False):
log.debug('channel_urls=' + repr(channel_urls))
# pool = ThreadPool(5)
index = {}
stdoutlog.info("Fetching package metadata ...")
session = CondaSession()
if not isinstance(channel_urls, dict):
channel_urls = {url: pri+1 for pri, url in enumerate(channel_urls)}
for url in iterkeys(channel_urls):
if config.allowed_channels and url not in config.allowed_channels:
sys.exit("""
Error: URL '%s' not in allowed channels.
Allowed channels are:
- %s
""" % (url, '\n - '.join(config.allowed_channels)))
try:
import concurrent.futures
from collections import OrderedDict
repodatas = []
with concurrent.futures.ThreadPoolExecutor(10) as executor:
future_to_url = OrderedDict([(executor.submit(
fetch_repodata, url, use_cache=use_cache,
session=session), url)
for url in iterkeys(channel_urls)])
for future in future_to_url:
url = future_to_url[future]
repodatas.append((url, future.result()))
except ImportError:
# concurrent.futures is only available in Python 3
repodatas = map(lambda url: (url, fetch_repodata(url,
use_cache=use_cache, session=session)),
iterkeys(channel_urls))
for channel, repodata in repodatas:
if repodata is None:
continue
new_index = repodata['packages']
url_s, priority = channel_urls[channel]
for fn, info in iteritems(new_index):
info['fn'] = fn
info['schannel'] = url_s
info['channel'] = channel
info['priority'] = priority
info['url'] = channel + fn
key = url_s + '::' + fn if url_s else fn
index[key] = info
stdoutlog.info('\n')
if unknown:
add_unknown(index, channel_urls)
if config.add_pip_as_python_dependency:
add_pip_dependency(index)
return index
def __init__(self, index, sort=False, processed=False):
self.index = index
if not processed:
for fkey, info in iteritems(index.copy()):
if fkey.endswith(']'):
continue
for fstr in chain(info.get('features', '').split(),
info.get('track_features', '').split(),
track_features or ()):
self.add_feature(fstr, group=False)
for fstr in iterkeys(info.get('with_features_depends', {})):
index['%s[%s]' % (fkey, fstr)] = info
self.add_feature(fstr, group=False)
groups = {}
trackers = {}
installed = set()
for fkey, info in iteritems(index):
groups.setdefault(info['name'], []).append(fkey)
for feat in info.get('track_features', '').split():
trackers.setdefault(feat, []).append(fkey)
if 'link' in info and not fkey.endswith(']'):
installed.add(fkey)
self.groups = groups
self.installed = installed
self.trackers = trackers
self.find_matches_ = {}
self.ms_depends_ = {}
if sort:
for name, group in iteritems(groups):
groups[name] = sorted(group, key=self.version_key, reverse=True)
def __init__(self, index, sort=False, processed=False):
if not processed:
for fkey, info in iteritems(index.copy()):
for fstr in chain(info.get('features', '').split(),
info.get('track_features', '').split()):
fpkg = fstr + '@'
if fpkg not in index:
index[fpkg] = {
'name': fpkg, 'channel': '@', 'priority': 0,
'version': '0', 'build_number': 0,
'build': '', 'depends': [], 'track_features': fstr}
for fstr in iterkeys(info.get('with_features_depends', {})):
index['%s[%s]' % (fkey, fstr)] = info
groups = {}
trackers = {}
installed = set()
for fkey, info in iteritems(index):
groups.setdefault(info['name'], []).append(fkey)
for feat in info.get('track_features', '').split():
trackers.setdefault(feat, []).append(fkey)
if 'link' in info:
installed.add(fkey)
self.index = index
self.groups = groups
self.installed = installed
self.trackers = trackers
self.find_matches_ = {}
self.ms_depends_ = {}
if sort:
for name, group in iteritems(groups):
groups[name] = sorted(group, key=self.version_key, reverse=True)
def __init__(self, index, sort=False, processed=False):
if not processed:
for fkey, info in iteritems(index.copy()):
for fstr in chain(info.get('features', '').split(),
info.get('track_features', '').split()):
fpkg = fstr + '@'
if fpkg not in index:
index[fpkg] = {
'name': fpkg, 'channel': '@', 'priority': 0,
'version': '0', 'build_number': 0,
'build': '', 'depends': [], 'track_features': fstr}
for fstr in iterkeys(info.get('with_features_depends', {})):
index['%s[%s]' % (fkey, fstr)] = info
groups = {}
trackers = {}
installed = set()
for fkey, info in iteritems(index):
groups.setdefault(info['name'], []).append(fkey)
for feat in info.get('track_features', '').split():
trackers.setdefault(feat, []).append(fkey)
if 'link' in info:
installed.add(fkey)
self.index = index
self.groups = groups
self.installed = installed
self.trackers = trackers
self.find_matches_ = {}
self.ms_depends_ = {}
if sort:
for name, group in iteritems(groups):
groups[name] = sorted(group, key=self.version_key, reverse=True)
def __init__(self, index, sort=False, processed=False):
if not processed:
for fkey, info in iteritems(index.copy()):
for fstr in iterkeys(info.get('with_features_depends', {})):
index['%s[%s]' % (fkey, fstr)] = info
groups = {}
trackers = {}
installed = set()
for fkey, info in iteritems(index):
groups.setdefault(info['name'], []).append(fkey)
for feat in info.get('track_features', '').split():
trackers.setdefault(feat, []).append(fkey)
if 'link' in info:
installed.add(fkey)
self.index = index
self.groups = groups
self.installed = installed
self.trackers = trackers
self.find_matches_ = {}
self.ms_depends_ = {}
if sort:
for name, group in iteritems(groups):
groups[name] = sorted(group, key=self.version_key, reverse=True)
def gen_clauses(self, groups, trackers, specs):
C = Clauses()
def push_MatchSpec(ms):
name = self.ms_to_v(ms)
m = C.from_name(name)
if m is None:
libs = [fn for fn in self.find_matches_group(ms, groups, trackers)]
m = C.Any(libs, polarity=None if ms.optional else True, name=name)
return m
# Create package variables
for group in itervalues(groups):
for fn in group:
C.new_var(fn)
# Create spec variables
for ms in specs:
push_MatchSpec(ms)
# Create feature variables
for name in iterkeys(trackers):
push_MatchSpec(MatchSpec('@' + name))
# Add dependency relationships
for group in itervalues(groups):
C.Require(C.AtMostOne_NSQ, group)
for fn in group:
for ms in self.ms_depends(fn):
if not ms.optional:
C.Require(C.Or, C.Not(fn), push_MatchSpec(ms))
return C
def test_normalize_urls(self):
current_platform = config.subdir
assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/'
assert config.rc.get('channel_alias') == 'https://your.repo/'
for channel in iterkeys(config.normalize_urls(['defaults', 'system',
'https://anaconda.org/username', 'file:///Users/username/repo',
'username'])):
assert (channel.endswith('/%s/' % current_platform) or
channel.endswith('/noarch/'))
self.assertEqual(config.normalize_urls([
'defaults', 'system', 'https://conda.anaconda.org/username',
'file:///Users/username/repo', 'username'
], 'osx-64'),
{'file:///Users/username/repo/noarch/': ('file:///Users/username/repo', 6),
'file:///Users/username/repo/osx-64/': ('file:///Users/username/repo', 6),
'http://repo.continuum.io/pkgs/free/noarch/': (None, 1),
'http://repo.continuum.io/pkgs/free/osx-64/': (None, 1),
'http://repo.continuum.io/pkgs/pro/noarch/': (None, 1),
'http://repo.continuum.io/pkgs/pro/osx-64/': (None, 1),
'http://some.custom/channel/noarch/': ('http://some.custom/channel', 3),
'http://some.custom/channel/osx-64/': ('http://some.custom/channel', 3),
'https://conda.anaconda.org/username/noarch/': ('https://conda.anaconda.org/username', 5),
'https://conda.anaconda.org/username/osx-64/': ('https://conda.anaconda.org/username', 5),
'https://your.repo/binstar_username/noarch/': ('binstar_username', 2),
'https://your.repo/binstar_username/osx-64/': ('binstar_username', 2),
'https://your.repo/username/noarch/': ('username', 7),
'https://your.repo/username/osx-64/': ('username', 7)})
def default_filter(self, features=None, filter=None):
if filter is None:
filter = {}
else:
filter.clear()
filter.update({fstr+'@': False for fstr in iterkeys(self.trackers)})
if features:
filter.update({fstr+'@': True for fstr in features})
return filter
def default_filter(self, features=None, filter=None):
if filter is None:
filter = {}
else:
filter.clear()
filter.update({fstr+'@': False for fstr in iterkeys(self.trackers)})
if features:
filter.update({fstr+'@': True for fstr in features})
return filter
def __init__(self, index):
self.index = index.copy()
self.feats = set()
for fn, info in iteritems(index):
for fstr in info.get('track_features','').split():
self.add_feature(fstr, False)
for fstr in iterkeys(info.get('with_features_depends',{})):
fn2 = fn + '[' + fstr + ']'
self.index[fn2] = info
self.groups = build_groups(self.index)
self.find_matches_ = {}
self.ms_depends_ = {}
def __init__(self, index):
self.index = index.copy()
self.feats = set()
for fn, info in iteritems(index):
for fstr in info.get('track_features','').split():
self.add_feature(fstr, False)
for fstr in iterkeys(info.get('with_features_depends',{})):
fn2 = fn + '[' + fstr + ']'
self.index[fn2] = info
self.groups = build_groups(self.index)
self.find_matches_ = {}
self.ms_depends_ = {}
def gen_clauses(self, groups, trackers, specs):
C = Clauses()
# Creates a variable that represents the proposition:
# Does the package set include a package that matches MatchSpec "ms"?
def push_MatchSpec(ms):
name = self.ms_to_v(ms)
m = C.from_name(name)
if m is None:
libs = [
fn for fn in self.find_matches_group(ms, groups, trackers)
]
# If the MatchSpec is optional, then there may be cases where we want
# to assert that it is *not* True. This requires polarity=None.
m = C.Any(libs,
polarity=None if ms.optional else True,
name=name)
return m
# Creates a variable that represents the proposition:
# Does the package set include package "fn"?
for group in itervalues(groups):
for fn in group:
C.new_var(fn)
# Install no more than one version of each package
C.Require(C.AtMostOne, group)
# Create a variable that represents the proposition:
# Is the feature "name" active in this package set?
# We mark this as "optional" below because sometimes we need to be able to
# assert the proposition is False during the feature minimization pass.
for name in iterkeys(trackers):
ms = MatchSpec('@' + name)
ms.optional = True
push_MatchSpec(ms)
# Create a variable that represents the proposition:
# Is the MatchSpec "ms" satisfied by the current package set?
for ms in specs:
push_MatchSpec(ms)
# Create propositions that assert:
# If package "fn" is installed, its dependencie must be satisfied
for group in itervalues(groups):
for fn in group:
for ms in self.ms_depends(fn):
if not ms.optional:
C.Require(C.Or, C.Not(fn), push_MatchSpec(ms))
return C
def __init__(self, index):
self.index = index.copy()
for fn, info in iteritems(index):
for fstr in chain(info.get('features', '').split(),
info.get('track_features', '').split()):
fpkg = fstr + '@'
if fpkg not in self.index:
self.index[fpkg] = {
'name': fpkg, 'version': '0', 'build_number': 0,
'build': '', 'depends': [], 'track_features': fstr}
for fstr in iterkeys(info.get('with_features_depends', {})):
fn2 = fn + '[' + fstr + ']'
self.index[fn2] = info
self.groups, self.trackers = build_groups(self.index)
self.find_matches_ = {}
self.ms_depends_ = {}
def gen_clauses(self, groups, trackers, specs):
C = Clauses()
# Creates a variable that represents the proposition:
# Does the package set include a package that matches MatchSpec "ms"?
def push_MatchSpec(ms):
name = self.ms_to_v(ms)
m = C.from_name(name)
if m is None:
libs = [fn for fn in self.find_matches_group(ms, groups, trackers)]
# If the MatchSpec is optional, then there may be cases where we want
# to assert that it is *not* True. This requires polarity=None.
m = C.Any(libs, polarity=None if ms.optional else True, name=name)
return m
# Creates a variable that represents the proposition:
# Does the package set include package "fn"?
for group in itervalues(groups):
for fn in group:
C.new_var(fn)
# Install no more than one version of each package
C.Require(C.AtMostOne, group)
# Create a variable that represents the proposition:
# Is the feature "name" active in this package set?
# We mark this as "optional" below because sometimes we need to be able to
# assert the proposition is False during the feature minimization pass.
for name in iterkeys(trackers):
ms = MatchSpec('@' + name)
ms.optional = True
push_MatchSpec(ms)
# Create a variable that represents the proposition:
# Is the MatchSpec "ms" satisfied by the current package set?
for ms in specs:
push_MatchSpec(ms)
# Create propositions that assert:
# If package "fn" is installed, its dependencie must be satisfied
for group in itervalues(groups):
for fn in group:
for ms in self.ms_depends(fn):
if not ms.optional:
C.Require(C.Or, C.Not(fn), push_MatchSpec(ms))
return C
def __init__(self, bad_deps, chains=True):
bad_deps = [list(map(lambda x: x.spec, dep)) for dep in bad_deps]
if chains:
chains = {}
for dep in sorted(bad_deps, key=len, reverse=True):
dep1 = [str(MatchSpec(s)).partition(' ') for s in dep[1:]]
key = (dep[0], ) + tuple(v[0] for v in dep1)
vals = ('', ) + tuple(v[2] for v in dep1)
found = False
for key2, csets in iteritems(chains):
if key2[:len(key)] == key:
for cset, val in zip(csets, vals):
cset.add(val)
found = True
if not found:
chains[key] = [{val} for val in vals]
bad_deps = []
for key, csets in iteritems(chains):
deps = []
for name, cset in zip(key, csets):
if '' not in cset:
pass
elif len(cset) == 1:
cset.clear()
else:
cset.remove('')
cset.add('*')
if name[0] == '@':
name = 'feature:' + name[1:]
deps.append('%s %s' %
(name,
'|'.join(sorted(cset))) if cset else name)
chains[key] = ' -> '.join(deps)
bad_deps = [chains[key] for key in sorted(iterkeys(chains))]
msg = '''The following specifications were found to be in conflict:%s
Use "conda info <package>" to see the dependencies for each package.'''
else:
bad_deps = [sorted(dep) for dep in bad_deps]
bad_deps = [', '.join(dep) for dep in sorted(bad_deps)]
msg = '''The following specifications were found to be incompatible with the
others, or with the existing package set:%s
Use "conda info <package>" to see the dependencies for each package.'''
msg = msg % dashlist(bad_deps)
super(Unsatisfiable, self).__init__(msg)
def __init__(self, bad_deps, chains=True):
bad_deps = [list(map(lambda x: x.spec, dep)) for dep in bad_deps]
if chains:
chains = {}
for dep in sorted(bad_deps, key=len, reverse=True):
dep1 = [str(MatchSpec(s)).partition(' ') for s in dep[1:]]
key = (dep[0],) + tuple(v[0] for v in dep1)
vals = ('',) + tuple(v[2] for v in dep1)
found = False
for key2, csets in iteritems(chains):
if key2[:len(key)] == key:
for cset, val in zip(csets, vals):
cset.add(val)
found = True
if not found:
chains[key] = [{val} for val in vals]
bad_deps = []
for key, csets in iteritems(chains):
deps = []
for name, cset in zip(key, csets):
if '' not in cset:
pass
elif len(cset) == 1:
cset.clear()
else:
cset.remove('')
cset.add('*')
if name[0] == '@':
name = 'feature:' + name[1:]
deps.append('%s %s' % (name, '|'.join(sorted(cset))) if cset else name)
chains[key] = ' -> '.join(deps)
bad_deps = [chains[key] for key in sorted(iterkeys(chains))]
msg = '''The following specifications were found to be in conflict:%s
Use "conda info <package>" to see the dependencies for each package.'''
else:
bad_deps = [sorted(dep) for dep in bad_deps]
bad_deps = [', '.join(dep) for dep in sorted(bad_deps)]
msg = '''The following specifications were found to be incompatible with the
others, or with the existing package set:%s
Use "conda info <package>" to see the dependencies for each package.'''
msg = msg % dashlist(bad_deps)
super(Unsatisfiable, self).__init__(msg)
def test_normalize_urls(self):
current_platform = config.subdir
assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/'
assert config.rc.get('channel_alias') == 'https://your.repo/'
for channel in iterkeys(
config.normalize_urls([
'defaults', 'system', 'https://anaconda.org/username',
'file:///Users/username/repo', 'username'
])):
assert (channel.endswith('/%s/' % current_platform)
or channel.endswith('/noarch/'))
self.assertEqual(
config.normalize_urls([
'defaults', 'system', 'https://conda.anaconda.org/username',
'file:///Users/username/repo', 'username'
], 'osx-64'), {
'file:///Users/username/repo/noarch/':
('file:///Users/username/repo', 6),
'file:///Users/username/repo/osx-64/':
('file:///Users/username/repo', 6),
'http://repo.continuum.io/pkgs/free/noarch/': (None, 1),
'http://repo.continuum.io/pkgs/free/osx-64/': (None, 1),
'http://repo.continuum.io/pkgs/pro/noarch/': (None, 1),
'http://repo.continuum.io/pkgs/pro/osx-64/': (None, 1),
'http://some.custom/channel/noarch/':
('http://some.custom/channel', 3),
'http://some.custom/channel/osx-64/':
('http://some.custom/channel', 3),
'https://conda.anaconda.org/username/noarch/':
('https://conda.anaconda.org/username', 5),
'https://conda.anaconda.org/username/osx-64/':
('https://conda.anaconda.org/username', 5),
'https://your.repo/binstar_username/noarch/':
('binstar_username', 2),
'https://your.repo/binstar_username/osx-64/':
('binstar_username', 2),
'https://your.repo/username/noarch/': ('username', 7),
'https://your.repo/username/osx-64/': ('username', 7)
})
def __init__(self, index):
self.index = index.copy()
for fn, info in iteritems(index):
for fstr in chain(
info.get('features', '').split(),
info.get('track_features', '').split()):
fpkg = fstr + '@'
if fpkg not in self.index:
self.index[fpkg] = {
'name': fpkg,
'version': '0',
'build_number': 0,
'build': '',
'depends': [],
'track_features': fstr
}
for fstr in iterkeys(info.get('with_features_depends', {})):
fn2 = fn + '[' + fstr + ']'
self.index[fn2] = info
self.groups, self.trackers = build_groups(self.index)
self.find_matches_ = {}
self.ms_depends_ = {}
def gen_clauses(self, groups, trackers, specs):
C = Clauses()
polarities = {}
def push_MatchSpec(ms, polarity=None):
name = self.ms_to_v(ms)
m = C.from_name(name)
if m is None:
m = C.Any(self.find_matches_group(ms, groups, trackers),
polarity=polarity,
name=name)
return m
# Create package variables
for group in itervalues(groups):
for fn in group:
C.new_var(fn)
# Create spec variables
for ms in specs:
push_MatchSpec(ms, polarity=None if ms.optional else True)
# Create feature variables
for name in iterkeys(trackers):
push_MatchSpec(MatchSpec('@' + name), polarity=True)
# Add dependency relationships
for group in itervalues(groups):
C.Require(C.AtMostOne_NSQ, group)
for fn in group:
for ms in self.ms_depends(fn):
if not ms.optional:
C.Require(C.Or, C.Not(fn),
push_MatchSpec(ms, polarity=True))
return C
def generate_feature_count(self, C):
return {self.ms_to_v('@' + name): 1 for name in iterkeys(self.trackers)}
def generate_feature_count(self, C):
return {self.push_MatchSpec(C, '@' + name): 1 for name in iterkeys(self.trackers)}
def generate_feature_count(self, C, trackers):
return {self.feat_to_v(name): 1 for name in iterkeys(trackers)}
def generate_feature_count(self, C):
return {self.ms_to_v('@' + name): 1 for name in iterkeys(self.trackers)}
def generate_feature_eq(self, v, groups, specs):
feats = {s.name for s in specs if s.name[-1] == '@' and not s.optional}
return [(1,v[name]) for name in iterkeys(groups) if name[-1] == '@' and name not in feats], len(feats)
def generate_feature_eq(self, v, groups, specs):
feats = {s.name for s in specs if s.name[-1] == '@' and not s.optional}
return [(1,v[name]) for name in iterkeys(groups) if name[-1] == '@' and name not in feats], len(feats)
def linked(prefix):
"""
Return the set of canonical names of linked packages in prefix.
"""
return set(iterkeys(linked_data(prefix)))
def generate_feature_count(self, C, trackers):
return {self.feat_to_v(name): 1 for name in iterkeys(trackers)}
