def get_index(channel_urls=(), prepend=True, platform=None,
use_local=False, use_cache=False, unknown=False,
offline=False, prefix=None):
"""
Return the index of packages available on the channels
If prepend=False, only the channels passed in as arguments are used.
If platform=None, then the current platform is used.
If prefix is supplied, then the packages installed in that prefix are added.
"""
if use_local:
channel_urls = ['local'] + list(channel_urls)
channel_urls = normalize_urls(channel_urls, platform, offline)
if prepend:
channel_urls.extend(get_channel_urls(platform, offline))
channel_urls = prioritize_channels(channel_urls)
index = fetch_index(channel_urls, use_cache=use_cache, unknown=unknown)
if prefix:
priorities = {c: p for c, p in itervalues(channel_urls)}
for dist, info in iteritems(install.linked_data(prefix)):
fn = info['fn']
schannel = info['schannel']
prefix = '' if schannel == 'defaults' else schannel + '::'
priority = priorities.get(schannel, 0)
key = prefix + fn
if key in index:
# Copy the link information so the resolver knows this is installed
index[key]['link'] = info.get('link')
else:
# only if the package in not in the repodata, use local
# conda-meta (with 'depends' defaulting to [])
info.setdefault('depends', [])
info['priority'] = priority
index[key] = info
return index
def fetch_index(channel_urls, use_cache=False, unknown=False, index=None):
log.debug('channel_urls=' + repr(channel_urls))
# pool = ThreadPool(5)
if index is None:
index = {}
stdoutlog.info("Fetching package metadata ...")
if not isinstance(channel_urls, dict):
channel_urls = prioritize_channels(channel_urls)
for url in iterkeys(channel_urls):
if allowed_channels and url not in allowed_channels:
sys.exit("""
Error: URL '%s' not in allowed channels.
Allowed channels are:
- %s
""" % (url, '\n - '.join(allowed_channels)))
try:
import concurrent.futures
executor = concurrent.futures.ThreadPoolExecutor(10)
except (ImportError, RuntimeError):
# concurrent.futures is only available in Python >= 3.2 or if futures is installed
# RuntimeError is thrown if number of threads are limited by OS
session = CondaSession()
repodatas = [(url, fetch_repodata(url, use_cache=use_cache, session=session))
for url in iterkeys(channel_urls)]
else:
try:
urls = tuple(channel_urls)
futures = tuple(executor.submit(fetch_repodata, url, use_cache=use_cache,
session=CondaSession()) for url in urls)
repodatas = [(u, f.result()) for u, f in zip(urls, futures)]
finally:
executor.shutdown(wait=True)
for channel, repodata in repodatas:
if repodata is None:
continue
new_index = repodata['packages']
url_s, priority = channel_urls[channel]
channel = channel.rstrip('/')
for fn, info in iteritems(new_index):
info['fn'] = fn
info['schannel'] = url_s
info['channel'] = channel
info['priority'] = priority
info['url'] = channel + '/' + fn
key = url_s + '::' + fn if url_s != 'defaults' else fn
index[key] = info
stdoutlog.info('\n')
if unknown:
add_unknown(index, channel_urls)
if add_pip_as_python_dependency:
add_pip_dependency(index)
return index
def test_normalize_urls(self):
current_platform = config.subdir
assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/'
assert config.rc.get('channel_alias') == 'https://your.repo/'
assert config.channel_alias == 'https://your.repo/'
normurls = config.normalize_urls([
'defaults', 'system', 'https://conda.anaconda.org/username',
'file:///Users/username/repo', 'username'
], 'osx-64')
assert normurls == [
'http://repo.continuum.io/pkgs/free/osx-64/',
'http://repo.continuum.io/pkgs/free/noarch/',
'http://repo.continuum.io/pkgs/pro/osx-64/',
'http://repo.continuum.io/pkgs/pro/noarch/',
'https://your.repo/binstar_username/osx-64/',
'https://your.repo/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
'http://repo.continuum.io/pkgs/free/osx-64/',
'http://repo.continuum.io/pkgs/free/noarch/',
'http://repo.continuum.io/pkgs/pro/osx-64/',
'http://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
'https://your.repo/username/osx-64/',
'https://your.repo/username/noarch/'
]
priurls = config.prioritize_channels(normurls)
assert dict(priurls) == {
'file:///Users/username/repo/noarch/':
('file:///Users/username/repo', 5),
'file:///Users/username/repo/osx-64/':
('file:///Users/username/repo', 5),
'http://repo.continuum.io/pkgs/free/noarch/': ('defaults', 1),
'http://repo.continuum.io/pkgs/free/osx-64/': ('defaults', 1),
'http://repo.continuum.io/pkgs/pro/noarch/': ('defaults', 1),
'http://repo.continuum.io/pkgs/pro/osx-64/': ('defaults', 1),
'http://some.custom/channel/noarch/':
('http://some.custom/channel', 3),
'http://some.custom/channel/osx-64/':
('http://some.custom/channel', 3),
'https://conda.anaconda.org/username/noarch/':
('https://conda.anaconda.org/username', 4),
'https://conda.anaconda.org/username/osx-64/':
('https://conda.anaconda.org/username', 4),
'https://your.repo/binstar_username/noarch/': ('binstar_username',
2),
'https://your.repo/binstar_username/osx-64/': ('binstar_username',
2),
'https://your.repo/username/noarch/': ('username', 6),
'https://your.repo/username/osx-64/': ('username', 6)
}
def test_normalize_urls(self):
current_platform = config.subdir
assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/'
assert config.rc.get('channel_alias') == 'https://your.repo/'
assert config.channel_alias == 'https://your.repo/'
normurls = config.normalize_urls([
'defaults', 'system', 'https://conda.anaconda.org/username',
'file:///Users/username/repo', 'username'
], 'osx-64')
assert normurls == [
'http://repo.continuum.io/pkgs/free/osx-64/',
'http://repo.continuum.io/pkgs/free/noarch/',
'http://repo.continuum.io/pkgs/pro/osx-64/',
'http://repo.continuum.io/pkgs/pro/noarch/',
'https://your.repo/binstar_username/osx-64/',
'https://your.repo/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
'http://repo.continuum.io/pkgs/free/osx-64/',
'http://repo.continuum.io/pkgs/free/noarch/',
'http://repo.continuum.io/pkgs/pro/osx-64/',
'http://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
'https://your.repo/username/osx-64/',
'https://your.repo/username/noarch/']
priurls = config.prioritize_channels(normurls)
assert dict(priurls) == {
'file:///Users/username/repo/noarch/': ('file:///Users/username/repo', 5),
'file:///Users/username/repo/osx-64/': ('file:///Users/username/repo', 5),
'http://repo.continuum.io/pkgs/free/noarch/': ('defaults', 1),
'http://repo.continuum.io/pkgs/free/osx-64/': ('defaults', 1),
'http://repo.continuum.io/pkgs/pro/noarch/': ('defaults', 1),
'http://repo.continuum.io/pkgs/pro/osx-64/': ('defaults', 1),
'http://some.custom/channel/noarch/': ('http://some.custom/channel', 3),
'http://some.custom/channel/osx-64/': ('http://some.custom/channel', 3),
'https://conda.anaconda.org/username/noarch/': ('https://conda.anaconda.org/username', 4),
'https://conda.anaconda.org/username/osx-64/': ('https://conda.anaconda.org/username', 4),
'https://your.repo/binstar_username/noarch/': ('binstar_username', 2),
'https://your.repo/binstar_username/osx-64/': ('binstar_username', 2),
'https://your.repo/username/noarch/': ('username', 6),
'https://your.repo/username/osx-64/': ('username', 6)}
def get_index(channel_urls=(),
prepend=True,
platform=None,
use_local=False,
use_cache=False,
unknown=False,
offline=False,
prefix=None):
"""
Return the index of packages available on the channels
If prepend=False, only the channels passed in as arguments are used.
If platform=None, then the current platform is used.
If prefix is supplied, then the packages installed in that prefix are added.
"""
if use_local:
channel_urls = ['local'] + list(channel_urls)
channel_urls = normalize_urls(channel_urls, platform, offline)
if prepend:
channel_urls.extend(get_channel_urls(platform, offline))
channel_urls = prioritize_channels(channel_urls)
index = fetch_index(channel_urls, use_cache=use_cache, unknown=unknown)
if prefix:
priorities = {c: p for c, p in itervalues(channel_urls)}
maxp = max(itervalues(priorities)) + 1 if priorities else 1
for dist, info in iteritems(install.linked_data(prefix)):
fn = info['fn']
schannel = info['schannel']
prefix = '' if schannel == 'defaults' else schannel + '::'
priority = priorities.get(schannel, maxp)
key = prefix + fn
if key in index:
# Copy the link information so the resolver knows this is installed
index[key]['link'] = info.get('link')
else:
# only if the package in not in the repodata, use local
# conda-meta (with 'depends' defaulting to [])
info.setdefault('depends', [])
info['priority'] = priority
index[key] = info
return index
def test_normalize_urls(self):
current_platform = config.subdir
assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/'
assert config.rc.get('channel_alias') == 'https://your.repo/'
assert config.channel_prefix(False) == 'https://your.repo/'
assert config.binstar_domain == 'https://mybinstar.com/'
assert config.binstar_domain_tok == 'https://mybinstar.com/t/01234abcde/'
channel_urls = [
'defaults', 'system',
'https://conda.anaconda.org/username',
'file:///Users/username/repo',
'https://mybinstar.com/t/5768wxyz/test2',
'https://mybinstar.com/test',
'https://conda.anaconda.org/t/abcdefgh/username',
'username'
]
platform = 'osx-64'
normurls = config.normalize_urls(channel_urls, platform)
assert normurls == [
# defaults
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
# system (condarc)
'https://your.repo/binstar_username/osx-64/',
'https://your.repo/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
# defaults is repeated in condarc; that's OK
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
# conda.anaconda.org is not our default binstar clinet
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
# mybinstar.com is not channel_alias, but we still add tokens
'https://mybinstar.com/t/5768wxyz/test2/osx-64/',
'https://mybinstar.com/t/5768wxyz/test2/noarch/',
# token already supplied, do not change/remove it
'https://mybinstar.com/t/01234abcde/test/osx-64/',
'https://mybinstar.com/t/01234abcde/test/noarch/',
# we do not remove tokens from conda.anaconda.org
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/',
'https://conda.anaconda.org/t/abcdefgh/username/noarch/',
# short channel; add channel_alias
'https://your.repo/username/osx-64/',
'https://your.repo/username/noarch/']
priurls = config.prioritize_channels(normurls)
assert dict(priurls) == {
# defaults appears twice, keep higher priority
'https://repo.continuum.io/pkgs/free/noarch/': ('defaults', 1),
'https://repo.continuum.io/pkgs/free/osx-64/': ('defaults', 1),
'https://repo.continuum.io/pkgs/pro/noarch/': ('defaults', 1),
'https://repo.continuum.io/pkgs/pro/osx-64/': ('defaults', 1),
'https://your.repo/binstar_username/noarch/': ('binstar_username', 2),
'https://your.repo/binstar_username/osx-64/': ('binstar_username', 2),
'http://some.custom/channel/noarch/': ('http://some.custom/channel', 3),
'http://some.custom/channel/osx-64/': ('http://some.custom/channel', 3),
'https://conda.anaconda.org/t/abcdefgh/username/noarch/': ('https://conda.anaconda.org/username', 4),
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/': ('https://conda.anaconda.org/username', 4),
'file:///Users/username/repo/noarch/': ('file:///Users/username/repo', 5),
'file:///Users/username/repo/osx-64/': ('file:///Users/username/repo', 5),
# the tokenized version came first, but we still give it the same priority
'https://conda.anaconda.org/username/noarch/': ('https://conda.anaconda.org/username', 4),
'https://conda.anaconda.org/username/osx-64/': ('https://conda.anaconda.org/username', 4),
'https://mybinstar.com/t/5768wxyz/test2/noarch/': ('https://mybinstar.com/test2', 6),
'https://mybinstar.com/t/5768wxyz/test2/osx-64/': ('https://mybinstar.com/test2', 6),
'https://mybinstar.com/t/01234abcde/test/noarch/': ('https://mybinstar.com/test', 7),
'https://mybinstar.com/t/01234abcde/test/osx-64/': ('https://mybinstar.com/test', 7),
'https://your.repo/username/noarch/': ('username', 8),
'https://your.repo/username/osx-64/': ('username', 8)
}
# Delete the channel alias so now the short channels point to binstar
del config.rc['channel_alias']
config.rc['offline'] = False
config.load_condarc()
config.binstar_client = BinstarTester()
normurls = config.normalize_urls(channel_urls, platform)
# all your.repo references should be changed to mybinstar.com
assert normurls == [
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://mybinstar.com/t/01234abcde/binstar_username/osx-64/',
'https://mybinstar.com/t/01234abcde/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
'https://mybinstar.com/t/5768wxyz/test2/osx-64/',
'https://mybinstar.com/t/5768wxyz/test2/noarch/',
'https://mybinstar.com/t/01234abcde/test/osx-64/',
'https://mybinstar.com/t/01234abcde/test/noarch/',
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/',
'https://conda.anaconda.org/t/abcdefgh/username/noarch/',
'https://mybinstar.com/t/01234abcde/username/osx-64/',
'https://mybinstar.com/t/01234abcde/username/noarch/'
]
# Delete the anaconda token
config.load_condarc()
config.binstar_client = BinstarTester(token=None)
normurls = config.normalize_urls(channel_urls, platform)
# tokens should not be added (but supplied tokens are kept)
assert normurls == [
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://mybinstar.com/binstar_username/osx-64/',
'https://mybinstar.com/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
'https://mybinstar.com/t/5768wxyz/test2/osx-64/',
'https://mybinstar.com/t/5768wxyz/test2/noarch/',
'https://mybinstar.com/test/osx-64/',
'https://mybinstar.com/test/noarch/',
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/',
'https://conda.anaconda.org/t/abcdefgh/username/noarch/',
'https://mybinstar.com/username/osx-64/',
'https://mybinstar.com/username/noarch/'
]
# Turn off add_anaconda_token
config.rc['add_binstar_token'] = False
config.load_condarc()
config.binstar_client = BinstarTester()
normurls2 = config.normalize_urls(channel_urls, platform)
# tokens should not be added (but supplied tokens are kept)
assert normurls == normurls2
# Disable binstar client altogether
config.load_condarc()
config.binstar_client = ()
normurls = config.normalize_urls(channel_urls, platform)
# should drop back to conda.anaconda.org
assert normurls == [
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/binstar_username/osx-64/',
'https://conda.anaconda.org/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
'https://mybinstar.com/t/5768wxyz/test2/osx-64/',
'https://mybinstar.com/t/5768wxyz/test2/noarch/',
'https://mybinstar.com/test/osx-64/',
'https://mybinstar.com/test/noarch/',
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/',
'https://conda.anaconda.org/t/abcdefgh/username/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/'
]
