def get_index(channel_urls=(), prepend=True, platform=None, use_local=False, use_cache=False, unknown=False, offline=False, prefix=None): """ Return the index of packages available on the channels If prepend=False, only the channels passed in as arguments are used. If platform=None, then the current platform is used. If prefix is supplied, then the packages installed in that prefix are added. """ if use_local: channel_urls = ['local'] + list(channel_urls) channel_urls = normalize_urls(channel_urls, platform, offline) if prepend: channel_urls.extend(get_channel_urls(platform, offline)) channel_urls = prioritize_channels(channel_urls) index = fetch_index(channel_urls, use_cache=use_cache, unknown=unknown) if prefix: priorities = {c: p for c, p in itervalues(channel_urls)} for dist, info in iteritems(install.linked_data(prefix)): fn = info['fn'] schannel = info['schannel'] prefix = '' if schannel == 'defaults' else schannel + '::' priority = priorities.get(schannel, 0) key = prefix + fn if key in index: # Copy the link information so the resolver knows this is installed index[key]['link'] = info.get('link') else: # only if the package in not in the repodata, use local # conda-meta (with 'depends' defaulting to []) info.setdefault('depends', []) info['priority'] = priority index[key] = info return index
def fetch_index(channel_urls, use_cache=False, unknown=False, index=None): log.debug('channel_urls=' + repr(channel_urls)) # pool = ThreadPool(5) if index is None: index = {} stdoutlog.info("Fetching package metadata ...") if not isinstance(channel_urls, dict): channel_urls = prioritize_channels(channel_urls) for url in iterkeys(channel_urls): if allowed_channels and url not in allowed_channels: sys.exit(""" Error: URL '%s' not in allowed channels. Allowed channels are: - %s """ % (url, '\n - '.join(allowed_channels))) try: import concurrent.futures executor = concurrent.futures.ThreadPoolExecutor(10) except (ImportError, RuntimeError): # concurrent.futures is only available in Python >= 3.2 or if futures is installed # RuntimeError is thrown if number of threads are limited by OS session = CondaSession() repodatas = [(url, fetch_repodata(url, use_cache=use_cache, session=session)) for url in iterkeys(channel_urls)] else: try: urls = tuple(channel_urls) futures = tuple(executor.submit(fetch_repodata, url, use_cache=use_cache, session=CondaSession()) for url in urls) repodatas = [(u, f.result()) for u, f in zip(urls, futures)] finally: executor.shutdown(wait=True) for channel, repodata in repodatas: if repodata is None: continue new_index = repodata['packages'] url_s, priority = channel_urls[channel] channel = channel.rstrip('/') for fn, info in iteritems(new_index): info['fn'] = fn info['schannel'] = url_s info['channel'] = channel info['priority'] = priority info['url'] = channel + '/' + fn key = url_s + '::' + fn if url_s != 'defaults' else fn index[key] = info stdoutlog.info('\n') if unknown: add_unknown(index, channel_urls) if add_pip_as_python_dependency: add_pip_dependency(index) return index
def test_normalize_urls(self): current_platform = config.subdir assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/' assert config.rc.get('channel_alias') == 'https://your.repo/' assert config.channel_alias == 'https://your.repo/' normurls = config.normalize_urls([ 'defaults', 'system', 'https://conda.anaconda.org/username', 'file:///Users/username/repo', 'username' ], 'osx-64') assert normurls == [ 'http://repo.continuum.io/pkgs/free/osx-64/', 'http://repo.continuum.io/pkgs/free/noarch/', 'http://repo.continuum.io/pkgs/pro/osx-64/', 'http://repo.continuum.io/pkgs/pro/noarch/', 'https://your.repo/binstar_username/osx-64/', 'https://your.repo/binstar_username/noarch/', 'http://some.custom/channel/osx-64/', 'http://some.custom/channel/noarch/', 'http://repo.continuum.io/pkgs/free/osx-64/', 'http://repo.continuum.io/pkgs/free/noarch/', 'http://repo.continuum.io/pkgs/pro/osx-64/', 'http://repo.continuum.io/pkgs/pro/noarch/', 'https://conda.anaconda.org/username/osx-64/', 'https://conda.anaconda.org/username/noarch/', 'file:///Users/username/repo/osx-64/', 'file:///Users/username/repo/noarch/', 'https://your.repo/username/osx-64/', 'https://your.repo/username/noarch/' ] priurls = config.prioritize_channels(normurls) assert dict(priurls) == { 'file:///Users/username/repo/noarch/': ('file:///Users/username/repo', 5), 'file:///Users/username/repo/osx-64/': ('file:///Users/username/repo', 5), 'http://repo.continuum.io/pkgs/free/noarch/': ('defaults', 1), 'http://repo.continuum.io/pkgs/free/osx-64/': ('defaults', 1), 'http://repo.continuum.io/pkgs/pro/noarch/': ('defaults', 1), 'http://repo.continuum.io/pkgs/pro/osx-64/': ('defaults', 1), 'http://some.custom/channel/noarch/': ('http://some.custom/channel', 3), 'http://some.custom/channel/osx-64/': ('http://some.custom/channel', 3), 'https://conda.anaconda.org/username/noarch/': ('https://conda.anaconda.org/username', 4), 'https://conda.anaconda.org/username/osx-64/': ('https://conda.anaconda.org/username', 4), 'https://your.repo/binstar_username/noarch/': ('binstar_username', 2), 'https://your.repo/binstar_username/osx-64/': ('binstar_username', 2), 'https://your.repo/username/noarch/': ('username', 6), 'https://your.repo/username/osx-64/': ('username', 6) }
def test_normalize_urls(self): current_platform = config.subdir assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/' assert config.rc.get('channel_alias') == 'https://your.repo/' assert config.channel_alias == 'https://your.repo/' normurls = config.normalize_urls([ 'defaults', 'system', 'https://conda.anaconda.org/username', 'file:///Users/username/repo', 'username' ], 'osx-64') assert normurls == [ 'http://repo.continuum.io/pkgs/free/osx-64/', 'http://repo.continuum.io/pkgs/free/noarch/', 'http://repo.continuum.io/pkgs/pro/osx-64/', 'http://repo.continuum.io/pkgs/pro/noarch/', 'https://your.repo/binstar_username/osx-64/', 'https://your.repo/binstar_username/noarch/', 'http://some.custom/channel/osx-64/', 'http://some.custom/channel/noarch/', 'http://repo.continuum.io/pkgs/free/osx-64/', 'http://repo.continuum.io/pkgs/free/noarch/', 'http://repo.continuum.io/pkgs/pro/osx-64/', 'http://repo.continuum.io/pkgs/pro/noarch/', 'https://conda.anaconda.org/username/osx-64/', 'https://conda.anaconda.org/username/noarch/', 'file:///Users/username/repo/osx-64/', 'file:///Users/username/repo/noarch/', 'https://your.repo/username/osx-64/', 'https://your.repo/username/noarch/'] priurls = config.prioritize_channels(normurls) assert dict(priurls) == { 'file:///Users/username/repo/noarch/': ('file:///Users/username/repo', 5), 'file:///Users/username/repo/osx-64/': ('file:///Users/username/repo', 5), 'http://repo.continuum.io/pkgs/free/noarch/': ('defaults', 1), 'http://repo.continuum.io/pkgs/free/osx-64/': ('defaults', 1), 'http://repo.continuum.io/pkgs/pro/noarch/': ('defaults', 1), 'http://repo.continuum.io/pkgs/pro/osx-64/': ('defaults', 1), 'http://some.custom/channel/noarch/': ('http://some.custom/channel', 3), 'http://some.custom/channel/osx-64/': ('http://some.custom/channel', 3), 'https://conda.anaconda.org/username/noarch/': ('https://conda.anaconda.org/username', 4), 'https://conda.anaconda.org/username/osx-64/': ('https://conda.anaconda.org/username', 4), 'https://your.repo/binstar_username/noarch/': ('binstar_username', 2), 'https://your.repo/binstar_username/osx-64/': ('binstar_username', 2), 'https://your.repo/username/noarch/': ('username', 6), 'https://your.repo/username/osx-64/': ('username', 6)}
def get_index(channel_urls=(), prepend=True, platform=None, use_local=False, use_cache=False, unknown=False, offline=False, prefix=None): """ Return the index of packages available on the channels If prepend=False, only the channels passed in as arguments are used. If platform=None, then the current platform is used. If prefix is supplied, then the packages installed in that prefix are added. """ if use_local: channel_urls = ['local'] + list(channel_urls) channel_urls = normalize_urls(channel_urls, platform, offline) if prepend: channel_urls.extend(get_channel_urls(platform, offline)) channel_urls = prioritize_channels(channel_urls) index = fetch_index(channel_urls, use_cache=use_cache, unknown=unknown) if prefix: priorities = {c: p for c, p in itervalues(channel_urls)} maxp = max(itervalues(priorities)) + 1 if priorities else 1 for dist, info in iteritems(install.linked_data(prefix)): fn = info['fn'] schannel = info['schannel'] prefix = '' if schannel == 'defaults' else schannel + '::' priority = priorities.get(schannel, maxp) key = prefix + fn if key in index: # Copy the link information so the resolver knows this is installed index[key]['link'] = info.get('link') else: # only if the package in not in the repodata, use local # conda-meta (with 'depends' defaulting to []) info.setdefault('depends', []) info['priority'] = priority index[key] = info return index
def test_normalize_urls(self): current_platform = config.subdir assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/' assert config.rc.get('channel_alias') == 'https://your.repo/' assert config.channel_prefix(False) == 'https://your.repo/' assert config.binstar_domain == 'https://mybinstar.com/' assert config.binstar_domain_tok == 'https://mybinstar.com/t/01234abcde/' channel_urls = [ 'defaults', 'system', 'https://conda.anaconda.org/username', 'file:///Users/username/repo', 'https://mybinstar.com/t/5768wxyz/test2', 'https://mybinstar.com/test', 'https://conda.anaconda.org/t/abcdefgh/username', 'username' ] platform = 'osx-64' normurls = config.normalize_urls(channel_urls, platform) assert normurls == [ # defaults 'https://repo.continuum.io/pkgs/free/osx-64/', 'https://repo.continuum.io/pkgs/free/noarch/', 'https://repo.continuum.io/pkgs/pro/osx-64/', 'https://repo.continuum.io/pkgs/pro/noarch/', # system (condarc) 'https://your.repo/binstar_username/osx-64/', 'https://your.repo/binstar_username/noarch/', 'http://some.custom/channel/osx-64/', 'http://some.custom/channel/noarch/', # defaults is repeated in condarc; that's OK 'https://repo.continuum.io/pkgs/free/osx-64/', 'https://repo.continuum.io/pkgs/free/noarch/', 'https://repo.continuum.io/pkgs/pro/osx-64/', 'https://repo.continuum.io/pkgs/pro/noarch/', # conda.anaconda.org is not our default binstar clinet 'https://conda.anaconda.org/username/osx-64/', 'https://conda.anaconda.org/username/noarch/', 'file:///Users/username/repo/osx-64/', 'file:///Users/username/repo/noarch/', # mybinstar.com is not channel_alias, but we still add tokens 'https://mybinstar.com/t/5768wxyz/test2/osx-64/', 'https://mybinstar.com/t/5768wxyz/test2/noarch/', # token already supplied, do not change/remove it 'https://mybinstar.com/t/01234abcde/test/osx-64/', 'https://mybinstar.com/t/01234abcde/test/noarch/', # we do not remove tokens from conda.anaconda.org 'https://conda.anaconda.org/t/abcdefgh/username/osx-64/', 'https://conda.anaconda.org/t/abcdefgh/username/noarch/', # short channel; add channel_alias 'https://your.repo/username/osx-64/', 'https://your.repo/username/noarch/'] priurls = config.prioritize_channels(normurls) assert dict(priurls) == { # defaults appears twice, keep higher priority 'https://repo.continuum.io/pkgs/free/noarch/': ('defaults', 1), 'https://repo.continuum.io/pkgs/free/osx-64/': ('defaults', 1), 'https://repo.continuum.io/pkgs/pro/noarch/': ('defaults', 1), 'https://repo.continuum.io/pkgs/pro/osx-64/': ('defaults', 1), 'https://your.repo/binstar_username/noarch/': ('binstar_username', 2), 'https://your.repo/binstar_username/osx-64/': ('binstar_username', 2), 'http://some.custom/channel/noarch/': ('http://some.custom/channel', 3), 'http://some.custom/channel/osx-64/': ('http://some.custom/channel', 3), 'https://conda.anaconda.org/t/abcdefgh/username/noarch/': ('https://conda.anaconda.org/username', 4), 'https://conda.anaconda.org/t/abcdefgh/username/osx-64/': ('https://conda.anaconda.org/username', 4), 'file:///Users/username/repo/noarch/': ('file:///Users/username/repo', 5), 'file:///Users/username/repo/osx-64/': ('file:///Users/username/repo', 5), # the tokenized version came first, but we still give it the same priority 'https://conda.anaconda.org/username/noarch/': ('https://conda.anaconda.org/username', 4), 'https://conda.anaconda.org/username/osx-64/': ('https://conda.anaconda.org/username', 4), 'https://mybinstar.com/t/5768wxyz/test2/noarch/': ('https://mybinstar.com/test2', 6), 'https://mybinstar.com/t/5768wxyz/test2/osx-64/': ('https://mybinstar.com/test2', 6), 'https://mybinstar.com/t/01234abcde/test/noarch/': ('https://mybinstar.com/test', 7), 'https://mybinstar.com/t/01234abcde/test/osx-64/': ('https://mybinstar.com/test', 7), 'https://your.repo/username/noarch/': ('username', 8), 'https://your.repo/username/osx-64/': ('username', 8) } # Delete the channel alias so now the short channels point to binstar del config.rc['channel_alias'] config.rc['offline'] = False config.load_condarc() config.binstar_client = BinstarTester() normurls = config.normalize_urls(channel_urls, platform) # all your.repo references should be changed to mybinstar.com assert normurls == [ 'https://repo.continuum.io/pkgs/free/osx-64/', 'https://repo.continuum.io/pkgs/free/noarch/', 'https://repo.continuum.io/pkgs/pro/osx-64/', 'https://repo.continuum.io/pkgs/pro/noarch/', 'https://mybinstar.com/t/01234abcde/binstar_username/osx-64/', 'https://mybinstar.com/t/01234abcde/binstar_username/noarch/', 'http://some.custom/channel/osx-64/', 'http://some.custom/channel/noarch/', 'https://repo.continuum.io/pkgs/free/osx-64/', 'https://repo.continuum.io/pkgs/free/noarch/', 'https://repo.continuum.io/pkgs/pro/osx-64/', 'https://repo.continuum.io/pkgs/pro/noarch/', 'https://conda.anaconda.org/username/osx-64/', 'https://conda.anaconda.org/username/noarch/', 'file:///Users/username/repo/osx-64/', 'file:///Users/username/repo/noarch/', 'https://mybinstar.com/t/5768wxyz/test2/osx-64/', 'https://mybinstar.com/t/5768wxyz/test2/noarch/', 'https://mybinstar.com/t/01234abcde/test/osx-64/', 'https://mybinstar.com/t/01234abcde/test/noarch/', 'https://conda.anaconda.org/t/abcdefgh/username/osx-64/', 'https://conda.anaconda.org/t/abcdefgh/username/noarch/', 'https://mybinstar.com/t/01234abcde/username/osx-64/', 'https://mybinstar.com/t/01234abcde/username/noarch/' ] # Delete the anaconda token config.load_condarc() config.binstar_client = BinstarTester(token=None) normurls = config.normalize_urls(channel_urls, platform) # tokens should not be added (but supplied tokens are kept) assert normurls == [ 'https://repo.continuum.io/pkgs/free/osx-64/', 'https://repo.continuum.io/pkgs/free/noarch/', 'https://repo.continuum.io/pkgs/pro/osx-64/', 'https://repo.continuum.io/pkgs/pro/noarch/', 'https://mybinstar.com/binstar_username/osx-64/', 'https://mybinstar.com/binstar_username/noarch/', 'http://some.custom/channel/osx-64/', 'http://some.custom/channel/noarch/', 'https://repo.continuum.io/pkgs/free/osx-64/', 'https://repo.continuum.io/pkgs/free/noarch/', 'https://repo.continuum.io/pkgs/pro/osx-64/', 'https://repo.continuum.io/pkgs/pro/noarch/', 'https://conda.anaconda.org/username/osx-64/', 'https://conda.anaconda.org/username/noarch/', 'file:///Users/username/repo/osx-64/', 'file:///Users/username/repo/noarch/', 'https://mybinstar.com/t/5768wxyz/test2/osx-64/', 'https://mybinstar.com/t/5768wxyz/test2/noarch/', 'https://mybinstar.com/test/osx-64/', 'https://mybinstar.com/test/noarch/', 'https://conda.anaconda.org/t/abcdefgh/username/osx-64/', 'https://conda.anaconda.org/t/abcdefgh/username/noarch/', 'https://mybinstar.com/username/osx-64/', 'https://mybinstar.com/username/noarch/' ] # Turn off add_anaconda_token config.rc['add_binstar_token'] = False config.load_condarc() config.binstar_client = BinstarTester() normurls2 = config.normalize_urls(channel_urls, platform) # tokens should not be added (but supplied tokens are kept) assert normurls == normurls2 # Disable binstar client altogether config.load_condarc() config.binstar_client = () normurls = config.normalize_urls(channel_urls, platform) # should drop back to conda.anaconda.org assert normurls == [ 'https://repo.continuum.io/pkgs/free/osx-64/', 'https://repo.continuum.io/pkgs/free/noarch/', 'https://repo.continuum.io/pkgs/pro/osx-64/', 'https://repo.continuum.io/pkgs/pro/noarch/', 'https://conda.anaconda.org/binstar_username/osx-64/', 'https://conda.anaconda.org/binstar_username/noarch/', 'http://some.custom/channel/osx-64/', 'http://some.custom/channel/noarch/', 'https://repo.continuum.io/pkgs/free/osx-64/', 'https://repo.continuum.io/pkgs/free/noarch/', 'https://repo.continuum.io/pkgs/pro/osx-64/', 'https://repo.continuum.io/pkgs/pro/noarch/', 'https://conda.anaconda.org/username/osx-64/', 'https://conda.anaconda.org/username/noarch/', 'file:///Users/username/repo/osx-64/', 'file:///Users/username/repo/noarch/', 'https://mybinstar.com/t/5768wxyz/test2/osx-64/', 'https://mybinstar.com/t/5768wxyz/test2/noarch/', 'https://mybinstar.com/test/osx-64/', 'https://mybinstar.com/test/noarch/', 'https://conda.anaconda.org/t/abcdefgh/username/osx-64/', 'https://conda.anaconda.org/t/abcdefgh/username/noarch/', 'https://conda.anaconda.org/username/osx-64/', 'https://conda.anaconda.org/username/noarch/' ]