def test_clone_offline_multichannel_with_untracked(self):
with make_temp_env("python") as prefix:
assert_package_is_installed(prefix, 'python')
from conda.config import get_rc_urls
assert 'r' not in get_rc_urls()
# assert conda search cannot find rpy2
stdout, stderr = run_command(Commands.SEARCH, prefix, "rpy2", "--json")
json_obj = json_loads(stdout.replace("Fetching package metadata ...", "").strip())
assert bool(json_obj) is False
# add r channel
run_command(Commands.CONFIG, prefix, "--add channels r")
stdout, stderr = run_command(Commands.CONFIG, prefix, "--get", "--json")
json_obj = json_loads(stdout)
assert json_obj['rc_path'] == join(prefix, 'condarc')
assert json_obj['get']['channels']
# assert conda search can now find rpy2
stdout, stderr = run_command(Commands.SEARCH, prefix, "rpy2", "--json")
json_obj = json_loads(stdout.replace("Fetching package metadata ...", "").strip())
assert len(json_obj['rpy2']) > 1
run_command(Commands.INSTALL, prefix, "rpy2")
assert_package_is_installed(prefix, 'rpy2')
run_command(Commands.LIST, prefix)
with enforce_offline():
with make_temp_env("--clone", prefix, "--offline") as clone_prefix:
assert_package_is_installed(clone_prefix, 'python')
assert_package_is_installed(clone_prefix, 'rpy2')
assert isfile(join(clone_prefix, 'condarc')) # untracked file
def test_clone_offline_multichannel_with_untracked(self):
with make_temp_env("python") as prefix:
assert_package_is_installed(prefix, 'python')
from conda.config import get_rc_urls
assert 'r' not in get_rc_urls()
# assert conda search cannot find rpy2
stdout, stderr = run_command(Commands.SEARCH, prefix, "rpy2",
"--json")
json_obj = json_loads(
stdout.replace("Fetching package metadata ...", "").strip())
assert bool(json_obj) is False
run_command(Commands.CONFIG, prefix, "--add channels r")
# assert conda search cannot find rpy2
stdout, stderr = run_command(Commands.SEARCH, prefix, "rpy2",
"--json")
json_obj = json_loads(
stdout.replace("Fetching package metadata ...", "").strip())
assert len(json_obj['rpy2']) > 1
run_command(Commands.INSTALL, prefix, "rpy2")
assert_package_is_installed(prefix, 'rpy2')
run_command(Commands.LIST, prefix)
with enforce_offline():
with make_temp_env("--clone", prefix,
"--offline") as clone_prefix:
assert_package_is_installed(clone_prefix, 'python')
assert_package_is_installed(clone_prefix, 'rpy2')
assert isfile(join(clone_prefix,
'condarc')) # untracked file
def from_environment(name, prefix, no_builds=False):
installed = install.linked(prefix)
conda_pkgs = copy(installed)
# json=True hides the output, data is added to installed
add_pip_installed(prefix, installed, json=True)
pip_pkgs = sorted(installed - conda_pkgs)
if no_builds:
dependencies = ['='.join(a.rsplit('-', 2)[0:2]) for a in sorted(conda_pkgs)]
else:
dependencies = ['='.join(a.rsplit('-', 2)) for a in sorted(conda_pkgs)]
if len(pip_pkgs) > 0:
dependencies.append({'pip': ['=='.join(a.rsplit('-', 2)[:2]) for a in pip_pkgs]})
channels = config.get_rc_urls()
return Environment(name=name, dependencies=dependencies, channels=channels, prefix=prefix)
def from_environment(name, prefix, no_builds=False):
installed = install.linked(prefix)
conda_pkgs = copy(installed)
# json=True hides the output, data is added to installed
add_pip_installed(prefix, installed, json=True)
pip_pkgs = sorted(installed - conda_pkgs)
if no_builds:
dependencies = [
'='.join(a.rsplit('-', 2)[0:2]) for a in sorted(conda_pkgs)
]
else:
dependencies = ['='.join(a.rsplit('-', 2)) for a in sorted(conda_pkgs)]
if len(pip_pkgs) > 0:
dependencies.append(
{'pip': ['=='.join(a.rsplit('-', 2)[:2]) for a in pip_pkgs]})
channels = config.get_rc_urls()
return Environment(name=name,
dependencies=dependencies,
channels=channels,
prefix=prefix)
def test_normalize_urls(self):
current_platform = config.subdir
assert config.DEFAULT_CHANNEL_ALIAS == 'https://conda.anaconda.org/'
assert config.rc.get('channel_alias') == 'https://your.repo/'
assert config.channel_prefix(False) == 'https://your.repo/'
assert config.binstar_domain == 'https://mybinstar.com/'
assert config.binstar_domain_tok == 'https://mybinstar.com/t/01234abcde/'
assert config.get_rc_urls() == ["binstar_username", "http://some.custom/channel", "defaults"]
channel_urls = [
'defaults', 'system',
'https://conda.anaconda.org/username',
'file:///Users/username/repo',
'https://mybinstar.com/t/5768wxyz/test2',
'https://mybinstar.com/test',
'https://conda.anaconda.org/t/abcdefgh/username',
'username'
]
platform = 'osx-64'
normurls = config.normalize_urls(channel_urls, platform)
assert normurls == [
# defaults
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
# system (condarc)
'https://your.repo/binstar_username/osx-64/',
'https://your.repo/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
# defaults is repeated in condarc; that's OK
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
# conda.anaconda.org is not our default binstar clinet
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
# mybinstar.com is not channel_alias, but we still add tokens
'https://mybinstar.com/t/5768wxyz/test2/osx-64/',
'https://mybinstar.com/t/5768wxyz/test2/noarch/',
# token already supplied, do not change/remove it
'https://mybinstar.com/t/01234abcde/test/osx-64/',
'https://mybinstar.com/t/01234abcde/test/noarch/',
# we do not remove tokens from conda.anaconda.org
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/',
'https://conda.anaconda.org/t/abcdefgh/username/noarch/',
# short channel; add channel_alias
'https://your.repo/username/osx-64/',
'https://your.repo/username/noarch/']
priurls = config.prioritize_channels(normurls)
assert dict(priurls) == {
# defaults appears twice, keep higher priority
'https://repo.continuum.io/pkgs/free/noarch/': ('defaults', 1),
'https://repo.continuum.io/pkgs/free/osx-64/': ('defaults', 1),
'https://repo.continuum.io/pkgs/pro/noarch/': ('defaults', 1),
'https://repo.continuum.io/pkgs/pro/osx-64/': ('defaults', 1),
'https://your.repo/binstar_username/noarch/': ('binstar_username', 2),
'https://your.repo/binstar_username/osx-64/': ('binstar_username', 2),
'http://some.custom/channel/noarch/': ('http://some.custom/channel', 3),
'http://some.custom/channel/osx-64/': ('http://some.custom/channel', 3),
'https://conda.anaconda.org/t/abcdefgh/username/noarch/': ('https://conda.anaconda.org/username', 4),
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/': ('https://conda.anaconda.org/username', 4),
'file:///Users/username/repo/noarch/': ('file:///Users/username/repo', 5),
'file:///Users/username/repo/osx-64/': ('file:///Users/username/repo', 5),
# the tokenized version came first, but we still give it the same priority
'https://conda.anaconda.org/username/noarch/': ('https://conda.anaconda.org/username', 4),
'https://conda.anaconda.org/username/osx-64/': ('https://conda.anaconda.org/username', 4),
'https://mybinstar.com/t/5768wxyz/test2/noarch/': ('https://mybinstar.com/test2', 6),
'https://mybinstar.com/t/5768wxyz/test2/osx-64/': ('https://mybinstar.com/test2', 6),
'https://mybinstar.com/t/01234abcde/test/noarch/': ('https://mybinstar.com/test', 7),
'https://mybinstar.com/t/01234abcde/test/osx-64/': ('https://mybinstar.com/test', 7),
'https://your.repo/username/noarch/': ('username', 8),
'https://your.repo/username/osx-64/': ('username', 8)
}
# Delete the channel alias so now the short channels point to binstar
del config.rc['channel_alias']
config.rc['offline'] = False
config.load_condarc()
config.binstar_client = BinstarTester()
normurls = config.normalize_urls(channel_urls, platform)
# all your.repo references should be changed to mybinstar.com
assert normurls == [
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://mybinstar.com/t/01234abcde/binstar_username/osx-64/',
'https://mybinstar.com/t/01234abcde/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
'https://mybinstar.com/t/5768wxyz/test2/osx-64/',
'https://mybinstar.com/t/5768wxyz/test2/noarch/',
'https://mybinstar.com/t/01234abcde/test/osx-64/',
'https://mybinstar.com/t/01234abcde/test/noarch/',
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/',
'https://conda.anaconda.org/t/abcdefgh/username/noarch/',
'https://mybinstar.com/t/01234abcde/username/osx-64/',
'https://mybinstar.com/t/01234abcde/username/noarch/'
]
# Delete the anaconda token
config.load_condarc()
config.binstar_client = BinstarTester(token=None)
normurls = config.normalize_urls(channel_urls, platform)
# tokens should not be added (but supplied tokens are kept)
assert normurls == [
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://mybinstar.com/binstar_username/osx-64/',
'https://mybinstar.com/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
'https://mybinstar.com/t/5768wxyz/test2/osx-64/',
'https://mybinstar.com/t/5768wxyz/test2/noarch/',
'https://mybinstar.com/test/osx-64/',
'https://mybinstar.com/test/noarch/',
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/',
'https://conda.anaconda.org/t/abcdefgh/username/noarch/',
'https://mybinstar.com/username/osx-64/',
'https://mybinstar.com/username/noarch/'
]
# Turn off add_anaconda_token
config.rc['add_binstar_token'] = False
config.load_condarc()
config.binstar_client = BinstarTester()
normurls2 = config.normalize_urls(channel_urls, platform)
# tokens should not be added (but supplied tokens are kept)
assert normurls == normurls2
# Disable binstar client altogether
config.load_condarc()
config.binstar_client = ()
normurls = config.normalize_urls(channel_urls, platform)
# should drop back to conda.anaconda.org
assert normurls == [
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/binstar_username/osx-64/',
'https://conda.anaconda.org/binstar_username/noarch/',
'http://some.custom/channel/osx-64/',
'http://some.custom/channel/noarch/',
'https://repo.continuum.io/pkgs/free/osx-64/',
'https://repo.continuum.io/pkgs/free/noarch/',
'https://repo.continuum.io/pkgs/pro/osx-64/',
'https://repo.continuum.io/pkgs/pro/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/',
'file:///Users/username/repo/osx-64/',
'file:///Users/username/repo/noarch/',
'https://mybinstar.com/t/5768wxyz/test2/osx-64/',
'https://mybinstar.com/t/5768wxyz/test2/noarch/',
'https://mybinstar.com/test/osx-64/',
'https://mybinstar.com/test/noarch/',
'https://conda.anaconda.org/t/abcdefgh/username/osx-64/',
'https://conda.anaconda.org/t/abcdefgh/username/noarch/',
'https://conda.anaconda.org/username/osx-64/',
'https://conda.anaconda.org/username/noarch/'
]
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os import makedirs
from os.path import abspath, isdir, isfile
import conda.config as cc
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
check_external()
# change globals in build module, see comment there as well
build.channel_urls = args.channel or ()
build.override_channels = args.override_channels
build.verbose = not args.quiet or args.debug
if on_win:
try:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe,
# used by jinja2. see https://github.com/conda/conda-build/pull/520
delete_trash(None)
except:
# when we can't delete the trash, don't crash on AssertionError,
# instead inform the user and try again next time.
# see https://github.com/conda/conda-build/pull/744
warnings.warn("Cannot delete trash; some c extension has been "
"imported that is hard-linked by files in the trash. "
"Will try again on next run.")
set_language_env_vars(args, parser, execute=execute)
if args.skip_existing:
for d in config.bldpkgs_dirs:
if not isdir(d):
makedirs(d)
update_index(d)
arg_channels = ['local']
if args.channel:
arg_channels.extend(args.channel)
index = build.get_build_index(clear_cache=True,
arg_channels=arg_channels)
already_built = set()
to_build_recursive = []
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
# recurse looking for meta.yaml that is potentially not in immediate folder
recipe_dir = find_recipe(recipe_dir)
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
# this fully renders any jinja templating, throwing an error if any data is missing
m, need_source_download, need_reparse_in_env = render_recipe(recipe_dir,
no_download_source=False,
verbose=False,
dirty=args.dirty)
if m.get_value('build/noarch_python'):
config.noarch = True
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if m.skip():
print("Skipped: The %s recipe defines build/skip for this "
"configuration." % m.dist())
continue
if args.skip_existing:
urls = cc.get_rc_urls() + cc.get_local_urls() + ['local', ]
if args.channel:
urls.extend(args.channel)
# will be empty if none found, and evalute to False
package_exists = [url for url in urls if url + '::' + m.pkg_fn() in index]
if (package_exists or m.pkg_fn() in index or m.pkg_fn() in already_built):
print(m.dist(), "is already built in {0}, skipping.".format(package_exists))
continue
if args.output:
print(bldpkg_path(m))
continue
elif args.test:
build.test(m, move_broken=False, debug=args.debug)
elif args.source:
if need_source_download:
source.provide(m.path, m.get_section('source'), verbose=build.verbose)
print('Source tree in:', source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
build.build(m, post=post,
include_recipe=args.include_recipe,
keep_old_work=args.keep_old_work,
need_source_download=need_source_download,
need_reparse_in_env=need_reparse_in_env,
dirty=args.dirty, activate=args.activate,
debug=args.debug)
except (NoPackagesFound, Unsatisfiable) as e:
error_str = str(e)
# Typically if a conflict is with one of these
# packages, the other package needs to be rebuilt
# (e.g., a conflict with 'python 3.5*' and 'x' means
# 'x' isn't build for Python 3.5 and needs to be
# rebuilt).
skip_names = ['python', 'r']
add_recipes = []
for line in error_str.splitlines():
if not line.startswith(' - '):
continue
pkg = line.lstrip(' - ').split(' -> ')[-1]
pkg = pkg.strip().split(' ')[0]
if pkg in skip_names:
sys.stderr.write("Warning: package conflict - you may have unresolved "
"dependencies. Try to conda install each of your "
"dependencies to figure out which has unresolved "
"dependencies.")
continue
recipe_glob = glob(pkg + '-[v0-9][0-9.]*')
if os.path.exists(pkg):
recipe_glob.append(pkg)
if recipe_glob:
try_again = True
for recipe_dir in recipe_glob:
if pkg in to_build_recursive:
sys.exit(str(e))
print(error_str)
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(pkg))
add_recipes.append(recipe_dir)
to_build_recursive.append(pkg)
else:
raise
recipes.appendleft(arg)
recipes.extendleft(reversed(add_recipes))
if try_again:
continue
if not args.notest:
build.test(m, activate=args.activate, debug=args.debug)
if need_cleanup:
shutil.rmtree(recipe_dir)
# outputs message, or does upload, depending on value of args.binstar_upload
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.add(m.pkg_fn())
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os import makedirs
from os.path import abspath, isdir, isfile
import conda.config as cc
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
check_external()
# change globals in build module, see comment there as well
build.channel_urls = args.channel or ()
build.override_channels = args.override_channels
build.verbose = not args.quiet or args.debug
if on_win:
try:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe,
# used by jinja2. see https://github.com/conda/conda-build/pull/520
delete_trash(None)
except:
# when we can't delete the trash, don't crash on AssertionError,
# instead inform the user and try again next time.
# see https://github.com/conda/conda-build/pull/744
warnings.warn("Cannot delete trash; some c extension has been "
"imported that is hard-linked by files in the trash. "
"Will try again on next run.")
set_language_env_vars(args, parser, execute=execute)
if args.skip_existing:
for d in config.bldpkgs_dirs:
if not isdir(d):
makedirs(d)
update_index(d)
arg_channels = ['local']
if args.channel:
arg_channels.extend(args.channel)
index = build.get_build_index(clear_cache=True,
arg_channels=arg_channels)
already_built = set()
to_build_recursive = []
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
# recurse looking for meta.yaml that is potentially not in immediate folder
recipe_dir = find_recipe(recipe_dir)
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
# this fully renders any jinja templating, throwing an error if any data is missing
m, need_source_download, need_reparse_in_env = render_recipe(recipe_dir,
no_download_source=False,
verbose=False,
dirty=args.dirty)
if m.get_value('build/noarch_python'):
config.noarch = True
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if m.skip():
print("Skipped: The %s recipe defines build/skip for this "
"configuration." % m.dist())
continue
if args.skip_existing:
urls = cc.get_rc_urls() + cc.get_local_urls() + ['local', ]
if args.channel:
urls.extend(args.channel)
# will be empty if none found, and evalute to False
package_exists = [url for url in urls if url + '::' + m.pkg_fn() in index]
if (package_exists or m.pkg_fn() in index or m.pkg_fn() in already_built):
print(m.dist(), "is already built in {0}, skipping.".format(package_exists))
continue
if args.output:
print(bldpkg_path(m))
continue
elif args.test:
build.test(m, move_broken=False, debug=args.debug)
elif args.source:
if need_source_download:
source.provide(m.path, m.get_section('source'), verbose=build.verbose)
print('Source tree in:', source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
build.build(m, post=post,
include_recipe=args.include_recipe,
keep_old_work=args.keep_old_work,
need_source_download=need_source_download,
need_reparse_in_env=need_reparse_in_env,
dirty=args.dirty, activate=args.activate,
debug=args.debug)
except (NoPackagesFound, Unsatisfiable) as e:
error_str = str(e)
# Typically if a conflict is with one of these
# packages, the other package needs to be rebuilt
# (e.g., a conflict with 'python 3.5*' and 'x' means
# 'x' isn't build for Python 3.5 and needs to be
# rebuilt).
skip_names = ['python', 'r']
add_recipes = []
for line in error_str.splitlines():
if not line.startswith(' - '):
continue
pkg = line.lstrip(' - ').split(' -> ')[-1]
pkg = pkg.strip().split(' ')[0]
if pkg in skip_names:
sys.stderr.write("Warning: package conflict - you may have unresolved "
"dependencies. Try to conda install each of your "
"dependencies to figure out which has unresolved "
"dependencies.")
continue
recipe_glob = glob(pkg + '-[v0-9][0-9.]*')
if os.path.exists(pkg):
recipe_glob.append(pkg)
if recipe_glob:
try_again = True
for recipe_dir in recipe_glob:
if pkg in to_build_recursive:
sys.exit(str(e))
print(error_str)
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(pkg))
add_recipes.append(recipe_dir)
to_build_recursive.append(pkg)
else:
raise
recipes.appendleft(arg)
recipes.extendleft(reversed(add_recipes))
if try_again:
continue
if not args.notest:
build.test(m, activate=args.activate, debug=args.debug)
if need_cleanup:
shutil.rmtree(recipe_dir)
# outputs message, or does upload, depending on value of args.binstar_upload
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.add(m.pkg_fn())
