def test_trash_long_paths(self):
unc_prefix = '\\\\?\\' if platform == 'win32' else ''
pkg_dir = config.pkgs_dirs[0]
longfoldername="trash_with_a_very_very_long_and_silly_name_indeed"
tmpdir=pkg_dir
for i in range(6):
tmpdir = join(tmpdir, longfoldername)
if not i:
toptmpdir = tmpdir
tmpfile = join(tmpdir, self.tmpfname)
makedirs(unc_prefix + tmpdir)
with open(unc_prefix + tmpfile, 'w') as fo:
fo.write('trashy')
delete_trash(config.pkgs_dirs[0])
move_path_to_trash(toptmpdir)
trash = _get_trash_dir(config.pkgs_dirs[0])
contents = [basename(dp) for dp, dn, fn in walk(trash)]
self.assertIn(longfoldername, contents)
delete_trash(config.pkgs_dirs[0])
contents = [basename(dp) for dp, dn, fn in walk(trash)]
self.assertTrue(longfoldername not in contents)
def test_trash_long_paths(self):
unc_prefix = u'\\\\?\\' if platform == 'win32' else ''
pkg_dir = config.pkgs_dirs[0]
longfoldername = "trash_with_a_very_very_long_and_silly_name_indeed"
tmpdir = pkg_dir
for i in range(6):
tmpdir = join(tmpdir, longfoldername)
if not i:
toptmpdir = tmpdir
tmpfile = join(tmpdir, self.tmpfname)
makedirs(unc_prefix + tmpdir)
with open(unc_prefix + tmpfile, 'w') as fo:
fo.write('trashy')
delete_trash(config.pkgs_dirs[0])
delete_trash(config.pkgs_dirs[0])
move_path_to_trash(toptmpdir)
trash = _get_trash_dir(config.pkgs_dirs[0])
contents = [basename(dp) for dp, dn, fn in walk(trash)]
self.assertIn(longfoldername, contents)
delete_trash(config.pkgs_dirs[0])
delete_trash(config.pkgs_dirs[0])
contents = [basename(dp) for dp, dn, fn in walk(trash)]
self.assertTrue(longfoldername not in contents)
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os.path import abspath, isdir, isfile
from conda.lock import Locked
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
from conda_build.metadata import MetaData
check_external()
channel_urls = args.channel or ()
if on_win:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe, used
# by jinja2. see https://github.com/conda/conda-build/pull/520
assert 'markupsafe' not in sys.modules
delete_trash(None)
conda_version = {
'python': 'CONDA_PY',
'numpy': 'CONDA_NPY',
'perl': 'CONDA_PERL',
'R': 'CONDA_R',
}
for lang in ['python', 'numpy', 'perl', 'R']:
versions = getattr(args, lang)
if not versions:
continue
if versions == ['all']:
if all_versions[lang]:
versions = all_versions[lang]
else:
parser.error("'all' is not supported for --%s" % lang)
if len(versions) > 1:
for ver in versions[:]:
setattr(args, lang, [str(ver)])
execute(args, parser)
# This is necessary to make all combinations build.
setattr(args, lang, versions)
return
else:
version = int(versions[0].replace('.', ''))
setattr(config, conda_version[lang], version)
if not len(str(version)) == 2 and lang in ['python', 'numpy']:
if all_versions[lang]:
raise RuntimeError("%s must be major.minor, like %s, not %s" %
(conda_version[lang], all_versions[lang][-1]/10, version))
else:
raise RuntimeError("%s must be major.minor, not %s" %
(conda_version[lang], version))
if args.skip_existing:
update_index(config.bldpkgs_dir)
index = build.get_build_index(clear_cache=True,
channel_urls=channel_urls,
override_channels=args.override_channels)
already_built = []
to_build = args.recipe[:]
with Locked(config.croot):
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir)
if m.get_value('build/noarch_python'):
config.noarch = True
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
binstar_upload = False
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if args.skip_existing:
if m.pkg_fn() in index or m.pkg_fn() in already_built:
print("%s is already built, skipping." % m.dist())
continue
if args.output:
print(build.bldpkg_path(m))
continue
elif args.test:
build.test(m, verbose=not args.quiet,
channel_urls=channel_urls, override_channels=args.override_channels)
elif args.source:
source.provide(m.path, m.get_section('source'))
print('Source tree in:', source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
build.build(m, verbose=not args.quiet, post=post,
channel_urls=channel_urls,
override_channels=args.override_channels, include_recipe=args.include_recipe)
except RuntimeError as e:
error_str = str(e)
if error_str.startswith('No packages found') or error_str.startswith('Could not find some'):
# Build dependency if recipe exists
dep_pkg = error_str.split(': ')[1]
# Handle package names that contain version deps.
if ' ' in dep_pkg:
dep_pkg = dep_pkg.split(' ')[0]
recipe_glob = glob(dep_pkg + '-[v0-9][0-9.]*')
if exists(dep_pkg):
recipe_glob.append(dep_pkg)
if recipe_glob:
recipes.appendleft(arg)
try_again = True
for recipe_dir in recipe_glob:
if dep_pkg in to_build:
sys.exit(str(e))
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(dep_pkg))
recipes.appendleft(recipe_dir)
to_build.append(dep_pkg)
else:
raise
else:
raise
if try_again:
continue
if not args.notest:
build.test(m, verbose=not args.quiet,
channel_urls=channel_urls, override_channels=args.override_channels)
binstar_upload = True
if need_cleanup:
shutil.rmtree(recipe_dir)
if binstar_upload:
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.append(m.pkg_fn())
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os.path import abspath, isdir, isfile
from conda.lock import Locked
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
from conda_build.metadata import MetaData
check_external()
channel_urls = args.channel or ()
if on_win:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe, used
# by jinja2. see https://github.com/conda/conda-build/pull/520
assert 'markupsafe' not in sys.modules
delete_trash(None)
conda_version = {
'python': 'CONDA_PY',
'numpy': 'CONDA_NPY',
'perl': 'CONDA_PERL',
'R': 'CONDA_R',
}
for lang in ['python', 'numpy', 'perl', 'R']:
versions = getattr(args, lang)
if not versions:
continue
if versions == ['all']:
if all_versions[lang]:
versions = all_versions[lang]
else:
parser.error("'all' is not supported for --%s" % lang)
if len(versions) > 1:
for ver in versions[:]:
setattr(args, lang, [str(ver)])
execute(args, parser)
# This is necessary to make all combinations build.
setattr(args, lang, versions)
return
else:
version = int(versions[0].replace('.', ''))
setattr(config, conda_version[lang], version)
if not len(str(version)) == 2 and lang in ['python', 'numpy']:
if all_versions[lang]:
raise RuntimeError("%s must be major.minor, like %s, not %s" %
(conda_version[lang],
all_versions[lang][-1] / 10, version))
else:
raise RuntimeError("%s must be major.minor, not %s" %
(conda_version[lang], version))
if args.skip_existing:
update_index(config.bldpkgs_dir)
index = build.get_build_index(clear_cache=True,
channel_urls=channel_urls,
override_channels=args.override_channels)
already_built = []
to_build = args.recipe[:]
with Locked(config.croot):
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir)
if m.get_value('build/noarch_python'):
config.noarch = True
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
binstar_upload = False
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if args.skip_existing:
if m.pkg_fn() in index or m.pkg_fn() in already_built:
print("%s is already built, skipping." % m.dist())
continue
if args.output:
print(build.bldpkg_path(m))
continue
elif args.test:
build.test(m,
verbose=not args.quiet,
channel_urls=channel_urls,
override_channels=args.override_channels)
elif args.source:
source.provide(m.path, m.get_section('source'))
print('Source tree in:', source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
build.build(m,
verbose=not args.quiet,
post=post,
channel_urls=channel_urls,
override_channels=args.override_channels,
include_recipe=args.include_recipe)
except RuntimeError as e:
error_str = str(e)
if error_str.startswith(
'No packages found') or error_str.startswith(
'Could not find some'):
# Build dependency if recipe exists
dep_pkg = error_str.split(': ')[1]
# Handle package names that contain version deps.
if ' ' in dep_pkg:
dep_pkg = dep_pkg.split(' ')[0]
recipe_glob = glob(dep_pkg + '-[v0-9][0-9.]*')
if exists(dep_pkg):
recipe_glob.append(dep_pkg)
if recipe_glob:
recipes.appendleft(arg)
try_again = True
for recipe_dir in recipe_glob:
if dep_pkg in to_build:
sys.exit(str(e))
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(dep_pkg))
recipes.appendleft(recipe_dir)
to_build.append(dep_pkg)
else:
raise
else:
raise
if try_again:
continue
if not args.notest:
build.test(m,
verbose=not args.quiet,
channel_urls=channel_urls,
override_channels=args.override_channels)
binstar_upload = True
if need_cleanup:
shutil.rmtree(recipe_dir)
if binstar_upload:
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.append(m.pkg_fn())
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os import makedirs
from os.path import abspath, isdir, isfile
from conda.lock import Locked
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
from conda_build.metadata import MetaData
check_external()
channel_urls = args.channel or ()
if on_win:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe, used
# by jinja2. see https://github.com/conda/conda-build/pull/520
assert "markupsafe" not in sys.modules
delete_trash(None)
conda_version = {"python": "CONDA_PY", "numpy": "CONDA_NPY", "perl": "CONDA_PERL", "R": "CONDA_R"}
for lang in ["python", "numpy", "perl", "R"]:
versions = getattr(args, lang)
if not versions:
continue
if versions == ["all"]:
if all_versions[lang]:
versions = all_versions[lang]
else:
parser.error("'all' is not supported for --%s" % lang)
if len(versions) > 1:
for ver in versions[:]:
setattr(args, lang, [str(ver)])
execute(args, parser)
# This is necessary to make all combinations build.
setattr(args, lang, versions)
return
else:
version = versions[0]
if lang in ("python", "numpy"):
version = int(version.replace(".", ""))
setattr(config, conda_version[lang], version)
if not len(str(version)) in (2, 3) and lang in ["python", "numpy"]:
if all_versions[lang]:
raise RuntimeError(
"%s must be major.minor, like %s, not %s"
% (conda_version[lang], all_versions[lang][-1] / 10, version)
)
else:
raise RuntimeError("%s must be major.minor, not %s" % (conda_version[lang], version))
# Using --python, --numpy etc. is equivalent to using CONDA_PY, CONDA_NPY, etc.
# Auto-set those env variables
for var in conda_version.values():
if getattr(config, var):
# Set the env variable.
os_environ[var] = str(getattr(config, var))
if args.skip_existing:
if not isdir(config.bldpkgs_dir):
makedirs(config.bldpkgs_dir)
update_index(config.bldpkgs_dir)
index = build.get_build_index(
clear_cache=True, channel_urls=channel_urls, override_channels=args.override_channels
)
already_built = []
to_build_recursive = []
with Locked(config.croot):
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or "utf-8")
if isfile(arg):
if arg.endswith((".tar", ".tar.gz", ".tgz", ".tar.bz2")):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, "r:*")
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir)
if m.get_value("build/noarch_python"):
config.noarch = True
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
binstar_upload = False
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if args.skip_existing:
if m.pkg_fn() in index or m.pkg_fn() in already_built:
print("%s is already built, skipping." % m.dist())
continue
if args.output:
print(build.bldpkg_path(m))
continue
elif args.test:
build.test(
m, verbose=not args.quiet, channel_urls=channel_urls, override_channels=args.override_channels
)
elif args.source:
source.provide(m.path, m.get_section("source"))
print("Source tree in:", source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
if m.skip():
print("Skipped: The %s recipe defines build/skip for this " "configuration." % m.dist())
continue
build.build(
m,
verbose=not args.quiet,
post=post,
channel_urls=channel_urls,
override_channels=args.override_channels,
include_recipe=args.include_recipe,
)
except (RuntimeError, SystemExit) as e:
error_str = str(e)
if error_str.startswith("No packages found") or error_str.startswith("Could not find some"):
# Build dependency if recipe exists
dep_pkg = error_str.split(": ")[1]
# Handle package names that contain version deps.
if " " in dep_pkg:
dep_pkg = dep_pkg.split(" ")[0]
recipe_glob = glob(dep_pkg + "-[v0-9][0-9.]*")
if exists(dep_pkg):
recipe_glob.append(dep_pkg)
if recipe_glob:
recipes.appendleft(arg)
try_again = True
for recipe_dir in recipe_glob:
if dep_pkg in to_build_recursive:
sys.exit(str(e))
print(
(
"Missing dependency {0}, but found"
+ " recipe directory, so building "
+ "{0} first"
).format(dep_pkg)
)
recipes.appendleft(recipe_dir)
to_build_recursive.append(dep_pkg)
else:
raise
elif error_str.strip().startswith("Hint:"):
lines = [line for line in error_str.splitlines() if line.strip().startswith("- ")]
pkgs = [line.lstrip("- ") for line in lines]
# Typically if a conflict is with one of these
# packages, the other package needs to be rebuilt
# (e.g., a conflict with 'python 3.5*' and 'x' means
# 'x' isn't build for Python 3.5 and needs to be
# rebuilt).
skip_names = ["python", "r"]
pkgs = [pkg for pkg in pkgs if pkg.split(" ")[0] not in skip_names]
for pkg in pkgs:
# Handle package names that contain version deps.
if " " in pkg:
pkg = pkg.split(" ")[0]
recipe_glob = glob(pkg + "-[v0-9][0-9.]*")
if exists(pkg):
recipe_glob.append(pkg)
if recipe_glob:
recipes.appendleft(arg)
try_again = True
for recipe_dir in recipe_glob:
if pkg in to_build_recursive:
sys.exit(str(e))
print(error_str)
print(
(
"Missing dependency {0}, but found"
+ " recipe directory, so building "
+ "{0} first"
).format(pkg)
)
recipes.appendleft(recipe_dir)
to_build_recursive.append(pkg)
else:
raise
else:
raise
if try_again:
continue
if not args.notest:
build.test(
m, verbose=not args.quiet, channel_urls=channel_urls, override_channels=args.override_channels
)
binstar_upload = True
if need_cleanup:
shutil.rmtree(recipe_dir)
if binstar_upload:
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.append(m.pkg_fn())
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os import makedirs
from os.path import abspath, isdir, isfile
from conda.lock import Locked
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
from conda_build.metadata import MetaData
check_external()
# change globals in build module, see comment there as well
build.channel_urls = args.channel or ()
build.override_channels = args.override_channels
build.verbose = not args.quiet
if on_win:
try:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe,
# used by jinja2. see https://github.com/conda/conda-build/pull/520
delete_trash(None)
except:
# when we can't delete the trash, don't crash on AssertionError,
# instead inform the user and try again next time.
# see https://github.com/conda/conda-build/pull/744
warnings.warn("Cannot delete trash; some c extension has been "
"imported that is hard-linked by files in the trash. "
"Will try again on next run.")
conda_version = {
'python': 'CONDA_PY',
'numpy': 'CONDA_NPY',
'perl': 'CONDA_PERL',
'R': 'CONDA_R',
}
for lang in ['python', 'numpy', 'perl', 'R']:
versions = getattr(args, lang)
if not versions:
continue
if versions == ['all']:
if all_versions[lang]:
versions = all_versions[lang]
else:
parser.error("'all' is not supported for --%s" % lang)
if len(versions) > 1:
for ver in versions[:]:
setattr(args, lang, [str(ver)])
execute(args, parser)
# This is necessary to make all combinations build.
setattr(args, lang, versions)
return
else:
version = versions[0]
if lang in ('python', 'numpy'):
version = int(version.replace('.', ''))
setattr(config, conda_version[lang], version)
if not len(str(version)) in (2, 3) and lang in ['python', 'numpy']:
if all_versions[lang]:
raise RuntimeError("%s must be major.minor, like %s, not %s" %
(conda_version[lang], all_versions[lang][-1]/10, version))
else:
raise RuntimeError("%s must be major.minor, not %s" %
(conda_version[lang], version))
# Using --python, --numpy etc. is equivalent to using CONDA_PY, CONDA_NPY, etc.
# Auto-set those env variables
for var in conda_version.values():
if getattr(config, var):
# Set the env variable.
os_environ[var] = str(getattr(config, var))
if args.skip_existing:
if not isdir(config.bldpkgs_dir):
makedirs(config.bldpkgs_dir)
update_index(config.bldpkgs_dir)
index = build.get_build_index(clear_cache=True)
already_built = []
to_build_recursive = []
with Locked(config.croot):
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir)
if m.get_value('build/noarch_python'):
config.noarch = True
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
binstar_upload = False
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if args.skip_existing:
if m.pkg_fn() in index or m.pkg_fn() in already_built:
print("%s is already built, skipping." % m.dist())
continue
if m.skip():
print("Skipped: The %s recipe defines build/skip for this "
"configuration." % m.dist())
continue
if args.output:
try:
m.parse_again(permit_undefined_jinja=False)
except SystemExit:
# Something went wrong; possibly due to undefined GIT_ jinja variables.
# Maybe we need to actually download the source in order to resolve the build_id.
source.provide(m.path, m.get_section('source'))
# Parse our metadata again because we did not initialize the source
# information before.
m.parse_again(permit_undefined_jinja=False)
print(build.bldpkg_path(m))
continue
elif args.test:
build.test(m, move_broken=False)
elif args.source:
source.provide(m.path, m.get_section('source'))
print('Source tree in:', source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
build.build(m, post=post,
include_recipe=args.include_recipe)
except (RuntimeError, SystemExit) as e:
error_str = str(e)
if error_str.startswith('No packages found') or error_str.startswith('Could not find some'):
# Build dependency if recipe exists
dep_pkg = error_str.split(': ')[1]
# Handle package names that contain version deps.
if ' ' in dep_pkg:
dep_pkg = dep_pkg.split(' ')[0]
recipe_glob = glob(dep_pkg + '-[v0-9][0-9.]*')
if exists(dep_pkg):
recipe_glob.append(dep_pkg)
if recipe_glob:
recipes.appendleft(arg)
try_again = True
for recipe_dir in recipe_glob:
if dep_pkg in to_build_recursive:
sys.exit(str(e))
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(dep_pkg))
recipes.appendleft(recipe_dir)
to_build_recursive.append(dep_pkg)
else:
raise
elif error_str.strip().startswith("Hint:"):
lines = [line for line in error_str.splitlines() if line.strip().startswith('- ')]
pkgs = [line.lstrip('- ') for line in lines]
# Typically if a conflict is with one of these
# packages, the other package needs to be rebuilt
# (e.g., a conflict with 'python 3.5*' and 'x' means
# 'x' isn't build for Python 3.5 and needs to be
# rebuilt).
skip_names = ['python', 'r']
pkgs = [pkg for pkg in pkgs if pkg.split(' ')[0] not
in skip_names]
for pkg in pkgs:
# Handle package names that contain version deps.
if ' ' in pkg:
pkg = pkg.split(' ')[0]
recipe_glob = glob(pkg + '-[v0-9][0-9.]*')
if exists(pkg):
recipe_glob.append(pkg)
if recipe_glob:
recipes.appendleft(arg)
try_again = True
for recipe_dir in recipe_glob:
if pkg in to_build_recursive:
sys.exit(str(e))
print(error_str)
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(pkg))
recipes.appendleft(recipe_dir)
to_build_recursive.append(pkg)
else:
raise
else:
raise
if try_again:
continue
if not args.notest:
build.test(m)
binstar_upload = True
if need_cleanup:
shutil.rmtree(recipe_dir)
if binstar_upload:
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.append(m.pkg_fn())
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os import makedirs
from os.path import abspath, isdir, isfile
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
check_external()
# change globals in build module, see comment there as well
build.channel_urls = args.channel or ()
build.override_channels = args.override_channels
build.verbose = not args.quiet
if on_win:
try:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe,
# used by jinja2. see https://github.com/conda/conda-build/pull/520
delete_trash(None)
except:
# when we can't delete the trash, don't crash on AssertionError,
# instead inform the user and try again next time.
# see https://github.com/conda/conda-build/pull/744
warnings.warn(
"Cannot delete trash; some c extension has been "
"imported that is hard-linked by files in the trash. "
"Will try again on next run.")
set_language_env_vars(args, parser, execute=execute)
if args.skip_existing:
for d in config.bldpkgs_dirs:
if not isdir(d):
makedirs(d)
update_index(d)
index = build.get_build_index(clear_cache=True)
already_built = set()
to_build_recursive = []
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
# recurse looking for meta.yaml that is potentially not in immediate folder
recipe_dir = find_recipe(recipe_dir)
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
# this fully renders any jinja templating, throwing an error if any data is missing
m, need_source_download = render_recipe(recipe_dir,
no_download_source=False,
verbose=False,
dirty=args.dirty)
if m.get_value('build/noarch_python'):
config.noarch = True
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if m.skip():
print("Skipped: The %s recipe defines build/skip for this "
"configuration." % m.dist())
continue
if args.skip_existing:
# 'or m.pkg_fn() in index' is for conda <4.1 and could be removed in the future.
if ('local::' + m.pkg_fn() in index or m.pkg_fn() in index
or m.pkg_fn() in already_built):
print(m.dist(), "is already built, skipping.")
continue
if args.output:
print(bldpkg_path(m))
continue
elif args.test:
build.test(m, move_broken=False)
elif args.source:
source.provide(m.path,
m.get_section('source'),
verbose=build.verbose)
print('Source tree in:', source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
build.build(m,
post=post,
include_recipe=args.include_recipe,
keep_old_work=args.keep_old_work,
need_source_download=need_source_download,
dirty=args.dirty)
except (NoPackagesFound, Unsatisfiable) as e:
error_str = str(e)
# Typically if a conflict is with one of these
# packages, the other package needs to be rebuilt
# (e.g., a conflict with 'python 3.5*' and 'x' means
# 'x' isn't build for Python 3.5 and needs to be
# rebuilt).
skip_names = ['python', 'r']
add_recipes = []
for line in error_str.splitlines():
if not line.startswith(' - '):
continue
pkg = line.lstrip(' - ').split(' -> ')[-1]
pkg = pkg.strip().split(' ')[0]
if pkg in skip_names:
continue
recipe_glob = glob(pkg + '-[v0-9][0-9.]*')
if os.path.exists(pkg):
recipe_glob.append(pkg)
if recipe_glob:
try_again = True
for recipe_dir in recipe_glob:
if pkg in to_build_recursive:
sys.exit(str(e))
print(error_str)
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(pkg))
add_recipes.append(recipe_dir)
to_build_recursive.append(pkg)
else:
raise
recipes.appendleft(arg)
recipes.extendleft(reversed(add_recipes))
if try_again:
continue
if not args.notest:
build.test(m)
if need_cleanup:
shutil.rmtree(recipe_dir)
# outputs message, or does upload, depending on value of args.binstar_upload
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.add(m.pkg_fn())
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os import makedirs
from os.path import abspath, isdir, isfile
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
check_external()
# change globals in build module, see comment there as well
build.channel_urls = args.channel or ()
build.override_channels = args.override_channels
build.verbose = not args.quiet
if on_win:
try:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe,
# used by jinja2. see https://github.com/conda/conda-build/pull/520
delete_trash(None)
except:
# when we can't delete the trash, don't crash on AssertionError,
# instead inform the user and try again next time.
# see https://github.com/conda/conda-build/pull/744
warnings.warn("Cannot delete trash; some c extension has been "
"imported that is hard-linked by files in the trash. "
"Will try again on next run.")
set_language_env_vars(args, parser, execute=execute)
if args.skip_existing:
for d in config.bldpkgs_dirs:
if not isdir(d):
makedirs(d)
update_index(d)
index = build.get_build_index(clear_cache=True)
already_built = set()
to_build_recursive = []
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
# recurse looking for meta.yaml that is potentially not in immediate folder
recipe_dir = find_recipe(recipe_dir)
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
# this fully renders any jinja templating, throwing an error if any data is missing
m, need_source_download = render_recipe(recipe_dir, no_download_source=False,
verbose=False)
if m.get_value('build/noarch_python'):
config.noarch = True
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if m.skip():
print("Skipped: The %s recipe defines build/skip for this "
"configuration." % m.dist())
continue
if args.skip_existing:
# 'or m.pkg_fn() in index' is for conda <4.1 and could be removed in the future.
if ('local::' + m.pkg_fn() in index or
m.pkg_fn() in index or
m.pkg_fn() in already_built):
print(m.dist(), "is already built, skipping.")
continue
if args.output:
print(bldpkg_path(m))
continue
elif args.test:
build.test(m, move_broken=False)
elif args.source:
source.provide(m.path, m.get_section('source'), verbose=build.verbose)
print('Source tree in:', source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
build.build(m, post=post,
include_recipe=args.include_recipe,
keep_old_work=args.keep_old_work,
need_source_download=need_source_download,
dirty=args.dirty)
except (NoPackagesFound, Unsatisfiable) as e:
error_str = str(e)
# Typically if a conflict is with one of these
# packages, the other package needs to be rebuilt
# (e.g., a conflict with 'python 3.5*' and 'x' means
# 'x' isn't build for Python 3.5 and needs to be
# rebuilt).
skip_names = ['python', 'r']
add_recipes = []
for line in error_str.splitlines():
if not line.startswith(' - '):
continue
pkg = line.lstrip(' - ').split(' -> ')[-1]
pkg = pkg.strip().split(' ')[0]
if pkg in skip_names:
continue
recipe_glob = glob(pkg + '-[v0-9][0-9.]*')
if os.path.exists(pkg):
recipe_glob.append(pkg)
if recipe_glob:
try_again = True
for recipe_dir in recipe_glob:
if pkg in to_build_recursive:
sys.exit(str(e))
print(error_str)
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(pkg))
add_recipes.append(recipe_dir)
to_build_recursive.append(pkg)
else:
raise
recipes.appendleft(arg)
recipes.extendleft(reversed(add_recipes))
if try_again:
continue
if not args.notest:
build.test(m)
if need_cleanup:
shutil.rmtree(recipe_dir)
# outputs message, or does upload, depending on value of args.binstar_upload
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.add(m.pkg_fn())
def execute(args, parser):
import sys
import shutil
import tarfile
import tempfile
from os import makedirs
from os.path import abspath, isdir, isfile
from conda.lock import Locked
import conda_build.build as build
import conda_build.source as source
from conda_build.config import config
from conda_build.metadata import MetaData
check_external()
# change globals in build module, see comment there as well
build.channel_urls = args.channel or ()
build.override_channels = args.override_channels
build.verbose = not args.quiet
if on_win:
try:
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe,
# used by jinja2. see https://github.com/conda/conda-build/pull/520
delete_trash(None)
except:
# when we can't delete the trash, don't crash on AssertionError,
# instead inform the user and try again next time.
# see https://github.com/conda/conda-build/pull/744
warnings.warn(
"Cannot delete trash; some c extension has been "
"imported that is hard-linked by files in the trash. "
"Will try again on next run.")
conda_version = {
'python': 'CONDA_PY',
'numpy': 'CONDA_NPY',
'perl': 'CONDA_PERL',
'R': 'CONDA_R',
}
for lang in ['python', 'numpy', 'perl', 'R']:
versions = getattr(args, lang)
if not versions:
continue
if versions == ['all']:
if all_versions[lang]:
versions = all_versions[lang]
else:
parser.error("'all' is not supported for --%s" % lang)
if len(versions) > 1:
for ver in versions[:]:
setattr(args, lang, [str(ver)])
execute(args, parser)
# This is necessary to make all combinations build.
setattr(args, lang, versions)
return
else:
version = versions[0]
if lang in ('python', 'numpy'):
version = int(version.replace('.', ''))
setattr(config, conda_version[lang], version)
if not len(str(version)) in (2, 3) and lang in ['python', 'numpy']:
if all_versions[lang]:
raise RuntimeError("%s must be major.minor, like %s, not %s" %
(conda_version[lang],
all_versions[lang][-1] / 10, version))
else:
raise RuntimeError("%s must be major.minor, not %s" %
(conda_version[lang], version))
# Using --python, --numpy etc. is equivalent to using CONDA_PY, CONDA_NPY, etc.
# Auto-set those env variables
for var in conda_version.values():
if getattr(config, var):
# Set the env variable.
os_environ[var] = str(getattr(config, var))
if args.skip_existing:
if not isdir(config.bldpkgs_dir):
makedirs(config.bldpkgs_dir)
update_index(config.bldpkgs_dir)
index = build.get_build_index(clear_cache=True)
already_built = []
to_build_recursive = []
with Locked(config.croot):
recipes = deque(args.recipe)
while recipes:
arg = recipes.popleft()
try_again = False
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
else:
print("Ignoring non-recipe: %s" % arg)
continue
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir)
if m.get_value('build/noarch_python'):
config.noarch = True
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
binstar_upload = False
if args.check and len(args.recipe) > 1:
print(m.path)
m.check_fields()
if args.check:
continue
if args.skip_existing:
if m.pkg_fn() in index or m.pkg_fn() in already_built:
print("%s is already built, skipping." % m.dist())
continue
if m.skip():
print("Skipped: The %s recipe defines build/skip for this "
"configuration." % m.dist())
continue
if args.output:
try:
m.parse_again(permit_undefined_jinja=False)
except SystemExit:
# Something went wrong; possibly due to undefined GIT_ jinja variables.
# Maybe we need to actually download the source in order to resolve the build_id.
source.provide(m.path, m.get_section('source'))
# Parse our metadata again because we did not initialize the source
# information before.
m.parse_again(permit_undefined_jinja=False)
print(build.bldpkg_path(m))
continue
elif args.test:
build.test(m, move_broken=False)
elif args.source:
source.provide(m.path, m.get_section('source'))
print('Source tree in:', source.get_dir())
else:
# This loop recursively builds dependencies if recipes exist
if args.build_only:
post = False
args.notest = True
args.binstar_upload = False
elif args.post:
post = True
args.notest = True
args.binstar_upload = False
else:
post = None
try:
build.build(m,
post=post,
include_recipe=args.include_recipe)
except (RuntimeError, SystemExit) as e:
error_str = str(e)
if error_str.startswith(
'No packages found') or error_str.startswith(
'Could not find some'):
# Build dependency if recipe exists
dep_pkg = error_str.split(': ')[1]
# Handle package names that contain version deps.
if ' ' in dep_pkg:
dep_pkg = dep_pkg.split(' ')[0]
recipe_glob = glob(dep_pkg + '-[v0-9][0-9.]*')
if exists(dep_pkg):
recipe_glob.append(dep_pkg)
if recipe_glob:
recipes.appendleft(arg)
try_again = True
for recipe_dir in recipe_glob:
if dep_pkg in to_build_recursive:
sys.exit(str(e))
print(("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(dep_pkg))
recipes.appendleft(recipe_dir)
to_build_recursive.append(dep_pkg)
else:
raise
elif error_str.strip().startswith("Hint:"):
lines = [
line for line in error_str.splitlines()
if line.strip().startswith('- ')
]
pkgs = [line.lstrip('- ') for line in lines]
# Typically if a conflict is with one of these
# packages, the other package needs to be rebuilt
# (e.g., a conflict with 'python 3.5*' and 'x' means
# 'x' isn't build for Python 3.5 and needs to be
# rebuilt).
skip_names = ['python', 'r']
pkgs = [
pkg for pkg in pkgs
if pkg.split(' ')[0] not in skip_names
]
for pkg in pkgs:
# Handle package names that contain version deps.
if ' ' in pkg:
pkg = pkg.split(' ')[0]
recipe_glob = glob(pkg + '-[v0-9][0-9.]*')
if exists(pkg):
recipe_glob.append(pkg)
if recipe_glob:
recipes.appendleft(arg)
try_again = True
for recipe_dir in recipe_glob:
if pkg in to_build_recursive:
sys.exit(str(e))
print(error_str)
print(
("Missing dependency {0}, but found" +
" recipe directory, so building " +
"{0} first").format(pkg))
recipes.appendleft(recipe_dir)
to_build_recursive.append(pkg)
else:
raise
else:
raise
if try_again:
continue
if not args.notest:
build.test(m)
binstar_upload = True
if need_cleanup:
shutil.rmtree(recipe_dir)
if binstar_upload:
handle_binstar_upload(build.bldpkg_path(m), args)
already_built.append(m.pkg_fn())
def main():
parser = argparse.ArgumentParser(
description='Build a wheel in a conda environment.')
parser.add_argument('--version',
action='version',
version=conda_build_wheel.__version__)
parser.add_argument('recipe',
help='The folder containing the wheel recipe to build.')
parser.add_argument('--upload-pypi',
nargs='*',
default=[],
help='The PyPI channel(s) to upload built wheels to.')
parser.add_argument(
'--wheel-dir',
default="./build",
metavar='WHEEL_DIR_PATH',
help="Path to the directory where the wheels should be created.")
parser.add_argument(
'--python',
action="append",
help="""Set the Python version used by wheel build. Can be passed
multiple times to build against multiple versions. Can be 'all' to
build against all known versions (%r)""" %
[i for i in PythonVersionCompleter() if '.' in i],
metavar="PYTHON_VER",
choices=PythonVersionCompleter(), )
parser.add_argument(
'--numpy',
action="append",
help="""Set the NumPy version used by the wheel build. Can be passed
multiple times to build against multiple versions. Can be 'all' to
build against all known versions (%r)""" %
[i for i in NumPyVersionCompleter() if '.' in i],
metavar="NUMPY_VER",
choices=NumPyVersionCompleter(), )
add_parser_channels(parser)
args = parser.parse_args()
if sys.platform == 'win32':
# needs to happen before any c extensions are imported that might be
# hard-linked by files in the trash. one of those is markupsafe, used
# by jinja2. see https://github.com/conda/conda-build/pull/520
assert 'markupsafe' not in sys.modules
from conda.install import delete_trash
delete_trash(None)
b = conda_build_wheel.builder.build_wheel(args.recipe,
versions_combis = {"python": args.python or [],
"numpy": args.numpy or []},
conda_channel_urls=args.channel or (),
conda_override_channels=args.override_channels or (),
upload=args.upload_pypi or [],
wheel_dir=args.wheel_dir or "./build")
