def provide(recipe_dir, meta, patch=True):
"""
given a recipe_dir:
- download (if necessary)
- unpack
- apply patches (if any)
"""
print("Removing old work directory")
rm_rf(WORK_DIR)
if 'fn' in meta:
unpack(meta)
elif 'git_url' in meta:
git_source(meta, recipe_dir)
elif 'hg_url' in meta:
hg_source(meta)
elif 'svn_url' in meta:
svn_source(meta)
elif 'path' in meta:
print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR))
copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)
else: # no source
os.makedirs(WORK_DIR)
if patch:
src_dir = get_dir()
for patch in meta.get('patches', []):
apply_patch(src_dir, join(recipe_dir, patch))
def get_pkginfo(package, filename, pypiurl, md5, python_version, config, setup_options):
# Unfortunately, two important pieces of metadata are only stored in
# the package itself: the dependencies, and the entry points (if the
# package uses distribute). Our strategy is to download the package
# and "fake" distribute/setuptools's setup() function to get this
# information from setup.py. If this sounds evil, keep in mind that
# distribute itself already works by monkeypatching distutils.
tempdir = mkdtemp('conda_skeleton_' + filename)
if not isdir(config.src_cache):
makedirs(config.src_cache)
try:
# Download it to the build source cache. That way, you have
# it.
download_path = join(config.src_cache, filename)
if not isfile(download_path) or \
hashsum_file(download_path, 'md5') != md5:
download(pypiurl, join(config.src_cache, filename))
else:
print("Using cached download")
print("Unpacking %s..." % package)
unpack(join(config.src_cache, filename), tempdir)
print("done")
print("working in %s" % tempdir)
src_dir = get_dir(tempdir)
# TODO: find args parameters needed by run_setuppy
run_setuppy(src_dir, tempdir, python_version, config=config, setup_options=setup_options)
with open(join(tempdir, 'pkginfo.yaml')) as fn:
pkginfo = yaml.load(fn)
finally:
rm_rf(tempdir)
return pkginfo
def test(recipedir_or_package_or_metadata, move_broken=True, config=None, **kwargs):
import os
from conda_build.conda_interface import url_path
from conda_build.build import test
from conda_build.render import render_recipe
from conda_build.utils import get_recipe_abspath, rm_rf
from conda_build import source
config = get_or_merge_config(config, **kwargs)
# we want to know if we're dealing with package input. If so, we can move the input on success.
is_package = False
if hasattr(recipedir_or_package_or_metadata, 'config'):
metadata = recipedir_or_package_or_metadata
recipe_config = metadata.config
else:
recipe_dir, need_cleanup = get_recipe_abspath(recipedir_or_package_or_metadata)
config.need_cleanup = need_cleanup
# This will create a new local build folder if and only if config doesn't already have one.
# What this means is that if we're running a test immediately after build, we use the one
# that the build already provided
metadata, _, _ = render_recipe(recipe_dir, config=config)
recipe_config = config
# this recipe came from an extracted tarball.
if need_cleanup:
# ensure that the local location of the package is indexed, so that conda can find the
# local package
local_location = os.path.dirname(recipedir_or_package_or_metadata)
# strip off extra subdir folders
for platform in ('win', 'linux', 'osx'):
if os.path.basename(local_location).startswith(platform + "-"):
local_location = os.path.dirname(local_location)
update_index(local_location, config=config)
local_url = url_path(local_location)
# channel_urls is an iterable, but we don't know if it's a tuple or list. Don't know
# how to add elements.
recipe_config.channel_urls = list(recipe_config.channel_urls)
recipe_config.channel_urls.insert(0, local_url)
is_package = True
if metadata.meta.get('test') and metadata.meta['test'].get('source_files'):
source.provide(metadata.path, metadata.get_section('source'), config=config)
rm_rf(recipe_dir)
with recipe_config:
# This will create a new local build folder if and only if config doesn't already have one.
# What this means is that if we're running a test immediately after build, we use the one
# that the build already provided
recipe_config.compute_build_id(metadata.name())
test_result = test(metadata, config=recipe_config, move_broken=move_broken)
if (test_result and is_package and hasattr(recipe_config, 'output_folder') and
recipe_config.output_folder):
os.rename(recipedir_or_package_or_metadata,
os.path.join(recipe_config.output_folder,
os.path.basename(recipedir_or_package_or_metadata)))
return test_result
def execute(args):
parser, args = parse_args(args)
config = Config(**args.__dict__)
build.check_external()
# change globals in build module, see comment there as well
config.channel_urls = args.channel or ()
config.override_channels = args.override_channels
config.verbose = not args.quiet or args.debug
if 'purge' in args.recipe:
build.clean_build(config)
return
if 'purge-all' in args.recipe:
build.clean_build(config)
config.clean_pkgs()
return
if on_win:
delete_trash(None)
set_language_env_vars(args, parser, config=config, execute=execute)
action = None
if args.output:
action = output_action
logging.basicConfig(level=logging.ERROR)
config.verbose = False
config.quiet = True
elif args.test:
action = test_action
elif args.source:
action = source_action
elif args.check:
action = check_action
if action:
for recipe in args.recipe:
recipe_dir, need_cleanup = get_recipe_abspath(recipe)
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
# this fully renders any jinja templating, throwing an error if any data is missing
m, _, _ = render_recipe(recipe_dir, no_download_source=False, config=config)
action(m, config)
if need_cleanup:
rm_rf(recipe_dir)
else:
api.build(args.recipe, post=args.post, build_only=args.build_only,
notest=args.notest, keep_old_work=args.keep_old_work,
already_built=None, config=config)
if not args.output and len(build.get_build_folders(config.croot)) > 0:
build.print_build_intermediate_warning(config)
def run_setuppy(src_dir, temp_dir, python_version, config, setup_options):
"""
Patch distutils and then run setup.py in a subprocess.
:param src_dir: Directory containing the source code
:type src_dir: str
:param temp_dir: Temporary directory for doing for storing pkginfo.yaml
:type temp_dir: str
"""
specs = ["python %s*" % python_version, "pyyaml", "setuptools"]
with open(os.path.join(src_dir, "setup.py")) as setup:
text = setup.read()
if "import numpy" in text or "from numpy" in text:
specs.append("numpy")
# Do everything in the build env in case the setup.py install goes
# haywire.
# TODO: Try with another version of Python if this one fails. Some
# packages are Python 2 or Python 3 only.
create_env(config.build_prefix, specs=specs, clear_cache=False, config=config)
stdlib_dir = join(config.build_prefix, "Lib" if sys.platform == "win32" else "lib/python%s" % python_version)
patch = join(temp_dir, "pypi-distutils.patch")
with open(patch, "w") as f:
f.write(DISTUTILS_PATCH.format(temp_dir.replace("\\", "\\\\")))
if exists(join(stdlib_dir, "distutils", "core.py-copy")):
rm_rf(join(stdlib_dir, "distutils", "core.py"))
copy2(join(stdlib_dir, "distutils", "core.py-copy"), join(stdlib_dir, "distutils", "core.py"))
# Avoid race conditions. Invalidate the cache.
if PY3:
rm_rf(join(stdlib_dir, "distutils", "__pycache__", "core.cpython-%s%s.pyc" % sys.version_info[:2]))
rm_rf(join(stdlib_dir, "distutils", "__pycache__", "core.cpython-%s%s.pyo" % sys.version_info[:2]))
else:
rm_rf(join(stdlib_dir, "distutils", "core.pyc"))
rm_rf(join(stdlib_dir, "distutils", "core.pyo"))
else:
copy2(join(stdlib_dir, "distutils", "core.py"), join(stdlib_dir, "distutils", "core.py-copy"))
apply_patch(join(stdlib_dir, "distutils"), patch, config=config)
# Save PYTHONPATH for later
env = os.environ.copy()
if "PYTHONPATH" in env:
env[str("PYTHONPATH")] = str(src_dir + ":" + env["PYTHONPATH"])
else:
env[str("PYTHONPATH")] = str(src_dir)
cwd = getcwd()
chdir(src_dir)
cmdargs = [config.build_python, "setup.py", "install"]
cmdargs.extend(setup_options)
try:
subprocess.check_call(cmdargs, env=env)
except subprocess.CalledProcessError:
print("$PYTHONPATH = %s" % env["PYTHONPATH"])
sys.exit("Error: command failed: %s" % " ".join(cmdargs))
finally:
chdir(cwd)
def hoist_single_extracted_folder(nested_folder):
"""Moves all files/folders one level up.
This is for when your archive extracts into its own folder, so that we don't need to
know exactly what that folder is called."""
flist = os.listdir(nested_folder)
parent = os.path.dirname(nested_folder)
for thing in flist:
shutil.move(os.path.join(nested_folder, thing), os.path.join(parent, thing))
rm_rf(nested_folder)
def get_pkginfo(package, filename, pypiurl, digest, python_version, extra_specs, config,
setup_options):
# Unfortunately, two important pieces of metadata are only stored in
# the package itself: the dependencies, and the entry points (if the
# package uses distribute). Our strategy is to download the package
# and "fake" distribute/setuptools's setup() function to get this
# information from setup.py. If this sounds evil, keep in mind that
# distribute itself already works by monkeypatching distutils.
tempdir = mkdtemp('conda_skeleton_' + filename)
if not isdir(config.src_cache):
makedirs(config.src_cache)
hash_type = digest[0]
hash_value = digest[1]
try:
# Download it to the build source cache. That way, you have
# it.
download_path = join(config.src_cache, filename)
if not isfile(download_path) or \
hashsum_file(download_path, hash_type) != hash_value:
download(pypiurl, join(config.src_cache, filename))
if hashsum_file(download_path, hash_type) != hash_value:
raise RuntimeError(' Download of {} failed'
' checksum type {} expected value {}. Please'
' try again.'.format(package, hash_type, hash_value))
else:
print("Using cached download")
# Calculate the preferred hash type here if necessary.
# Needs to be done in this block because this is where we have
# access to the source file.
if hash_type != POSSIBLE_DIGESTS[0]:
new_hash_value = hashsum_file(download_path, POSSIBLE_DIGESTS[0])
else:
new_hash_value = ''
print("Unpacking %s..." % package)
unpack(join(config.src_cache, filename), tempdir)
print("done")
print("working in %s" % tempdir)
src_dir = get_dir(tempdir)
# TODO: find args parameters needed by run_setuppy
run_setuppy(src_dir, tempdir, python_version, extra_specs=extra_specs, config=config,
setup_options=setup_options)
try:
with open(join(tempdir, 'pkginfo.yaml')) as fn:
pkg_info = yaml.safe_load(fn)
except IOError:
pkg_info = pkginfo.SDist(download_path).__dict__
if new_hash_value:
pkg_info['new_hash_value'] = (POSSIBLE_DIGESTS[0], new_hash_value)
finally:
rm_rf(tempdir)
return pkg_info
def hoist_single_extracted_folder(nested_folder):
"""Moves all files/folders one level up.
This is for when your archive extracts into its own folder, so that we don't need to
know exactly what that folder is called."""
parent = os.path.dirname(nested_folder)
flist = os.listdir(nested_folder)
with TemporaryDirectory() as tmpdir:
for entry in flist:
shutil.move(os.path.join(nested_folder, entry), os.path.join(tmpdir, entry))
rm_rf(nested_folder)
for entry in flist:
shutil.move(os.path.join(tmpdir, entry), os.path.join(parent, entry))
def check_install(packages, platform=None, channel_urls=(), prepend=True,
minimal_hint=False):
prefix = tempfile.mkdtemp('conda')
try:
specs = specs_from_args(packages)
index = get_index(channel_urls=channel_urls, prepend=prepend,
platform=platform, prefix=prefix)
actions = install_actions(prefix, index, specs, pinned=False,
minimal_hint=minimal_hint)
display_actions(actions, index)
return actions
finally:
rm_rf(prefix)
return None
def build(m, get_src=True, pypi=False):
rm_rf(prefix)
create_env(prefix, [ms.spec for ms in m.ms_depends('build')], pypi)
print("BUILD START:", m.dist())
if get_src:
source.provide(m.path, m.get_section('source'))
assert isdir(source.WORK_DIR)
if os.listdir(source.get_dir()):
print("source tree in:", source.get_dir())
else:
print("no source")
rm_rf(info_dir)
files1 = prefix_files()
if sys.platform == 'win32':
import conda_build.windows as windows
windows.build(m)
else:
env = environ.get_dict(m)
cmd = ['/bin/bash', '-x', '-e', join(m.path, 'build.sh')]
_check_call(cmd, env=env, cwd=source.get_dir())
create_post_scripts(m)
create_entry_points(m.get_value('build/entry_points'))
post_process(preserve_egg_dir=bool(
m.get_value('build/preserve_egg_dir')))
assert not exists(info_dir)
files2 = prefix_files()
post_build(sorted(files2 - files1))
create_info_files(m, sorted(files2 - files1))
files3 = prefix_files()
fix_permissions(files3 - files1)
path = bldpkg_path(m)
t = tarfile.open(path, 'w:bz2')
for f in sorted(files3 - files1):
t.add(join(prefix, f), f)
t.close()
print("BUILD END:", m.dist())
# we're done building, perform some checks
tarcheck.check_all(path)
update_index(bldpkgs_dir)
def make_hardlink_copy(path, prefix):
"""Hardlinks create invalid packages. Copy files to break the link.
Symlinks are OK, and unaffected here."""
if not os.path.isabs(path):
path = os.path.normpath(os.path.join(prefix, path))
fn = os.path.basename(path)
if os.lstat(path).st_nlink > 1:
with TemporaryDirectory() as dest:
# copy file to new name
utils.copy_into(path, dest)
# remove old file
utils.rm_rf(path)
# rename copy to original filename
# It is essential here to use copying (as opposed to os.rename), so that
# crossing volume boundaries works
utils.copy_into(os.path.join(dest, fn), path)
def render_recipe(recipe_path, config, no_download_source=False):
arg = recipe_path
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
elif arg.endswith('.yaml'):
recipe_dir = os.path.dirname(arg)
need_cleanup = False
else:
print("Ignoring non-recipe: %s" % arg)
return
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
if config.set_build_id:
# updates a unique build id if not already computed
config.compute_build_id(os.path.basename(recipe_dir))
try:
m = MetaData(recipe_dir, config=config)
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
m, need_download, need_reparse_in_env = parse_or_try_download(m,
no_download_source=no_download_source,
config=config)
if need_download and no_download_source:
raise ValueError("no_download_source specified, but can't fully render recipe without"
" downloading source. Please fix the recipe, or don't use "
"no_download_source.")
config.noarch = bool(m.get_value('build/noarch'))
if need_cleanup:
rm_rf(recipe_dir)
return m, need_download, need_reparse_in_env
def make_hardlink_copy(path, prefix):
"""Hardlinks create invalid packages. Copy files to break the link.
Symlinks are OK, and unaffected here."""
if not os.path.isabs(path) and not os.path.exists(path):
path = os.path.normpath(os.path.join(prefix, path))
nlinks = os.lstat(path).st_nlink
dest = 'tmpfile'
if os.path.isabs(path):
dest = os.path.join(os.getcwd(), dest)
if nlinks > 1:
# copy file to new name
utils.copy_into(path, dest)
# remove old file
utils.rm_rf(path)
# rename copy to original filename
utils.copy_into(dest, path)
utils.rm_rf(dest)
def remove_easy_install_pth(files, prefix, config, preserve_egg_dir=False):
"""
remove the need for easy-install.pth and finally remove easy-install.pth
itself
"""
absfiles = [os.path.join(prefix, f) for f in files]
py_ver = '.'.join(config.variant['python'].split('.')[:2])
sp_dir = utils.get_site_packages(prefix, py_ver)
for egg_path in glob(os.path.join(sp_dir, '*-py*.egg')):
if os.path.isdir(egg_path):
if preserve_egg_dir or not any(os.path.join(egg_path, i) in absfiles for i
in walk_prefix(egg_path, False, windows_forward_slashes=False)):
write_pth(egg_path, config=config)
continue
print('found egg dir:', egg_path)
try:
os.rename(os.path.join(egg_path, 'EGG-INFO'),
egg_path + '-info')
except OSError:
pass
utils.rm_rf(os.path.join(egg_path, 'EGG-INFO'))
for fn in os.listdir(egg_path):
if fn == '__pycache__':
utils.rm_rf(os.path.join(egg_path, fn))
else:
# this might be a name-space package
# so the package directory already exists
# from another installed dependency
if os.path.exists(os.path.join(sp_dir, fn)):
try:
utils.copy_into(os.path.join(egg_path, fn),
os.path.join(sp_dir, fn), config.timeout,
locking=config.locking)
utils.rm_rf(os.path.join(egg_path, fn))
except IOError as e:
fn = os.path.basename(str(e).split()[-1])
raise IOError("Tried to merge folder {egg_path} into {sp_dir}, but {fn}"
" exists in both locations. Please either add "
"build/preserve_egg_dir: True to meta.yaml, or manually "
"remove the file during your install process to avoid "
"this conflict."
.format(egg_path=egg_path, sp_dir=sp_dir, fn=fn))
else:
os.rename(os.path.join(egg_path, fn), os.path.join(sp_dir, fn))
elif os.path.isfile(egg_path):
if egg_path not in absfiles:
continue
print('found egg:', egg_path)
write_pth(egg_path, config=config)
utils.rm_rf(os.path.join(sp_dir, 'easy-install.pth'))
def clean_pkg_cache(dist, timeout):
cc.pkgs_dirs = cc.pkgs_dirs[:1]
locks = []
for folder in cc.pkgs_dirs:
locks.append(filelock.SoftFileLock(join(folder, ".conda_lock")))
for lock in locks:
lock.acquire(timeout=timeout)
try:
rmplan = [
'RM_EXTRACTED {0} local::{0}'.format(dist),
'RM_FETCHED {0} local::{0}'.format(dist),
]
plan.execute_plan(rmplan)
# Conda does not seem to do a complete cleanup sometimes. This is supplemental.
# Conda's cleanup is still necessary - it keeps track of its own in-memory
# list of downloaded things.
for folder in cc.pkgs_dirs:
try:
assert not os.path.exists(os.path.join(folder, dist))
assert not os.path.exists(os.path.join(folder, dist + '.tar.bz2'))
for pkg_id in [dist, 'local::' + dist]:
assert pkg_id not in package_cache()
except AssertionError:
log.debug("Conda caching error: %s package remains in cache after removal", dist)
log.debug("Clearing package cache to compensate")
cache = package_cache()
keys = [key for key in cache.keys() if dist in key]
for pkg_id in keys:
if pkg_id in cache:
del cache[pkg_id]
for entry in glob(os.path.join(folder, dist + '*')):
rm_rf(entry)
except:
raise
finally:
for lock in locks:
lock.release()
if os.path.isfile(lock._lock_file):
os.remove(lock._lock_file)
def create_all_test_files(m, test_dir=None):
if test_dir:
rm_rf(test_dir)
os.makedirs(test_dir)
# this happens when we're finishing the build.
test_deps = m.meta.get('test', {}).get('requires', [])
if test_deps:
with open(os.path.join(test_dir, 'test_time_dependencies.json'), 'w') as f:
json.dump(test_deps, f)
else:
# this happens when we're running a package's tests
test_dir = m.config.test_dir
files = create_files(m, test_dir)
pl_files = create_pl_files(m, test_dir)
py_files = create_py_files(m, test_dir)
r_files = create_r_files(m, test_dir)
lua_files = create_lua_files(m, test_dir)
shell_files = create_shell_files(m, test_dir)
return files, pl_files, py_files, r_files, lua_files, shell_files
def sp_dir(request):
'''
create site-packges/ directory in same place where test is located. This
is where tests look conda.pth file. It is a session scoped fixture and
it has a finalizer function invoked in the end to remove site-packages/
directory
'''
base_dir = dirname(__file__)
sp = join(base_dir, 'site-packages')
if exists(sp):
rm_rf(sp)
os.mkdir(sp)
def cleanup():
# session scoped cleanup is called at end of the session
rm_rf(sp)
request.addfinalizer(cleanup)
return sp
def test_env_creation_with_short_prefix_does_not_deadlock(caplog):
test_base = os.path.expanduser("~/cbtmp")
config = api.Config(croot=test_base, anaconda_upload=False, verbose=True)
recipe_path = os.path.join(metadata_dir, "has_prefix_files")
metadata, _, _ = api.render(recipe_path, config=config)
metadata.meta['package']['name'] = 'test_env_creation_with_short_prefix'
fn = api.get_output_file_path(metadata)
if os.path.isfile(fn):
os.remove(fn)
config.prefix_length = 80
try:
api.build(metadata)
pkg_name = os.path.basename(fn).replace("-1.0-0.tar.bz2", "")
assert not api.inspect_prefix_length(fn, 255)
config.prefix_length = 255
build.create_env(config.build_prefix, specs=["python", pkg_name], config=config)
except:
raise
finally:
rm_rf(test_base)
assert 'One or more of your package dependencies needs to be rebuilt' in caplog.text()
def remove_easy_install_pth(preserve_egg_dir=False):
"""
remove the need for easy-install.pth and finally remove easy-install.pth
itself
"""
sp_dir = environ.SP_DIR
for egg_path in glob(join(sp_dir, "*-py*.egg")):
if isdir(egg_path):
if preserve_egg_dir:
write_pth(egg_path)
continue
print("found egg dir:", egg_path)
try:
os.rename(join(egg_path, "EGG-INFO/PKG-INFO"), egg_path + "-info")
except OSError:
pass
utils.rm_rf(join(egg_path, "EGG-INFO"))
for fn in os.listdir(egg_path):
if fn == "__pycache__":
utils.rm_rf(join(egg_path, fn))
else:
os.rename(join(egg_path, fn), join(sp_dir, fn))
elif isfile(egg_path):
print("found egg:", egg_path)
write_pth(egg_path)
utils.rm_rf(join(sp_dir, "easy-install.pth"))
def remove_easy_install_pth(preserve_egg_dir=False):
"""
remove the need for easy-install.pth and finally remove easy-install.pth
itself
"""
sp_dir = environ.get_sp_dir()
for egg_path in glob(join(sp_dir, '*-py*.egg')):
if isdir(egg_path):
if preserve_egg_dir:
write_pth(egg_path)
continue
print('found egg dir:', egg_path)
try:
os.rename(join(egg_path, 'EGG-INFO/PKG-INFO'),
egg_path + '-info')
except OSError:
pass
utils.rm_rf(join(egg_path, 'EGG-INFO'))
for fn in os.listdir(egg_path):
if fn == '__pycache__':
utils.rm_rf(join(egg_path, fn))
else:
os.rename(join(egg_path, fn), join(sp_dir, fn))
elif isfile(egg_path):
print('found egg:', egg_path)
write_pth(egg_path)
utils.rm_rf(join(sp_dir, 'easy-install.pth'))
def clean_pkg_cache(dist, config):
locks = []
conda_log_level = logging.WARN
if config.debug:
conda_log_level = logging.DEBUG
_pkgs_dirs = pkgs_dirs[:1]
if config.locking:
locks = [utils.get_lock(folder, timeout=config.timeout) for folder in _pkgs_dirs]
with utils.LoggingContext(conda_log_level):
with utils.try_acquire_locks(locks, timeout=config.timeout):
rmplan = [
'RM_EXTRACTED {0} local::{0}'.format(dist),
'RM_FETCHED {0} local::{0}'.format(dist),
]
execute_plan(rmplan)
# Conda does not seem to do a complete cleanup sometimes. This is supplemental.
# Conda's cleanup is still necessary - it keeps track of its own in-memory
# list of downloaded things.
for folder in pkgs_dirs:
try:
assert not os.path.exists(os.path.join(folder, dist))
assert not os.path.exists(os.path.join(folder, dist + '.tar.bz2'))
for pkg_id in [dist, 'local::' + dist]:
assert pkg_id not in package_cache()
except AssertionError:
log = utils.get_logger(__name__)
log.debug("Conda caching error: %s package remains in cache after removal",
dist)
log.debug("manually removing to compensate")
cache = package_cache()
keys = [key for key in cache.keys() if dist in key]
for pkg_id in keys:
if pkg_id in cache:
del cache[pkg_id]
for entry in glob(os.path.join(folder, dist + '*')):
utils.rm_rf(entry)
def create_files(m):
"""
Create the test files for pkg in the directory given. The resulting
test files are configuration (i.e. platform, architecture, Python and
numpy version, ...) independent.
Return False, if the package has no tests (for any configuration), and
True if it has.
"""
has_files = False
rm_rf(m.config.test_dir)
for fn in ensure_list(m.get_value('test/files', [])):
has_files = True
path = join(m.path, fn)
# disable locking to avoid locking a temporary directory (the extracted test folder)
copy_into(path, join(m.config.test_dir, fn), m.config.timeout, locking=False,
clobber=True)
# need to re-download source in order to do tests
if m.get_value('test/source_files') and not isdir(m.config.work_dir):
source.provide(m)
for pattern in ensure_list(m.get_value('test/source_files', [])):
if on_win and '\\' in pattern:
raise RuntimeError("test/source_files paths must use / "
"as the path delimiter on Windows")
has_files = True
files = glob.glob(join(m.config.work_dir, pattern))
if not files:
raise RuntimeError("Did not find any source_files for test with pattern %s", pattern)
for f in files:
try:
# disable locking to avoid locking a temporary directory (the extracted test folder)
copy_into(f, f.replace(m.config.work_dir, m.config.test_dir), m.config.timeout,
locking=False)
except OSError as e:
log = logging.getLogger(__name__)
log.warn("Failed to copy {0} into test files. Error was: {1}".format(f, str(e)))
for ext in '.pyc', '.pyo':
for f in get_ext_files(m.config.test_dir, ext):
os.remove(f)
return has_files
def provide(recipe_dir, meta, patch=True):
"""
given a recipe_dir:
- download (if necessary)
- unpack
- apply patches (if any)
"""
rm_rf(WORK_DIR)
if "fn" in meta:
unpack(meta)
elif "git_url" in meta:
git_source(meta)
elif "hg_url" in meta:
hg_source(meta)
elif "svn_url" in meta:
svn_source(meta)
else: # no source
os.makedirs(WORK_DIR)
if patch:
src_dir = get_dir()
for patch in meta.get("patches", []):
apply_patch(src_dir, join(recipe_dir, patch))
def get_pkginfo(package, filename, pypiurl, md5, python_version, config, setup_options):
# Unfortunately, two important pieces of metadata are only stored in
# the package itself: the dependencies, and the entry points (if the
# package uses distribute). Our strategy is to download the package
# and "fake" distribute/setuptools's setup() function to get this
# information from setup.py. If this sounds evil, keep in mind that
# distribute itself already works by monkeypatching distutils.
tempdir = mkdtemp('conda_skeleton_' + filename)
if not isdir(config.src_cache):
makedirs(config.src_cache)
try:
# Download it to the build source cache. That way, you have
# it.
download_path = join(config.src_cache, filename)
if not isfile(download_path) or \
hashsum_file(download_path, 'md5') != md5:
download(pypiurl, join(config.src_cache, filename))
else:
print("Using cached download")
print("Unpacking %s..." % package)
unpack(join(config.src_cache, filename), tempdir)
print("done")
print("working in %s" % tempdir)
src_dir = get_dir(tempdir)
# TODO: find args parameters needed by run_setuppy
run_setuppy(src_dir, tempdir, python_version, config=config, setup_options=setup_options)
try:
with open(join(tempdir, 'pkginfo.yaml')) as fn:
pkg_info = yaml.load(fn)
except IOError:
pkg_info = pkginfo.SDist(download_path).__dict__
finally:
rm_rf(tempdir)
return pkg_info
def provide(recipe_dir, meta, verbose=False, patch=True, dirty=False):
"""
given a recipe_dir:
- download (if necessary)
- unpack
- apply patches (if any)
"""
if not dirty:
if sys.platform == 'win32':
if isdir(WORK_DIR):
move_to_trash(WORK_DIR, '')
else:
rm_rf(WORK_DIR)
if not os.path.exists(WORK_DIR):
if any(k in meta for k in ('fn', 'url')):
unpack(meta, verbose=verbose)
elif 'git_url' in meta:
git_source(meta, recipe_dir, verbose=verbose)
# build to make sure we have a work directory with source in it. We want to make sure that
# whatever version that is does not interfere with the test we run next.
elif 'hg_url' in meta:
hg_source(meta, verbose=verbose)
elif 'svn_url' in meta:
svn_source(meta, verbose=verbose)
elif 'path' in meta:
if verbose:
print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR))
copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)
else: # no source
os.makedirs(WORK_DIR)
if patch:
src_dir = get_dir()
for patch in meta.get('patches', []):
apply_patch(src_dir, join(recipe_dir, patch))
def provide(recipe_dir, meta, verbose=False, patch=True):
"""
given a recipe_dir:
- download (if necessary)
- unpack
- apply patches (if any)
"""
if sys.platform == 'win32':
if isdir(WORK_DIR):
move_to_trash(WORK_DIR, '')
else:
rm_rf(WORK_DIR)
if any(k in meta for k in ('fn', 'url')):
unpack(meta, verbose=verbose)
elif 'git_url' in meta:
git_source(meta, recipe_dir, verbose=verbose)
# build to make sure we have a work directory with source in it. We want to make sure that
# whatever version that is does not interfere with the test we run next.
elif 'hg_url' in meta:
hg_source(meta, verbose=verbose)
elif 'svn_url' in meta:
svn_source(meta, verbose=verbose)
elif 'path' in meta:
if verbose:
print("Copying %s to %s" %
(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR))
copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)
else: # no source
if not isdir(WORK_DIR):
os.makedirs(WORK_DIR)
if patch:
src_dir = get_dir()
for patch in meta.get('patches', []):
apply_patch(src_dir, join(recipe_dir, patch))
def test_env_creation_with_short_prefix_does_not_deadlock(caplog):
test_base = os.path.expanduser("~/cbtmp")
config = api.Config(croot=test_base, anaconda_upload=False, verbose=True)
recipe_path = os.path.join(metadata_dir, "has_prefix_files")
metadata, _, _ = api.render(recipe_path, config=config)
metadata.meta['package']['name'] = 'test_env_creation_with_short_prefix'
fn = api.get_output_file_path(metadata)
if os.path.isfile(fn):
os.remove(fn)
config.prefix_length = 80
try:
api.build(metadata)
pkg_name = os.path.basename(fn).replace("-1.0-0.tar.bz2", "")
assert not api.inspect_prefix_length(fn, 255)
config.prefix_length = 255
build.create_env(config.build_prefix,
specs=["python", pkg_name],
config=config)
except:
raise
finally:
rm_rf(test_base)
assert 'One or more of your package dependencies needs to be rebuilt' in caplog.text(
)
def hoist_single_extracted_folder(nested_folder):
"""Moves all files/folders one level up.
This is for when your archive extracts into its own folder, so that we don't need to
know exactly what that folder is called."""
flist = os.listdir(nested_folder)
parent = os.path.dirname(nested_folder)
# only hoist if the parent folder contains ONLY our nested folder
nested_folders_to_remove = [nested_folder]
for thing in flist:
if not os.path.isdir(os.path.join(parent, thing)):
shutil.move(os.path.join(nested_folder, thing),
os.path.join(parent, thing))
else:
copy_into(os.path.join(nested_folder, thing),
os.path.join(parent, thing))
nested_folders_to_remove.append(os.path.join(nested_folder, thing))
# handle nested similar folder names
fn = os.path.basename(nested_folder)
if (os.path.join(nested_folder, fn) in nested_folders_to_remove
and nested_folder in nested_folders_to_remove):
nested_folders_to_remove.remove(nested_folder)
for folder in nested_folders_to_remove:
rm_rf(folder)
def remove_easy_install_pth(files, preserve_egg_dir=False):
"""
remove the need for easy-install.pth and finally remove easy-install.pth
itself
"""
absfiles = [join(config.build_prefix, f) for f in files]
sp_dir = environ.get_sp_dir()
for egg_path in glob(join(sp_dir, '*-py*.egg')):
if isdir(egg_path):
if preserve_egg_dir or not any(
join(egg_path, i) in absfiles
for i in walk_prefix(egg_path, False)):
write_pth(egg_path)
continue
print('found egg dir:', egg_path)
try:
os.rename(join(egg_path, 'EGG-INFO/PKG-INFO'),
egg_path + '-info')
except OSError:
pass
utils.rm_rf(join(egg_path, 'EGG-INFO'))
for fn in os.listdir(egg_path):
if fn == '__pycache__':
utils.rm_rf(join(egg_path, fn))
else:
# this might be a name-space package
# so the package directory already exists
# from another installed dependency
if os.path.exists(join(sp_dir, fn)):
utils.copy_into(join(egg_path, fn), join(sp_dir, fn))
utils.rm_rf(join(egg_path, fn))
else:
os.rename(join(egg_path, fn), join(sp_dir, fn))
elif isfile(egg_path):
if not egg_path in absfiles:
continue
print('found egg:', egg_path)
write_pth(egg_path)
utils.rm_rf(join(sp_dir, 'easy-install.pth'))
def remove_easy_install_pth(files, preserve_egg_dir=False):
"""
remove the need for easy-install.pth and finally remove easy-install.pth
itself
"""
absfiles = [join(config.build_prefix, f) for f in files]
sp_dir = environ.get_sp_dir()
for egg_path in glob(join(sp_dir, '*-py*.egg')):
if isdir(egg_path):
if preserve_egg_dir or not any(join(egg_path, i) in absfiles for i
in walk_prefix(egg_path, False, windows_forward_slashes=False)):
write_pth(egg_path)
continue
print('found egg dir:', egg_path)
try:
os.rename(join(egg_path, 'EGG-INFO'),
egg_path + '-info')
except OSError:
pass
utils.rm_rf(join(egg_path, 'EGG-INFO'))
for fn in os.listdir(egg_path):
if fn == '__pycache__':
utils.rm_rf(join(egg_path, fn))
else:
# this might be a name-space package
# so the package directory already exists
# from another installed dependency
if os.path.exists(join(sp_dir, fn)):
utils.copy_into(join(egg_path, fn), join(sp_dir, fn))
utils.rm_rf(join(egg_path, fn))
else:
os.rename(join(egg_path, fn), join(sp_dir, fn))
elif isfile(egg_path):
if not egg_path in absfiles:
continue
print('found egg:', egg_path)
write_pth(egg_path)
utils.rm_rf(join(sp_dir, 'easy-install.pth'))
def remove_easy_install_pth(preserve_egg_dir=False):
"""
remove the need for easy-install.pth and finally remove easy-install.pth
itself
"""
sp_dir = environ.get_sp_dir()
for egg_path in glob(join(sp_dir, '*-py*.egg')):
if isdir(egg_path):
if preserve_egg_dir:
write_pth(egg_path)
continue
print('found egg dir:', egg_path)
try:
os.rename(join(egg_path, 'EGG-INFO/PKG-INFO'),
egg_path + '-info')
except OSError:
pass
utils.rm_rf(join(egg_path, 'EGG-INFO'))
for fn in os.listdir(egg_path):
if fn == '__pycache__':
utils.rm_rf(join(egg_path, fn))
else:
# this might be a name-space package
# so the package directory already exists
# from another installed dependency
if os.path.exists(join(sp_dir, fn)):
utils.copy_into(join(egg_path, fn), join(sp_dir, fn))
utils.rm_rf(join(egg_path, fn))
else:
os.rename(join(egg_path, fn), join(sp_dir, fn))
elif isfile(egg_path):
print('found egg:', egg_path)
write_pth(egg_path)
utils.rm_rf(join(sp_dir, 'easy-install.pth'))
def test(m, pypi=False):
# remove from package cache
rm_pkgs_cache(m.dist())
tmp_dir = join(config.croot, 'test-tmp_dir')
rm_rf(tmp_dir)
os.makedirs(tmp_dir)
if not create_files(tmp_dir, m):
print("Nothing to test for:", m.dist())
return
print("TEST START:", m.dist())
rm_rf(prefix)
rm_rf(config.test_prefix)
specs = ['%s %s %s' % (m.name(), m.version(), m.build_id()),
# as the tests are run by python, we need to specify it
'python %s*' % environ.py_ver]
# add packages listed in test/requires
for spec in m.get_value('test/requires'):
specs.append(spec)
create_env(config.test_prefix, specs, pypi)
env = dict(os.environ)
# prepend bin (or Scripts) directory
env['PATH'] = (join(config.test_prefix, bin_dirname) + os.pathsep +
env['PATH'])
for varname in 'CONDA_PY', 'CONDA_NPY':
env[varname] = str(getattr(config, varname))
env['PREFIX'] = config.test_prefix
try:
subprocess.check_call([config.test_python, join(tmp_dir, 'run_test.py')],
env=env, cwd=tmp_dir)
except subprocess.CalledProcessError:
if not isdir(broken_dir):
os.makedirs(broken_dir)
shutil.move(bldpkg_path(m), join(broken_dir, "%s.tar.bz2" % m.dist()))
sys.exit("TESTS FAILED: " + m.dist())
print("TEST END:", m.dist())
def get_install_actions(prefix,
specs,
env,
retries=0,
subdir=None,
verbose=True,
debug=False,
locking=True,
bldpkgs_dirs=None,
timeout=900,
disable_pip=False,
max_env_retry=3,
output_folder=None,
channel_urls=None):
global cached_actions
global last_index_ts
actions = {}
log = utils.get_logger(__name__)
conda_log_level = logging.WARN
specs = list(specs)
if specs:
specs.extend(create_default_packages)
if verbose or debug:
capture = contextlib.contextmanager(lambda: (yield))
if debug:
conda_log_level = logging.DEBUG
else:
capture = utils.capture
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
bldpkgs_dirs = ensure_list(bldpkgs_dirs)
index, index_ts, _ = get_build_index(subdir,
list(bldpkgs_dirs)[0],
output_folder=output_folder,
channel_urls=channel_urls,
debug=debug,
verbose=verbose,
locking=locking,
timeout=timeout)
specs = tuple(
utils.ensure_valid_spec(spec) for spec in specs
if not str(spec).endswith('@'))
if ((specs, env, subdir, channel_urls, disable_pip) in cached_actions
and last_index_ts >= index_ts):
actions = cached_actions[(specs, env, subdir, channel_urls,
disable_pip)].copy()
if "PREFIX" in actions:
actions['PREFIX'] = prefix
elif specs:
# this is hiding output like:
# Fetching package metadata ...........
# Solving package specifications: ..........
with utils.LoggingContext(conda_log_level):
with capture():
try:
actions = install_actions(prefix, index, specs, force=True)
except (NoPackagesFoundError, UnsatisfiableError) as exc:
raise DependencyNeedsBuildingError(exc, subdir=subdir)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError,
AssertionError, BuildLockError) as exc:
if 'lock' in str(exc):
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)
or isinstance(exc, AssertionError)):
locks = utils.get_conda_operation_locks(
locking, bldpkgs_dirs, timeout)
with utils.try_acquire_locks(locks, timeout=timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if pkg_dir in pkgs_dirs and os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retries < max_env_retry:
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
actions = get_install_actions(
prefix,
tuple(specs),
env,
retries=retries + 1,
subdir=subdir,
verbose=verbose,
debug=debug,
locking=locking,
bldpkgs_dirs=tuple(bldpkgs_dirs),
timeout=timeout,
disable_pip=disable_pip,
max_env_retry=max_env_retry,
output_folder=output_folder,
channel_urls=tuple(channel_urls))
else:
log.error(
"Failed to get install actions, max retries exceeded."
)
raise
if disable_pip:
for pkg in ('pip', 'setuptools', 'wheel'):
# specs are the raw specifications, not the conda-derived actual specs
# We're testing that pip etc. are manually specified
if not any(
re.match(r'^%s(?:$|[\s=].*)' % pkg, str(dep))
for dep in specs):
actions['LINK'] = [
spec for spec in actions['LINK'] if spec.name != pkg
]
utils.trim_empty_keys(actions)
cached_actions[(specs, env, subdir, channel_urls,
disable_pip)] = actions.copy()
last_index_ts = index_ts
return actions
def create_env(prefix,
specs,
config,
subdir,
clear_cache=True,
retry=0,
index=None,
locks=None):
'''
Create a conda envrionment for the given prefix and specs.
'''
if config.debug:
utils.get_logger("conda_build").setLevel(logging.DEBUG)
external_logger_context = utils.LoggingContext(logging.DEBUG)
else:
utils.get_logger("conda_build").setLevel(logging.INFO)
external_logger_context = utils.LoggingContext(logging.ERROR)
with external_logger_context:
log = utils.get_logger(__name__)
if os.path.isdir(prefix):
utils.rm_rf(prefix)
specs = list(set(specs))
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
if specs: # Don't waste time if there is nothing to do
log.debug("Creating environment in %s", prefix)
log.debug(str(specs))
with utils.path_prepended(prefix):
if not locks:
locks = utils.get_conda_operation_locks(config)
try:
with utils.try_acquire_locks(locks,
timeout=config.timeout):
if not index:
index = get_build_index(config=config,
subdir=subdir)
actions = get_install_actions(prefix, index, specs,
config)
plan.display_actions(actions, index)
if utils.on_win:
for k, v in os.environ.items():
os.environ[k] = str(v)
plan.execute_actions(actions,
index,
verbose=config.debug)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError) as exc:
if (("too short in" in str(exc) or re.search(
'post-link failed for: (?:[a-zA-Z]*::)?openssl',
str(exc)) or isinstance(exc, PaddingError))
and config.prefix_length > 80):
if config.prefix_length_fallback:
log.warn(
"Build prefix failed with prefix length %d",
config.prefix_length)
log.warn("Error was: ")
log.warn(str(exc))
log.warn(
"One or more of your package dependencies needs to be rebuilt "
"with a longer prefix length.")
log.warn(
"Falling back to legacy prefix length of 80 characters."
)
log.warn(
"Your package will not install into prefixes > 80 characters."
)
config.prefix_length = 80
# Set this here and use to create environ
# Setting this here is important because we use it below (symlink)
prefix = config.build_prefix
create_env(prefix,
specs,
config=config,
subdir=subdir,
clear_cache=clear_cache)
else:
raise
elif 'lock' in str(exc):
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs,
config=config,
subdir=subdir,
clear_cache=clear_cache,
retry=retry + 1)
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs,
config=config,
subdir=subdir,
clear_cache=clear_cache,
retry=retry + 1)
else:
log.error(
"Failed to create env, max retries exceeded.")
raise
else:
raise
# HACK: some of the time, conda screws up somehow and incomplete packages result.
# Just retry.
except (AssertionError, IOError, ValueError, RuntimeError,
LockError) as exc:
if isinstance(exc, AssertionError):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = os.path.dirname(os.path.dirname(
str(exc)))
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs,
config=config,
subdir=subdir,
clear_cache=clear_cache,
retry=retry + 1)
else:
log.error(
"Failed to create env, max retries exceeded.")
raise
if utils.on_win:
shell = "cmd.exe"
else:
shell = "bash"
symlink_conda(prefix, sys.prefix, shell)
def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False):
''' Download a source to the local cache. '''
if verbose:
log.info('Source cache directory is: %s' % cache_folder)
if not isdir(cache_folder) and not os.path.islink(cache_folder):
os.makedirs(cache_folder)
source_urls = source_dict['url']
if not isinstance(source_urls, list):
source_urls = [source_urls]
unhashed_fn = fn = source_dict['fn'] if 'fn' in source_dict else basename(
source_urls[0])
hash_added = False
for hash_type in ('md5', 'sha1', 'sha256'):
if hash_type in source_dict:
if source_dict[hash_type] in (None, ""):
raise ValueError('Empty {} hash provided for {}'.format(
hash_type, fn))
fn = append_hash_to_fn(fn, source_dict[hash_type])
hash_added = True
break
else:
log.warn(
"No hash (md5, sha1, sha256) provided for {}. Source download forced. "
"Add hash to recipe to use source cache.".format(unhashed_fn))
path = join(cache_folder, fn)
if isfile(path):
if verbose:
log.info('Found source in cache: %s' % fn)
else:
if verbose:
log.info('Downloading source to cache: %s' % fn)
for url in source_urls:
if "://" not in url:
if url.startswith('~'):
url = expanduser(url)
if not os.path.isabs(url):
url = os.path.normpath(os.path.join(recipe_path, url))
url = url_path(url)
else:
if url.startswith('file:///~'):
url = 'file:///' + expanduser(url[8:]).replace('\\', '/')
try:
if verbose:
log.info("Downloading %s" % url)
with LoggingContext():
download(url, path)
except CondaHTTPError as e:
log.warn("Error: %s" % str(e).strip())
rm_rf(path)
except RuntimeError as e:
log.warn("Error: %s" % str(e).strip())
rm_rf(path)
else:
if verbose:
log.info("Success")
break
else: # no break
rm_rf(path)
raise RuntimeError("Could not download %s" % url)
hashed = None
for tp in ('md5', 'sha1', 'sha256'):
if tp in source_dict:
expected_hash = source_dict[tp]
hashed = hashsum_file(path, tp)
if expected_hash != hashed:
rm_rf(path)
raise RuntimeError("%s mismatch: '%s' != '%s'" %
(tp.upper(), hashed, expected_hash))
break
# this is really a fallback. If people don't provide the hash, we still need to prevent
# collisions in our source cache, but the end user will get no benefirt from the cache.
if not hash_added:
if not hashed:
hashed = hashsum_file(path, 'sha256')
dest_path = append_hash_to_fn(path, hashed)
if not os.path.isfile(dest_path):
shutil.move(path, dest_path)
path = dest_path
return path, unhashed_fn
def build(m,
get_src=True,
verbose=True,
post=None,
channel_urls=(),
override_channels=False,
include_recipe=True):
'''
Build the package with the specified metadata.
:param m: Package metadata
:type m: Metadata
:param get_src: Should we download the source?
:type get_src: bool
:type post: bool or None. None means run the whole build. True means run
post only. False means stop just before the post.
'''
if (m.get_value('build/detect_binary_files_with_prefix')
or m.binary_has_prefix_files()):
# We must use a long prefix here as the package will only be
# installable into prefixes shorter than this one.
config.use_long_build_prefix = True
else:
# In case there are multiple builds in the same process
config.use_long_build_prefix = False
if post in [False, None]:
print("Removing old build environment")
if on_win:
if isdir(config.short_build_prefix):
move_to_trash(config.short_build_prefix, '')
if isdir(config.long_build_prefix):
move_to_trash(config.long_build_prefix, '')
else:
rm_rf(config.short_build_prefix)
rm_rf(config.long_build_prefix)
print("Removing old work directory")
if on_win:
if isdir(source.WORK_DIR):
move_to_trash(source.WORK_DIR, '')
else:
rm_rf(source.WORK_DIR)
# Display the name only
# Version number could be missing due to dependency on source info.
print("BUILD START:", m.dist())
create_env(config.build_prefix,
[ms.spec for ms in m.ms_depends('build')],
verbose=verbose,
channel_urls=channel_urls,
override_channels=override_channels)
if m.name() in [
i.rsplit('-', 2)[0] for i in linked(config.build_prefix)
]:
print("%s is installed as a build dependency. Removing." %
m.name())
index = get_build_index(clear_cache=False,
channel_urls=channel_urls,
override_channels=override_channels)
actions = plan.remove_actions(config.build_prefix, [m.name()],
index=index)
assert not plan.nothing_to_do(actions), actions
plan.display_actions(actions, index)
plan.execute_actions(actions, index)
if get_src:
source.provide(m.path, m.get_section('source'))
# Parse our metadata again because we did not initialize the source
# information before.
m.parse_again()
print("Package:", m.dist())
assert isdir(source.WORK_DIR)
src_dir = source.get_dir()
contents = os.listdir(src_dir)
if contents:
print("source tree in:", src_dir)
else:
print("no source")
rm_rf(config.info_dir)
files1 = prefix_files()
for pat in m.always_include_files():
has_matches = False
for f in set(files1):
if fnmatch.fnmatch(f, pat):
print("Including in package existing file", f)
files1.discard(f)
has_matches = True
if not has_matches:
sys.exit(
"Error: Glob %s from always_include_files does not match any files"
% pat)
# Save this for later
with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
f.write(u'\n'.join(sorted(list(files1))))
f.write(u'\n')
if sys.platform == 'win32':
import conda_build.windows as windows
windows.build(m)
else:
env = environ.get_dict(m)
build_file = join(m.path, 'build.sh')
script = m.get_value('build/script', None)
if script:
if isinstance(script, list):
script = '\n'.join(script)
build_file = join(source.get_dir(), 'conda_build.sh')
with open(build_file, 'w') as bf:
bf.write(script)
os.chmod(build_file, 0o766)
if isfile(build_file):
cmd = ['/bin/bash', '-x', '-e', build_file]
_check_call(cmd, env=env, cwd=src_dir)
if post in [True, None]:
if post == True:
with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
files1 = set(f.read().splitlines())
get_build_metadata(m)
create_post_scripts(m)
create_entry_points(m.get_value('build/entry_points'))
assert not exists(config.info_dir)
files2 = prefix_files()
post_process(sorted(files2 - files1),
preserve_egg_dir=bool(
m.get_value('build/preserve_egg_dir')))
# The post processing may have deleted some files (like easy-install.pth)
files2 = prefix_files()
if any(config.meta_dir in join(config.build_prefix, f)
for f in files2 - files1):
sys.exit(
indent(
"""Error: Untracked file(s) %s found in conda-meta directory. This error
usually comes from using conda in the build script. Avoid doing this, as it
can lead to packages that include their dependencies.""" %
(tuple(f
for f in files2 - files1 if config.meta_dir in join(
config.build_prefix, f)), )))
post_build(m, sorted(files2 - files1))
create_info_files(m,
sorted(files2 - files1),
include_recipe=bool(m.path) and include_recipe)
if m.get_value('build/noarch_python'):
import conda_build.noarch_python as noarch_python
noarch_python.transform(m, sorted(files2 - files1))
files3 = prefix_files()
fix_permissions(files3 - files1)
path = bldpkg_path(m)
t = tarfile.open(path, 'w:bz2')
for f in sorted(files3 - files1):
t.add(join(config.build_prefix, f), f)
t.close()
print("BUILD END:", m.dist())
# we're done building, perform some checks
tarcheck.check_all(path)
update_index(config.bldpkgs_dir)
else:
print("STOPPING BUILD BEFORE POST:", m.dist())
def skeletonize(packages,
output_dir=".",
version=None,
git_tag=None,
cran_url="https://cran.r-project.org/",
recursive=False,
archive=True,
version_compare=False,
update_outdated=False,
config=None):
if not config:
config = Config()
if len(packages) > 1 and version_compare:
raise ValueError(
"--version-compare only works with one package at a time")
if not update_outdated and not packages:
raise ValueError("At least one package must be supplied")
package_dicts = {}
cran_metadata = get_cran_metadata(cran_url, output_dir)
if update_outdated:
packages = get_outdated(output_dir, cran_metadata, packages)
for pkg in packages:
rm_rf(join(output_dir[0], 'r-' + pkg))
while packages:
package = packages.pop()
is_github_url = 'github.com' in package
url = package
if is_github_url:
rm_rf(config.work_dir)
m = metadata.MetaData.fromdict({'source': {
'git_url': package
}},
config=config)
source.git_source(m.get_section('source'), m.config.git_cache,
m.config.work_dir)
git_tag = git_tag[0] if git_tag else get_latest_git_tag(config)
p = subprocess.Popen(['git', 'checkout', git_tag],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=config.work_dir)
stdout, stderr = p.communicate()
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
if p.returncode:
sys.exit(
"Error: 'git checkout %s' failed (%s).\nInvalid tag?" %
(git_tag, stderr.strip()))
if stdout:
print(stdout, file=sys.stdout)
if stderr:
print(stderr, file=sys.stderr)
DESCRIPTION = join(config.work_dir, "DESCRIPTION")
if not isfile(DESCRIPTION):
sub_description_pkg = join(config.work_dir, 'pkg',
"DESCRIPTION")
sub_description_name = join(config.work_dir,
package.split('/')[-1],
"DESCRIPTION")
if isfile(sub_description_pkg):
DESCRIPTION = sub_description_pkg
elif isfile(sub_description_name):
DESCRIPTION = sub_description_name
else:
sys.exit(
"%s does not appear to be a valid R package "
"(no DESCRIPTION file in %s, %s)" %
(package, sub_description_pkg, sub_description_name))
with open(DESCRIPTION) as f:
description_text = clear_trailing_whitespace(f.read())
d = dict_from_cran_lines(
remove_package_line_continuations(
description_text.splitlines()))
d['orig_description'] = description_text
package = d['Package'].lower()
cran_metadata[package] = d
if package.startswith('r-'):
package = package[2:]
if package.endswith('/'):
package = package[:-1]
if package.lower() not in cran_metadata:
sys.exit("Package %s not found" % package)
# Make sure package is always uses the CRAN capitalization
package = cran_metadata[package.lower()]['Package']
if not is_github_url:
session = get_session(output_dir)
cran_metadata[package.lower()].update(
get_package_metadata(cran_url, package, session))
dir_path = join(output_dir, 'r-' + package.lower())
if exists(dir_path) and not version_compare:
raise RuntimeError("directory already exists: %s" % dir_path)
cran_package = cran_metadata[package.lower()]
d = package_dicts.setdefault(
package,
{
'cran_packagename': package,
'packagename': 'r-' + package.lower(),
'build_depends': '',
'run_depends': '',
# CRAN doesn't seem to have this metadata :(
'home_comment': '#',
'homeurl': '',
'summary_comment': '#',
'summary': '',
})
if is_github_url:
d['url_key'] = ''
d['fn_key'] = ''
d['git_url_key'] = 'git_url:'
d['git_tag_key'] = 'git_tag:'
d['hash_entry'] = '# You can add a hash for the file here, like md5, sha1 or sha256'
d['filename'] = ''
d['cranurl'] = ''
d['git_url'] = url
d['git_tag'] = git_tag
else:
d['url_key'] = 'url:'
d['fn_key'] = 'fn:'
d['git_url_key'] = ''
d['git_tag_key'] = ''
d['git_url'] = ''
d['git_tag'] = ''
d['hash_entry'] = ''
if version:
d['version'] = version
raise NotImplementedError(
"Package versions from CRAN are not yet implemented")
d['cran_version'] = cran_package['Version']
# Conda versions cannot have -. Conda (verlib) will treat _ as a .
d['conda_version'] = d['cran_version'].replace('-', '_')
if version_compare:
sys.exit(not version_compare(dir_path, d['conda_version']))
if not is_github_url:
filename = '{}_{}.tar.gz'
contrib_url = cran_url + 'src/contrib/'
package_url = contrib_url + filename.format(
package, d['cran_version'])
# calculate sha256 by downloading source
sha256 = hashlib.sha256()
print("Downloading source from {}".format(package_url))
sha256.update(urlopen(package_url).read())
d['hash_entry'] = 'sha256: {}'.format(sha256.hexdigest())
d['filename'] = filename.format(package, '{{ version }}')
if archive:
d['cranurl'] = (INDENT + contrib_url + d['filename'] + INDENT +
contrib_url + 'Archive/{}/'.format(package) +
d['filename'])
else:
d['cranurl'] = ' ' + cran_url + 'src/contrib/' + d['filename']
d['cran_metadata'] = '\n'.join(
['# %s' % l for l in cran_package['orig_lines'] if l])
# XXX: We should maybe normalize these
d['license'] = cran_package.get("License", "None")
d['license_family'] = guess_license_family(d['license'],
allowed_license_families)
if 'License_is_FOSS' in cran_package:
d['license'] += ' (FOSS)'
if cran_package.get('License_restricts_use') == 'yes':
d['license'] += ' (Restricts use)'
if "URL" in cran_package:
d['home_comment'] = ''
d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL'])
else:
# use CRAN page as homepage if nothing has been specified
d['home_comment'] = ''
d['homeurl'] = ' https://CRAN.R-project.org/package={}'.format(
package)
if 'Description' in cran_package:
d['summary_comment'] = ''
d['summary'] = ' ' + yaml_quote_string(cran_package['Description'])
if "Suggests" in cran_package:
d['suggests'] = "# Suggests: %s" % cran_package['Suggests']
else:
d['suggests'] = ''
# Every package depends on at least R.
# I'm not sure what the difference between depends and imports is.
depends = [
s.strip() for s in cran_package.get('Depends', '').split(',')
if s.strip()
]
imports = [
s.strip() for s in cran_package.get('Imports', '').split(',')
if s.strip()
]
links = [
s.strip() for s in cran_package.get("LinkingTo", '').split(',')
if s.strip()
]
dep_dict = {}
for s in set(chain(depends, imports, links)):
match = VERSION_DEPENDENCY_REGEX.match(s)
if not match:
sys.exit("Could not parse version from dependency of %s: %s" %
(package, s))
name = match.group('name')
archs = match.group('archs')
relop = match.group('relop') or ''
version = match.group('version') or ''
version = version.replace('-', '_')
# If there is a relop there should be a version
assert not relop or version
if archs:
sys.exit("Don't know how to handle archs from dependency of "
"package %s: %s" % (package, s))
dep_dict[name] = '{relop}{version}'.format(relop=relop,
version=version)
if 'R' not in dep_dict:
dep_dict['R'] = ''
for dep_type in ['build', 'run']:
deps = []
for name in sorted(dep_dict):
if name in R_BASE_PACKAGE_NAMES:
continue
if name == 'R':
# Put R first
# Regarless of build or run, and whether this is a recommended package or not,
# it can only depend on 'r-base' since anything else can and will cause cycles
# in the dependency graph. The cran metadata lists all dependencies anyway, even
# those packages that are in the recommended group.
r_name = 'r-base'
# We don't include any R version restrictions because we
# always build R packages against an exact R version
deps.insert(
0, '{indent}{r_name}'.format(indent=INDENT,
r_name=r_name))
else:
conda_name = 'r-' + name.lower()
if dep_dict[name]:
deps.append('{indent}{name} {version}'.format(
name=conda_name,
version=dep_dict[name],
indent=INDENT))
else:
deps.append('{indent}{name}'.format(name=conda_name,
indent=INDENT))
if recursive:
if not exists(join(output_dir, conda_name)):
packages.append(name)
if cran_package.get("NeedsCompilation", 'no') == 'yes':
if dep_type == 'build':
deps.append('{indent}posix # [win]'.format(
indent=INDENT))
deps.append(
'{indent}{{{{native}}}}toolchain # [win]'.format(
indent=INDENT))
deps.append(
'{indent}gcc # [not win]'.format(
indent=INDENT))
elif dep_type == 'run':
deps.append(
'{indent}{{{{native}}}}gcc-libs # [win]'.format(
indent=INDENT))
deps.append(
'{indent}libgcc # [not win]'.format(
indent=INDENT))
d['%s_depends' % dep_type] = ''.join(deps)
for package in package_dicts:
d = package_dicts[package]
name = d['packagename']
# Normalize the metadata values
d = {
k: unicodedata.normalize("NFKD",
text_type(v)).encode('ascii',
'ignore').decode()
for k, v in iteritems(d)
}
makedirs(join(output_dir, name))
print("Writing recipe for %s" % package.lower())
with open(join(output_dir, name, 'meta.yaml'), 'w') as f:
f.write(clear_trailing_whitespace(CRAN_META.format(**d)))
with open(join(output_dir, name, 'build.sh'), 'w') as f:
f.write(CRAN_BUILD_SH.format(**d))
with open(join(output_dir, name, 'bld.bat'), 'w') as f:
f.write(CRAN_BLD_BAT.format(**d))
print("Done")
def main(config=None):
print(banner)
parser = argparse.ArgumentParser(
description="Boa, the fast, mamba powered-build tool for conda packages."
)
subparsers = parser.add_subparsers(help="sub-command help", dest="command")
parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument("recipe_dir", type=str)
render_parser = subparsers.add_parser(
"render", parents=[parent_parser], help="render a recipe"
)
convert_parser = subparsers.add_parser(
"convert", parents=[parent_parser], help="convert recipe.yaml to old-style meta.yaml"
)
build_parser = subparsers.add_parser(
"build", parents=[parent_parser], help="build a recipe"
)
args = parser.parse_args()
command = args.command
if command == 'convert':
from boa.cli import convert
convert.main(args.recipe_dir)
exit()
folder = args.recipe_dir
cbc, config = get_config(folder)
if not os.path.exists(config.output_folder):
mkdir_p(config.output_folder)
print(f"Updating build index: {(config.output_folder)}\n")
update_index(config.output_folder, verbose=config.debug, threads=1)
recipe_path = os.path.join(folder, "recipe.yaml")
# step 1: parse YAML
with open(recipe_path) as fi:
loader = YAML(typ="safe")
ydoc = loader.load(fi)
# step 2: fill out context dict
context_dict = ydoc.get("context") or {}
jenv = jinja2.Environment()
for key, value in context_dict.items():
if isinstance(value, str):
tmpl = jenv.from_string(value)
context_dict[key] = tmpl.render(context_dict)
if ydoc.get("context"):
del ydoc["context"]
# step 3: recursively loop over the entire recipe and render jinja with context
jenv.globals.update(jinja_functions(config, context_dict))
for key in ydoc:
render_recursive(ydoc[key], context_dict, jenv)
flatten_selectors(ydoc, ns_cfg(config))
normalize_recipe(ydoc)
# pprint(ydoc)
# We need to assemble the variants for each output
variants = {}
# if we have a outputs section, use that order the outputs
if ydoc.get("outputs"):
for o in ydoc["outputs"]:
# inherit from global package
pkg_meta = {}
pkg_meta.update(ydoc["package"])
pkg_meta.update(o["package"])
o["package"] = pkg_meta
build_meta = {}
build_meta.update(ydoc.get("build"))
build_meta.update(o.get("build") or {})
o["build"] = build_meta
variants[o["package"]["name"]] = get_dependency_variants(
o.get("requirements", {}), cbc, config
)
else:
# we only have one output
variants[ydoc["package"]["name"]] = get_dependency_variants(
ydoc.get("requirements", {}), cbc, config
)
# this takes in all variants and outputs, builds a dependency tree and returns
# the final metadata
sorted_outputs = to_build_tree(ydoc, variants, config)
# then we need to solve and build from the bottom up
# we can't first solve all packages without finalizing everything
#
# FLOW:
# =====
# - solve the package
# - solv build, add weak run exports to
# - add run exports from deps!
print('\n')
if command == "render":
for o in sorted_outputs:
print(o)
exit()
# TODO this should be done cleaner
top_name = ydoc['package']['name']
o0 = sorted_outputs[0]
o0.is_first = True
o0.config.compute_build_id(top_name)
solver = MambaSolver(["conda-forge"], context.subdir)
print("\n")
download_source(MetaData(recipe_path, o0))
cached_source = o0.sections['source']
for o in sorted_outputs:
solver.replace_channels()
o.finalize_solve(sorted_outputs, solver)
# print(o)
o.config._build_id = o0.config.build_id
if "build" in o.transactions:
if isdir(o.config.build_prefix):
utils.rm_rf(o.config.build_prefix)
mkdir_p(o.config.build_prefix)
o.transactions['build'].execute(PrefixData(o.config.build_prefix), PackageCacheData.first_writable().pkgs_dir)
if "host" in o.transactions:
mkdir_p(o.config.host_prefix)
o.transactions['host'].execute(PrefixData(o.config.host_prefix), PackageCacheData.first_writable().pkgs_dir)
meta = MetaData(recipe_path, o)
o.final_build_id = meta.build_id()
if cached_source != o.sections['source']:
download_source(meta)
build(meta, None)
for o in sorted_outputs:
print("\n")
print(o)
def execute(args):
parser, args = parse_args(args)
config = Config(**args.__dict__)
build.check_external()
# change globals in build module, see comment there as well
config.channel_urls = args.channel or ()
config.override_channels = args.override_channels
config.verbose = not args.quiet or args.debug
if 'purge' in args.recipe:
build.clean_build(config)
return
if 'purge-all' in args.recipe:
build.clean_build(config)
config.clean_pkgs()
return
set_language_env_vars(args, parser, config=config, execute=execute)
action = None
if args.output:
action = output_action
logging.basicConfig(level=logging.ERROR)
config.verbose = False
config.quiet = True
elif args.test:
action = test_action
elif args.source:
action = source_action
elif args.check:
action = check_action
if action:
for recipe in args.recipe:
recipe_dir, need_cleanup = get_recipe_abspath(recipe)
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
# this fully renders any jinja templating, throwing an error if any data is missing
m, _, _ = render_recipe(recipe_dir,
no_download_source=False,
config=config)
action(m, config)
if need_cleanup:
rm_rf(recipe_dir)
else:
api.build(args.recipe,
post=args.post,
build_only=args.build_only,
notest=args.notest,
keep_old_work=args.keep_old_work,
already_built=None,
config=config,
noverify=args.no_verify)
if not args.output and len(build.get_build_folders(config.croot)) > 0:
build.print_build_intermediate_warning(config)
def distribute_variants(metadata,
variants,
permit_unsatisfiable_variants=False,
allow_no_other_outputs=False,
bypass_env_check=False):
rendered_metadata = {}
need_source_download = True
# don't bother distributing python if it's a noarch package, and figure out
# which python version we prefer. `python_age` can use used to tweak which
# python gets used here.
if metadata.noarch or metadata.noarch_python:
from .conda_interface import VersionOrder
age = int(
metadata.get_value('build/noarch_python_build_age',
metadata.config.noarch_python_build_age))
versions = []
for variant in variants:
if 'python' in variant:
vo = variant['python']
if vo not in versions:
versions.append(vo)
version_indices = sorted(
range(len(versions)),
key=lambda k: VersionOrder(versions[k].split(' ')[0]))
if age < 0:
age = 0
elif age > len(versions) - 1:
age = len(versions) - 1
build_ver = versions[version_indices[len(versions) - 1 - age]]
variants = filter_by_key_value(variants, 'python', build_ver,
'noarch_python_reduction')
# store these for reference later
metadata.config.variants = variants
# These are always the full set. just 'variants' is the one that gets
# used mostly, and can be reduced
metadata.config.input_variants = variants
recipe_requirements = metadata.extract_requirements_text()
recipe_package_and_build_text = metadata.extract_package_and_build_text()
recipe_text = recipe_package_and_build_text + recipe_requirements
if PY3 and hasattr(recipe_text, 'decode'):
recipe_text = recipe_text.decode()
elif not PY3 and hasattr(recipe_text, 'encode'):
recipe_text = recipe_text.encode()
metadata.config.variant = variants[0]
used_variables = metadata.get_used_loop_vars(force_global=False)
top_loop = metadata.get_reduced_variant_set(used_variables)
for variant in top_loop:
from conda_build.build import get_all_replacements
get_all_replacements(variant)
mv = metadata.copy()
mv.config.variant = variant
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
conform_dict = {}
for key in used_variables:
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
conform_dict[key] = variant[key]
for key, values in conform_dict.items():
mv.config.variants = (filter_by_key_value(
mv.config.variants, key, values,
'distribute_variants_reduction') or mv.config.variants)
get_all_replacements(mv.config.variants)
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
numpy_pinned_variants = []
for _variant in mv.config.variants:
_variant['pin_run_as_build'] = pin_run_as_build
numpy_pinned_variants.append(_variant)
mv.config.variants = numpy_pinned_variants
mv.config.squished_variants = list_of_dicts_to_dict_of_lists(
mv.config.variants)
if mv.needs_source_for_render and mv.variant_in_source:
mv.parse_again()
utils.rm_rf(mv.config.work_dir)
source.provide(mv)
mv.parse_again()
try:
mv.parse_until_resolved(
allow_no_other_outputs=allow_no_other_outputs,
bypass_env_check=bypass_env_check)
except SystemExit:
pass
need_source_download = (not mv.needs_source_for_render
or not mv.source_provided)
rendered_metadata[(mv.dist(),
mv.config.variant.get('target_platform', mv.config.subdir),
tuple((var, mv.config.variant.get(var))
for var in mv.get_used_vars()))] = \
(mv, need_source_download, None)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
def download_to_cache(cache_folder, recipe_path, source_dict):
''' Download a source to the local cache. '''
print('Source cache directory is: %s' % cache_folder)
if not isdir(cache_folder):
os.makedirs(cache_folder)
source_urls = source_dict['url']
if not isinstance(source_urls, list):
source_urls = [source_urls]
unhashed_fn = fn = source_dict['fn'] if 'fn' in source_dict else basename(source_urls[0])
hash_added = False
for hash_type in ('md5', 'sha1', 'sha256'):
if hash_type in source_dict:
fn = append_hash_to_fn(fn, source_dict[hash_type])
hash_added = True
break
else:
log = get_logger(__name__)
log.warn("No hash (md5, sha1, sha256) provided for {}. Source download forced. "
"Add hash to recipe to use source cache.".format(unhashed_fn))
path = join(cache_folder, fn)
if isfile(path):
print('Found source in cache: %s' % fn)
else:
print('Downloading source to cache: %s' % fn)
for url in source_urls:
if "://" not in url:
if url.startswith('~'):
url = expanduser(url)
if not os.path.isabs(url):
url = os.path.normpath(os.path.join(recipe_path, url))
url = url_path(url)
else:
if url.startswith('file:///~'):
url = 'file:///' + expanduser(url[8:]).replace('\\', '/')
try:
print("Downloading %s" % url)
with LoggingContext():
download(url, path)
except CondaHTTPError as e:
print("Error: %s" % str(e).strip(), file=sys.stderr)
rm_rf(path)
except RuntimeError as e:
print("Error: %s" % str(e).strip(), file=sys.stderr)
rm_rf(path)
else:
print("Success")
break
else: # no break
rm_rf(path)
raise RuntimeError("Could not download %s" % url)
hashed = None
for tp in ('md5', 'sha1', 'sha256'):
if tp in source_dict:
expected_hash = source_dict[tp]
hashed = hashsum_file(path, tp)
if expected_hash != hashed:
rm_rf(path)
raise RuntimeError("%s mismatch: '%s' != '%s'" %
(tp.upper(), hashed, expected_hash))
break
# this is really a fallback. If people don't provide the hash, we still need to prevent
# collisions in our source cache, but the end user will get no benefirt from the cache.
if not hash_added:
if not hashed:
hashed = hashsum_file(path, 'sha256')
dest_path = append_hash_to_fn(path, hashed)
if not os.path.isfile(dest_path):
shutil.move(path, dest_path)
path = dest_path
return path, unhashed_fn
def render_recipe(recipe_path,
config,
no_download_source=False,
variants=None,
permit_unsatisfiable_variants=True,
reset_build_id=True):
arg = recipe_path
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
elif arg.endswith('.yaml'):
recipe_dir = os.path.dirname(arg)
need_cleanup = False
else:
print("Ignoring non-recipe: %s" % arg)
return None, None
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir, config=config)
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
if config.set_build_id:
m.config.compute_build_id(m.name(), reset=reset_build_id)
if m.needs_source_for_render and (not os.path.isdir(m.config.work_dir) or
len(os.listdir(m.config.work_dir)) == 0):
try_download(m, no_download_source=no_download_source)
rendered_metadata = {}
if m.final:
rendered_metadata = [
(m, False, False),
]
index = None
else:
variants = dict_of_lists_to_list_of_dicts(
variants) if variants else get_package_variants(m)
index = get_build_index(m.config, m.config.build_subdir)
rendered_metadata = distribute_variants(
m,
variants,
index,
permit_unsatisfiable_variants=permit_unsatisfiable_variants)
if not rendered_metadata:
raise ValueError(
"No variants were satisfiable - no valid recipes could be rendered."
)
if need_cleanup:
utils.rm_rf(recipe_dir)
return rendered_metadata, index
def create_env(prefix, specs, config, clear_cache=True):
'''
Create a conda envrionment for the given prefix and specs.
'''
if config.debug:
logging.getLogger("conda").setLevel(logging.DEBUG)
logging.getLogger("binstar").setLevel(logging.DEBUG)
logging.getLogger("install").setLevel(logging.DEBUG)
logging.getLogger("conda.install").setLevel(logging.DEBUG)
logging.getLogger("fetch").setLevel(logging.DEBUG)
logging.getLogger("print").setLevel(logging.DEBUG)
logging.getLogger("progress").setLevel(logging.DEBUG)
logging.getLogger("dotupdate").setLevel(logging.DEBUG)
logging.getLogger("stdoutlog").setLevel(logging.DEBUG)
logging.getLogger("requests").setLevel(logging.DEBUG)
else:
silence_loggers(show_warnings_and_errors=True)
if os.path.isdir(prefix):
rm_rf(prefix)
specs = list(specs)
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
if specs: # Don't waste time if there is nothing to do
with path_prepended(prefix):
locks = []
try:
cc.pkgs_dirs = cc.pkgs_dirs[:1]
locked_folders = cc.pkgs_dirs + list(config.bldpkgs_dirs)
for folder in locked_folders:
if not os.path.isdir(folder):
os.makedirs(folder)
lock = filelock.SoftFileLock(join(folder, '.conda_lock'))
update_index(folder, config=config, lock=lock)
locks.append(lock)
for lock in locks:
lock.acquire(timeout=config.timeout)
index = get_build_index(config=config, clear_cache=True)
actions = plan.install_actions(prefix, index, specs)
if config.disable_pip:
actions['LINK'] = [
spec for spec in actions['LINK']
if not spec.startswith('pip-')
] # noqa
actions['LINK'] = [
spec for spec in actions['LINK']
if not spec.startswith('setuptools-')
] # noqa
plan.display_actions(actions, index)
if on_win:
for k, v in os.environ.items():
os.environ[k] = str(v)
plan.execute_actions(actions, index, verbose=config.debug)
except (SystemExit, PaddingError, LinkError) as exc:
if (("too short in" in str(exc)
or 'post-link failed for: openssl' in str(exc)
or isinstance(exc, PaddingError))
and config.prefix_length > 80):
log.warn("Build prefix failed with prefix length %d",
config.prefix_length)
log.warn("Error was: ")
log.warn(str(exc))
log.warn(
"One or more of your package dependencies needs to be rebuilt "
"with a longer prefix length.")
log.warn(
"Falling back to legacy prefix length of 80 characters."
)
log.warn(
"Your package will not install into prefixes > 80 characters."
)
config.prefix_length = 80
# Set this here and use to create environ
# Setting this here is important because we use it below (symlink)
prefix = config.build_prefix
for lock in locks:
lock.release()
if os.path.isfile(lock._lock_file):
os.remove(lock._lock_file)
create_env(prefix,
specs,
config=config,
clear_cache=clear_cache)
else:
for lock in locks:
lock.release()
if os.path.isfile(lock._lock_file):
os.remove(lock._lock_file)
raise
finally:
for lock in locks:
lock.release()
if os.path.isfile(lock._lock_file):
os.remove(lock._lock_file)
warn_on_old_conda_build(index=index)
# ensure prefix exists, even if empty, i.e. when specs are empty
if not isdir(prefix):
os.makedirs(prefix)
if on_win:
shell = "cmd.exe"
else:
shell = "bash"
symlink_conda(prefix, sys.prefix, shell)
def distribute_variants(metadata,
variants,
permit_unsatisfiable_variants=False,
allow_no_other_outputs=False,
bypass_env_check=False):
rendered_metadata = {}
need_source_download = True
# don't bother distributing python if it's a noarch package
if metadata.noarch or metadata.noarch_python:
conform_dict = {'python': variants[0]['python']}
variants = conform_variants_to_value(variants, conform_dict)
# store these for reference later
metadata.config.variants = variants
# These are always the full set. just 'variants' is the one that gets
# used mostly, and can be reduced
metadata.config.input_variants = variants
squished_variants = list_of_dicts_to_dict_of_lists(variants)
recipe_requirements = metadata.extract_requirements_text()
recipe_package_and_build_text = metadata.extract_package_and_build_text()
recipe_text = recipe_package_and_build_text + recipe_requirements
if PY3 and hasattr(recipe_text, 'decode'):
recipe_text = recipe_text.decode()
elif not PY3 and hasattr(recipe_text, 'encode'):
recipe_text = recipe_text.encode()
for variant in variants:
mv = metadata.copy()
# this determines which variants were used, and thus which ones should be locked for
# future rendering
mv.final = False
mv.config.variant = {}
mv.parse_again(permit_undefined_jinja=True,
allow_no_other_outputs=True,
bypass_env_check=True)
vars_in_recipe = set(mv.undefined_jinja_vars)
mv.config.variant = variant
conform_dict = {}
for key in vars_in_recipe:
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
if re.search(r"\s*\{\{\s*%s\s*(?:.*?)?\}\}" % key, recipe_text):
if key in variant:
variant_index = squished_variants[key].index(variant[key])
zipped_keys = [key]
if 'zip_keys' in variant:
zip_key_groups = variant['zip_keys']
if zip_key_groups and not isinstance(
zip_key_groups[0], list):
zip_key_groups = [zip_key_groups]
for group in zip_key_groups:
if key in group:
zipped_keys = group
break
for zipped_key in zipped_keys:
conform_dict[zipped_key] = squished_variants[
zipped_key][variant_index]
conform_dict.update({
key: val
for key, val in variant.items()
if key in mv.meta.get('requirements').get('build', []) +
mv.meta.get('requirements').get('host', [])
})
compiler_matches = re.findall(r"compiler\([\'\"](.*)[\'\"].*\)",
recipe_requirements)
if compiler_matches:
from conda_build.jinja_context import native_compiler
for match in compiler_matches:
compiler_key = '{}_compiler'.format(match)
conform_dict[compiler_key] = variant.get(
compiler_key, native_compiler(match, mv.config))
conform_dict['target_platform'] = variant['target_platform']
build_reqs = mv.meta.get('requirements', {}).get('build', [])
host_reqs = mv.meta.get('requirements', {}).get('host', [])
if 'python' in build_reqs or 'python' in host_reqs:
conform_dict['python'] = variant['python']
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
mv.config.variants = conform_variants_to_value(mv.config.variants,
conform_dict)
numpy_pinned_variants = []
for _variant in mv.config.variants:
_variant['pin_run_as_build'] = pin_run_as_build
numpy_pinned_variants.append(_variant)
mv.config.variants = numpy_pinned_variants
if mv.needs_source_for_render and mv.variant_in_source:
mv.parse_again()
utils.rm_rf(mv.config.work_dir)
source.provide(mv)
mv.parse_again()
mv.parse_until_resolved(allow_no_other_outputs=allow_no_other_outputs,
bypass_env_check=bypass_env_check)
need_source_download = (bool(mv.meta.get('source'))
and not mv.needs_source_for_render
and not os.listdir(mv.config.work_dir))
# if python is in the build specs, but doesn't have a specific associated
# version, make sure to add one to newly parsed 'requirements/build'.
for env in ('build', 'host', 'run'):
utils.insert_variant_versions(mv, env)
fm = mv.copy()
# HACK: trick conda-build into thinking this is final, and computing a hash based
# on the current meta.yaml. The accuracy doesn't matter, all that matters is
# our ability to differentiate configurations
fm.final = True
rendered_metadata[fm.dist()] = (mv, need_source_download, None)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
def build(m,
config,
post=None,
need_source_download=True,
need_reparse_in_env=False):
'''
Build the package with the specified metadata.
:param m: Package metadata
:type m: Metadata
:type post: bool or None. None means run the whole build. True means run
post only. False means stop just before the post.
:type keep_old_work: bool: Keep any previous work directory.
:type need_source_download: bool: if rendering failed to download source
(due to missing tools), retry here after build env is populated
'''
if m.skip():
print_skip_message(m)
return False
if config.skip_existing:
package_exists = is_package_built(m, config)
if package_exists:
print(m.dist(),
"is already built in {0}, skipping.".format(package_exists))
return False
if post in [False, None]:
print("BUILD START:", m.dist())
if m.uses_jinja and (need_source_download or need_reparse_in_env):
print(
" (actual version deferred until further download or env creation)"
)
specs = [ms.spec for ms in m.ms_depends('build')]
create_env(config.build_prefix, specs, config=config)
vcs_source = m.uses_vcs_in_build
if vcs_source and vcs_source not in specs:
vcs_executable = "hg" if vcs_source == "mercurial" else vcs_source
has_vcs_available = os.path.isfile(
external.find_executable(vcs_executable, config.build_prefix)
or "")
if not has_vcs_available:
if (vcs_source != "mercurial" or not any(
spec.startswith('python') and "3." in spec
for spec in specs)):
specs.append(vcs_source)
log.warn(
"Your recipe depends on %s at build time (for templates), "
"but you have not listed it as a build dependency. Doing "
"so for this build.", vcs_source)
# Display the name only
# Version number could be missing due to dependency on source info.
create_env(config.build_prefix, specs, config=config)
else:
raise ValueError(
"Your recipe uses mercurial in build, but mercurial"
" does not yet support Python 3. Please handle all of "
"your mercurial actions outside of your build script.")
if need_source_download:
# Execute any commands fetching the source (e.g., git) in the _build environment.
# This makes it possible to provide source fetchers (eg. git, hg, svn) as build
# dependencies.
with path_prepended(config.build_prefix):
m, need_source_download, need_reparse_in_env = parse_or_try_download(
m,
no_download_source=False,
force_download=True,
config=config)
assert not need_source_download, "Source download failed. Please investigate."
if m.uses_jinja:
print("BUILD START (revised):", m.dist())
if need_reparse_in_env:
reparse(m, config=config)
print("BUILD START (revised):", m.dist())
if m.name() in [
i.rsplit('-', 2)[0] for i in linked(config.build_prefix)
]:
print("%s is installed as a build dependency. Removing." %
m.name())
index = get_build_index(config=config, clear_cache=False)
actions = plan.remove_actions(config.build_prefix, [m.name()],
index=index)
assert not plan.nothing_to_do(actions), actions
plan.display_actions(actions, index)
plan.execute_actions(actions, index)
print("Package:", m.dist())
with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"),
timeout=config.timeout):
# get_dir here might be just work, or it might be one level deeper,
# dependening on the source.
src_dir = source.get_dir(config)
if isdir(src_dir):
print("source tree in:", src_dir)
else:
print("no source - creating empty work folder")
os.makedirs(src_dir)
rm_rf(config.info_dir)
files1 = prefix_files(prefix=config.build_prefix)
for pat in m.always_include_files():
has_matches = False
for f in set(files1):
if fnmatch.fnmatch(f, pat):
print("Including in package existing file", f)
files1.discard(f)
has_matches = True
if not has_matches:
log.warn(
"Glob %s from always_include_files does not match any files",
pat)
# Save this for later
with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
f.write(u'\n'.join(sorted(list(files1))))
f.write(u'\n')
# Use script from recipe?
script = m.get_value('build/script', None)
if script:
if isinstance(script, list):
script = '\n'.join(script)
if isdir(src_dir):
if on_win:
build_file = join(m.path, 'bld.bat')
if script:
build_file = join(src_dir, 'bld.bat')
with open(build_file, 'w') as bf:
bf.write(script)
import conda_build.windows as windows
windows.build(m, build_file, config=config)
else:
build_file = join(m.path, 'build.sh')
# There is no sense in trying to run an empty build script.
if isfile(build_file) or script:
with path_prepended(config.build_prefix):
env = environ.get_dict(config=config, m=m)
env["CONDA_BUILD_STATE"] = "BUILD"
work_file = join(source.get_dir(config),
'conda_build.sh')
if script:
with open(work_file, 'w') as bf:
bf.write(script)
if config.activate:
if isfile(build_file):
data = open(build_file).read()
else:
data = open(work_file).read()
with open(work_file, 'w') as bf:
bf.write(
"source {conda_root}activate {build_prefix} &> "
"/dev/null\n".format(
conda_root=root_script_dir +
os.path.sep,
build_prefix=config.build_prefix))
bf.write(data)
else:
if not isfile(work_file):
copy_into(build_file, work_file,
config.timeout)
os.chmod(work_file, 0o766)
if isfile(work_file):
cmd = [shell_path, '-x', '-e', work_file]
# this should raise if any problems occur while building
_check_call(cmd, env=env, cwd=src_dir)
if post in [True, None]:
if post:
with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
files1 = set(f.read().splitlines())
get_build_metadata(m, config=config)
create_post_scripts(m, config=config)
create_entry_points(m.get_value('build/entry_points'), config=config)
files2 = prefix_files(prefix=config.build_prefix)
post_process(sorted(files2 - files1),
prefix=config.build_prefix,
config=config,
preserve_egg_dir=bool(
m.get_value('build/preserve_egg_dir')))
# The post processing may have deleted some files (like easy-install.pth)
files2 = prefix_files(prefix=config.build_prefix)
if any(config.meta_dir in join(config.build_prefix, f)
for f in files2 - files1):
meta_files = (tuple(
f for f in files2 - files1
if config.meta_dir in join(config.build_prefix, f)), )
sys.exit(
indent(
"""Error: Untracked file(s) %s found in conda-meta directory.
This error usually comes from using conda in the build script. Avoid doing this, as it
can lead to packages that include their dependencies.""" % meta_files))
post_build(m,
sorted(files2 - files1),
prefix=config.build_prefix,
build_python=config.build_python,
croot=config.croot)
create_info_files(m,
sorted(files2 - files1),
config=config,
prefix=config.build_prefix)
if m.get_value('build/noarch_python'):
import conda_build.noarch_python as noarch_python
noarch_python.transform(m, sorted(files2 - files1),
config.build_prefix)
files3 = prefix_files(prefix=config.build_prefix)
fix_permissions(files3 - files1, config.build_prefix)
path = bldpkg_path(m, config)
# lock the output directory while we build this file
# create the tarball in a temporary directory to minimize lock time
with TemporaryDirectory() as tmp:
tmp_path = os.path.join(tmp, os.path.basename(path))
t = tarfile.open(tmp_path, 'w:bz2')
def order(f):
# we don't care about empty files so send them back via 100000
fsize = os.stat(join(config.build_prefix, f)).st_size or 100000
# info/* records will be False == 0, others will be 1.
info_order = int(os.path.dirname(f) != 'info')
return info_order, fsize
# add files in order of a) in info directory, b) increasing size so
# we can access small manifest or json files without decompressing
# possible large binary or data files
for f in sorted(files3 - files1, key=order):
t.add(join(config.build_prefix, f), f)
t.close()
# we're done building, perform some checks
tarcheck.check_all(tmp_path)
copy_into(tmp_path, path, config.timeout)
update_index(config.bldpkgs_dir, config)
else:
print("STOPPING BUILD BEFORE POST:", m.dist())
# returning true here says package is OK to test
return True
def build(m,
get_src=True,
verbose=True,
post=None,
channel_urls=(),
override_channels=False):
'''
Build the package with the specified metadata.
:param m: Package metadata
:type m: Metadata
:param get_src: Should we download the source?
:type get_src: bool
:type post: bool or None. None means run the whole build. True means run
post only. False means stop just before the post.
'''
if (m.get_value('build/detect_binary_files_with_prefix')
or m.binary_has_prefix_files()):
# We must use a long prefix here as the package will only be
# installable into prefixes shorter than this one.
config.use_long_build_prefix = True
else:
# In case there are multiple builds in the same process
config.use_long_build_prefix = False
if post in [False, None]:
print("Removing old build directory")
rm_rf(config.short_build_prefix)
rm_rf(config.long_build_prefix)
print("Removing old work directory")
rm_rf(source.WORK_DIR)
# Display the name only
# Version number could be missing due to dependency on source info.
print("BUILD START:", m.dist())
create_env(config.build_prefix,
[ms.spec for ms in m.ms_depends('build')],
verbose=verbose,
channel_urls=channel_urls,
override_channels=override_channels)
if get_src:
source.provide(m.path, m.get_section('source'))
# Parse our metadata again because we did not initialize the source
# information before.
m.parse_again()
print("Package:", m.dist())
assert isdir(source.WORK_DIR)
src_dir = source.get_dir()
contents = os.listdir(src_dir)
if contents:
print("source tree in:", src_dir)
else:
print("no source")
rm_rf(config.info_dir)
files1 = prefix_files()
for f in m.always_include_files():
if f not in files1:
sys.exit("Error: File %s from always_include_files not found" %
f)
files1 = files1.difference(set(m.always_include_files()))
# Save this for later
with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
f.write(u'\n'.join(sorted(list(files1))))
f.write(u'\n')
if sys.platform == 'win32':
import conda_build.windows as windows
windows.build(m)
else:
env = environ.get_dict(m)
build_file = join(m.path, 'build.sh')
script = m.get_value('build/script', None)
if script:
if isinstance(script, list):
script = '\n'.join(script)
build_file = join(source.get_dir(), 'conda_build.sh')
with open(build_file, 'w') as bf:
bf.write(script)
os.chmod(build_file, 0o766)
if exists(build_file):
cmd = ['/bin/bash', '-x', '-e', build_file]
_check_call(cmd, env=env, cwd=src_dir)
if post in [True, None]:
if post == True:
with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
files1 = set(f.read().splitlines())
get_build_metadata(m)
create_post_scripts(m)
create_entry_points(m.get_value('build/entry_points'))
assert not exists(config.info_dir)
files2 = prefix_files()
post_process(sorted(files2 - files1),
preserve_egg_dir=bool(
m.get_value('build/preserve_egg_dir')))
# The post processing may have deleted some files (like easy-install.pth)
files2 = prefix_files()
post_build(m, sorted(files2 - files1))
create_info_files(m,
sorted(files2 - files1),
include_recipe=bool(m.path))
if m.get_value('build/noarch_python'):
import conda_build.noarch_python as noarch_python
noarch_python.transform(m, sorted(files2 - files1))
files3 = prefix_files()
fix_permissions(files3 - files1)
path = bldpkg_path(m)
t = tarfile.open(path, 'w:bz2')
for f in sorted(files3 - files1):
t.add(join(config.build_prefix, f), f)
t.close()
print("BUILD END:", m.dist())
# we're done building, perform some checks
tarcheck.check_all(path)
update_index(config.bldpkgs_dir)
else:
print("STOPPING BUILD BEFORE POST:", m.dist())
def create_env(prefix,
specs_or_actions,
env,
config,
subdir,
clear_cache=True,
retry=0,
locks=None,
is_cross=False,
is_conda=False):
'''
Create a conda envrionment for the given prefix and specs.
'''
if config.debug:
external_logger_context = utils.LoggingContext(logging.DEBUG)
else:
external_logger_context = utils.LoggingContext(logging.WARN)
if os.path.exists(prefix):
for entry in glob(os.path.join(prefix, "*")):
utils.rm_rf(entry)
with external_logger_context:
log = utils.get_logger(__name__)
# if os.path.isdir(prefix):
# utils.rm_rf(prefix)
if specs_or_actions: # Don't waste time if there is nothing to do
log.debug("Creating environment in %s", prefix)
log.debug(str(specs_or_actions))
if not locks:
locks = utils.get_conda_operation_locks(config)
try:
with utils.try_acquire_locks(locks, timeout=config.timeout):
# input is a list - it's specs in MatchSpec format
if not hasattr(specs_or_actions, 'keys'):
specs = list(set(specs_or_actions))
actions = get_install_actions(
prefix,
tuple(specs),
env,
subdir=subdir,
verbose=config.verbose,
debug=config.debug,
locking=config.locking,
bldpkgs_dirs=tuple(config.bldpkgs_dirs),
timeout=config.timeout,
disable_pip=config.disable_pip,
max_env_retry=config.max_env_retry,
output_folder=config.output_folder,
channel_urls=tuple(config.channel_urls))
else:
actions = specs_or_actions
index, _, _ = get_build_index(
subdir=subdir,
bldpkgs_dir=config.bldpkgs_dir,
output_folder=config.output_folder,
channel_urls=config.channel_urls,
debug=config.debug,
verbose=config.verbose,
locking=config.locking,
timeout=config.timeout)
utils.trim_empty_keys(actions)
display_actions(actions, index)
if utils.on_win:
for k, v in os.environ.items():
os.environ[k] = str(v)
with env_var('CONDA_QUIET', not config.verbose,
reset_context):
with env_var('CONDA_JSON', not config.verbose,
reset_context):
execute_actions(actions, index)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError,
BuildLockError) as exc:
if (("too short in" in str(exc) or re.search(
'post-link failed for: (?:[a-zA-Z]*::)?openssl',
str(exc)) or isinstance(exc, PaddingError))
and config.prefix_length > 80):
if config.prefix_length_fallback:
log.warn("Build prefix failed with prefix length %d",
config.prefix_length)
log.warn("Error was: ")
log.warn(str(exc))
log.warn(
"One or more of your package dependencies needs to be rebuilt "
"with a longer prefix length.")
log.warn(
"Falling back to legacy prefix length of 80 characters."
)
log.warn(
"Your package will not install into prefixes > 80 characters."
)
config.prefix_length = 80
host = '_h_env' in prefix
# Set this here and use to create environ
# Setting this here is important because we use it below (symlink)
prefix = config.host_prefix if host else config.build_prefix
actions['PREFIX'] = prefix
create_env(prefix,
actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
is_cross=is_cross)
else:
raise
elif 'lock' in str(exc):
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs_or_actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs_or_actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
else:
log.error(
"Failed to create env, max retries exceeded.")
raise
else:
raise
# HACK: some of the time, conda screws up somehow and incomplete packages result.
# Just retry.
except (AssertionError, IOError, ValueError, RuntimeError,
LockError) as exc:
if isinstance(exc, AssertionError):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = os.path.dirname(os.path.dirname(str(exc)))
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs_or_actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
else:
log.error("Failed to create env, max retries exceeded.")
raise
def test(m, verbose=True, channel_urls=(), override_channels=False):
'''
Execute any test scripts for the given package.
:param m: Package's metadata.
:type m: Metadata
'''
# remove from package cache
rm_pkgs_cache(m.dist())
tmp_dir = join(config.croot, 'test-tmp_dir')
rm_rf(tmp_dir)
os.makedirs(tmp_dir)
create_files(tmp_dir, m)
# Make Perl or Python-specific test files
if m.name().startswith('perl-'):
pl_files = create_pl_files(tmp_dir, m)
py_files = False
else:
py_files = create_py_files(tmp_dir, m)
pl_files = False
shell_files = create_shell_files(tmp_dir, m)
if not (py_files or shell_files or pl_files):
print("Nothing to test for:", m.dist())
return
print("TEST START:", m.dist())
rm_rf(config.build_prefix)
rm_rf(config.test_prefix)
specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]
if py_files:
# as the tests are run by python, we need to specify it
specs += ['python %s*' % environ.get_py_ver()]
if pl_files:
# as the tests are run by perl, we need to specify it
specs += ['perl %s*' % environ.get_perl_ver()]
# add packages listed in test/requires
for spec in m.get_value('test/requires', []):
specs.append(spec)
create_env(config.test_prefix, specs, verbose=verbose,
channel_urls=channel_urls, override_channels=override_channels)
env = dict(os.environ)
# TODO: Include all the same environment variables that are used in
# building.
env.update(environ.get_dict(m, prefix=config.test_prefix))
# prepend bin (or Scripts) directory
env['PATH'] = (join(config.test_prefix, bin_dirname) + os.pathsep +
os.getenv('PATH'))
for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL':
env[varname] = str(getattr(config, varname))
env['PREFIX'] = config.test_prefix
# Python 2 Windows requires that envs variables be string, not unicode
env = {str(key): str(value) for key, value in env.items()}
if py_files:
try:
subprocess.check_call([config.test_python, '-s',
join(tmp_dir, 'run_test.py')],
env=env, cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m)
if pl_files:
try:
subprocess.check_call([config.test_perl,
join(tmp_dir, 'run_test.pl')],
env=env, cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m)
if shell_files:
if sys.platform == 'win32':
test_file = join(tmp_dir, 'run_test.bat')
cmd = [os.environ['COMSPEC'], '/c', 'call', test_file]
try:
subprocess.check_call(cmd, env=env, cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m)
else:
test_file = join(tmp_dir, 'run_test.sh')
# TODO: Run the test/commands here instead of in run_test.py
cmd = ['/bin/bash', '-x', '-e', test_file]
try:
subprocess.check_call(cmd, env=env, cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m)
print("TEST END:", m.dist())
def build_recipe(args, recipe_path, cbc, config):
if args.features:
assert args.features.startswith("[") and args.features.endswith("]")
features = [f.strip() for f in args.features[1:-1].split(",")]
else:
features = []
selected_features = {}
for f in features:
if f.startswith("~"):
selected_features[f[1:]] = False
else:
selected_features[f] = True
ydoc = render(recipe_path, config=config)
# We need to assemble the variants for each output
variants = {}
# if we have a outputs section, use that order the outputs
if ydoc.get("outputs"):
for o in ydoc["outputs"]:
# inherit from global package
pkg_meta = {}
pkg_meta.update(ydoc["package"])
pkg_meta.update(o["package"])
o["package"] = pkg_meta
build_meta = {}
build_meta.update(ydoc.get("build"))
build_meta.update(o.get("build") or {})
o["build"] = build_meta
o["selected_features"] = selected_features
variants[o["package"]["name"]] = get_dependency_variants(
o.get("requirements", {}), cbc, config, features
)
else:
# we only have one output
variants[ydoc["package"]["name"]] = get_dependency_variants(
ydoc.get("requirements", {}), cbc, config, features
)
# this takes in all variants and outputs, builds a dependency tree and returns
# the final metadata
sorted_outputs = to_build_tree(ydoc, variants, config, selected_features)
# then we need to solve and build from the bottom up
# we can't first solve all packages without finalizing everything
#
# FLOW:
# =====
# - solve the package
# - solv build, add weak run exports to
# - add run exports from deps!
if args.command == "render":
for o in sorted_outputs:
console.print(o)
exit()
# TODO this should be done cleaner
top_name = ydoc["package"]["name"]
o0 = sorted_outputs[0]
o0.is_first = True
o0.config.compute_build_id(top_name)
console.print("\n[yellow]Initializing mamba solver[/yellow]\n")
solver = MambaSolver([], context.subdir)
console.print("\n[yellow]Downloading source[/yellow]\n")
download_source(MetaData(recipe_path, o0), args.interactive)
cached_source = o0.sections["source"]
for o in sorted_outputs:
console.print(
f"\n[yellow]Preparing environment for [bold]{o.name}[/bold][/yellow]\n"
)
solver.replace_channels()
o.finalize_solve(sorted_outputs, solver)
o.config._build_id = o0.config.build_id
if "build" in o.transactions:
if os.path.isdir(o.config.build_prefix):
rm_rf(o.config.build_prefix)
mkdir_p(o.config.build_prefix)
try:
o.transactions["build"].execute(
PrefixData(o.config.build_prefix),
PackageCacheData.first_writable().pkgs_dir,
)
except Exception:
# This currently enables windows-multi-build...
print("Could not instantiate build environment")
if "host" in o.transactions:
mkdir_p(o.config.host_prefix)
o.transactions["host"].execute(
PrefixData(o.config.host_prefix),
PackageCacheData.first_writable().pkgs_dir,
)
meta = MetaData(recipe_path, o)
o.set_final_build_id(meta)
if cached_source != o.sections["source"]:
download_source(meta, args.interactive)
console.print(f"\n[yellow]Starting build for [bold]{o.name}[/bold][/yellow]\n")
build(meta, None, allow_interactive=args.interactive)
for o in sorted_outputs:
print("\n\n")
console.print(o)
def test(m, verbose=True):
'''
Execute any test scripts for the given package.
:param m: Package's metadata.
:type m: Metadata
'''
# remove from package cache
rm_pkgs_cache(m.dist())
tmp_dir = join(config.croot, 'test-tmp_dir')
rm_rf(tmp_dir)
os.makedirs(tmp_dir)
create_files(tmp_dir, m)
# Make Perl or Python-specific test files
if m.name().startswith('perl-'):
pl_files = create_pl_files(tmp_dir, m)
py_files = False
else:
py_files = create_py_files(tmp_dir, m)
pl_files = False
shell_files = create_shell_files(tmp_dir, m)
if not (py_files or shell_files or pl_files):
print("Nothing to test for:", m.dist())
return
print("TEST START:", m.dist())
rm_rf(prefix)
rm_rf(config.test_prefix)
specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]
if py_files:
# as the tests are run by python, we need to specify it
specs += ['python %s*' % environ.PY_VER]
if pl_files:
# as the tests are run by perl, we need to specify it
specs += ['perl %s*' % environ.PERL_VER]
# add packages listed in test/requires
for spec in m.get_value('test/requires'):
specs.append(spec)
create_env(config.test_prefix, specs, verbose=verbose)
env = dict(os.environ)
# TODO: Include all the same environment variables that are used in
# building.
env.update(environ.get_dict(m, prefix=config.test_prefix))
# prepend bin (or Scripts) directory
env['PATH'] = (join(config.test_prefix, bin_dirname) + os.pathsep +
env['PATH'])
for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL':
env[varname] = str(getattr(config, varname))
env['PREFIX'] = config.test_prefix
# Python 2 Windows requires that envs variables be string, not unicode
env = {str(key): str(value) for key, value in env.items()}
if py_files:
try:
subprocess.check_call(
[config.test_python,
join(tmp_dir, 'run_test.py')],
env=env,
cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m)
if pl_files:
try:
subprocess.check_call(
[config.test_perl,
join(tmp_dir, 'run_test.pl')],
env=env,
cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m)
if shell_files:
if sys.platform == 'win32':
test_file = join(tmp_dir, 'run_test.bat')
cmd = [os.environ['COMSPEC'], '/c', test_file]
try:
subprocess.check_call(cmd, env=env, cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m)
else:
test_file = join(tmp_dir, 'run_test.sh')
# TODO: Run the test/commands here instead of in run_test.py
cmd = ['/bin/bash', '-x', '-e', test_file]
try:
subprocess.check_call(cmd, env=env, cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m)
print("TEST END:", m.dist())
def build(m, get_src=True, verbose=True, post=None, channel_urls=(), override_channels=False):
'''
Build the package with the specified metadata.
:param m: Package metadata
:type m: Metadata
:param get_src: Should we download the source?
:type get_src: bool
:type post: bool or None. None means run the whole build. True means run
post only. False means stop just before the post.
'''
if post in [False, None]:
print("Removing old build directory")
rm_rf(config.short_build_prefix)
rm_rf(config.long_build_prefix)
print("Removing old work directory")
rm_rf(source.WORK_DIR)
if (m.get_value('build/detect_binary_files_with_prefix')
or m.binary_has_prefix_files()):
# We must use a long prefix here as the package will only be
# installable into prefixes shorter than this one.
config.use_long_build_prefix = True
else:
# In case there are multiple builds in the same process
config.use_long_build_prefix = False
# Display the name only
# Version number could be missing due to dependency on source info.
print("BUILD START:", m.dist())
create_env(config.build_prefix,
[ms.spec for ms in m.ms_depends('build')],
verbose=verbose, channel_urls=channel_urls,
override_channels=override_channels)
if get_src:
source.provide(m.path, m.get_section('source'))
# Parse our metadata again because we did not initialize the source
# information before.
m.parse_again()
print("Package:", m.dist())
assert isdir(source.WORK_DIR)
src_dir = source.get_dir()
contents = os.listdir(src_dir)
if contents:
print("source tree in:", src_dir)
else:
print("no source")
rm_rf(config.info_dir)
files1 = prefix_files().difference(set(m.always_include_files()))
# Save this for later
with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
f.write(u'\n'.join(sorted(list(files1))))
f.write(u'\n')
if sys.platform == 'win32':
import conda_build.windows as windows
windows.build(m)
else:
env = environ.get_dict(m)
build_file = join(m.path, 'build.sh')
script = m.get_value('build/script', None)
if script:
if isinstance(script, list):
script = '\n'.join(script)
build_file = join(source.get_dir(), 'conda_build.sh')
with open(build_file, 'w') as bf:
bf.write(script)
os.chmod(build_file, 0o766)
if exists(build_file):
cmd = ['/bin/bash', '-x', '-e', build_file]
_check_call(cmd, env=env, cwd=src_dir)
if post in [True, None]:
if post == True:
with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
files1 = set(f.read().splitlines())
get_build_metadata(m)
create_post_scripts(m)
create_entry_points(m.get_value('build/entry_points'))
assert not exists(config.info_dir)
files2 = prefix_files()
post_process(sorted(files2 - files1), preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))
# The post processing may have deleted some files (like easy-install.pth)
files2 = prefix_files()
post_build(m, sorted(files2 - files1))
create_info_files(m, sorted(files2 - files1),
include_recipe=bool(m.path))
if m.get_value('build/noarch_python'):
import conda_build.noarch_python as noarch_python
noarch_python.transform(m, sorted(files2 - files1))
files3 = prefix_files()
fix_permissions(files3 - files1)
path = bldpkg_path(m)
t = tarfile.open(path, 'w:bz2')
for f in sorted(files3 - files1):
t.add(join(config.build_prefix, f), f)
t.close()
print("BUILD END:", m.dist())
# we're done building, perform some checks
tarcheck.check_all(path)
update_index(config.bldpkgs_dir)
else:
print("STOPPING BUILD BEFORE POST:", m.dist())
def get_run_dists(m, config):
prefix = join(envs_dirs[0], '_run')
rm_rf(prefix)
create_env(prefix, [ms.spec for ms in m.ms_depends('run')], config=config)
return sorted(linked(prefix))
def run_test(
recipedir_or_package_or_metadata,
config,
stats,
move_broken=True,
provision_only=False,
solver=None,
):
"""
Execute any test scripts for the given package.
:param m: Package's metadata.
:type m: Metadata
"""
# we want to know if we're dealing with package input. If so, we can move the input on success.
hash_input = {}
# store this name to keep it consistent. By changing files, we change the hash later.
# It matches the build hash now, so let's keep it around.
test_package_name = (
recipedir_or_package_or_metadata.dist()
if hasattr(recipedir_or_package_or_metadata, "dist")
else recipedir_or_package_or_metadata
)
if not provision_only:
print("TEST START:", test_package_name)
if hasattr(recipedir_or_package_or_metadata, "config"):
metadata = recipedir_or_package_or_metadata
utils.rm_rf(metadata.config.test_dir)
else:
metadata, hash_input = construct_metadata_for_test(
recipedir_or_package_or_metadata, config
)
trace = "-x " if metadata.config.debug else ""
# Must download *after* computing build id, or else computing build id will change
# folder destination
_extract_test_files_from_package(metadata)
# When testing a .tar.bz2 in the pkgs dir, clean_pkg_cache() will remove it.
# Prevent this. When https://github.com/conda/conda/issues/5708 gets fixed
# I think we can remove this call to clean_pkg_cache().
in_pkg_cache = (
not hasattr(recipedir_or_package_or_metadata, "config")
and os.path.isfile(recipedir_or_package_or_metadata)
and recipedir_or_package_or_metadata.endswith(CONDA_PACKAGE_EXTENSIONS)
and os.path.dirname(recipedir_or_package_or_metadata) in pkgs_dirs[0]
)
if not in_pkg_cache:
environ.clean_pkg_cache(metadata.dist(), metadata.config)
copy_test_source_files(metadata, metadata.config.test_dir)
# this is also copying tests/source_files from work_dir to testing workdir
_, pl_files, py_files, r_files, lua_files, shell_files = create_all_test_files(
metadata
)
if (
not any([py_files, shell_files, pl_files, lua_files, r_files])
and not metadata.config.test_run_post
):
print("Nothing to test for:", test_package_name)
return True
if metadata.config.remove_work_dir:
for name, prefix in (
("host", metadata.config.host_prefix),
("build", metadata.config.build_prefix),
):
if os.path.isdir(prefix):
# move host folder to force hardcoded paths to host env to break during tests
# (so that they can be properly addressed by recipe author)
dest = os.path.join(
os.path.dirname(prefix),
"_".join(
(
"%s_prefix_moved" % name,
metadata.dist(),
getattr(metadata.config, "%s_subdir" % name),
)
),
)
# Needs to come after create_files in case there's test/source_files
shutil_move_more_retrying(prefix, dest, "{} prefix".format(prefix))
# nested if so that there's no warning when we just leave the empty workdir in place
if metadata.source_provided:
dest = os.path.join(
os.path.dirname(metadata.config.work_dir),
"_".join(("work_moved", metadata.dist(), metadata.config.host_subdir)),
)
# Needs to come after create_files in case there's test/source_files
shutil_move_more_retrying(config.work_dir, dest, "work")
else:
log.warn(
"Not moving work directory after build. Your package may depend on files "
"in the work directory that are not included with your package"
)
# looks like a dead function to me
# get_build_metadata(metadata)
specs = metadata.get_test_deps(py_files, pl_files, lua_files, r_files)
with utils.path_prepended(metadata.config.test_prefix):
env = dict(os.environ.copy())
env.update(environ.get_dict(m=metadata, prefix=config.test_prefix))
env["CONDA_BUILD_STATE"] = "TEST"
env["CONDA_BUILD"] = "1"
if env_path_backup_var_exists:
env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]
if not metadata.config.activate or metadata.name() == "conda":
# prepend bin (or Scripts) directory
env = utils.prepend_bin_path(
env, metadata.config.test_prefix, prepend_prefix=True
)
if utils.on_win:
env["PATH"] = metadata.config.test_prefix + os.pathsep + env["PATH"]
env["PREFIX"] = metadata.config.test_prefix
if "BUILD_PREFIX" in env:
del env["BUILD_PREFIX"]
# In the future, we will need to support testing cross compiled
# packages on physical hardware. until then it is expected that
# something like QEMU or Wine will be used on the build machine,
# therefore, for now, we use host_subdir.
# ensure that the test prefix isn't kept between variants
utils.rm_rf(metadata.config.test_prefix)
if solver is None:
solver, pkg_cache_path = get_solver(metadata.config.host_subdir)
else:
pkg_cache_path = PackageCacheData.first_writable().pkgs_dir
solver.replace_channels()
transaction = solver.solve(specs, [pkg_cache_path])
downloaded = transaction.fetch_extract_packages(
pkg_cache_path, solver.repos + list(solver.local_repos.values()),
)
if not downloaded:
raise RuntimeError("Did not succeed in downloading packages.")
mkdir_p(metadata.config.test_prefix)
transaction.execute(
PrefixData(metadata.config.test_prefix), pkg_cache_path,
)
with utils.path_prepended(metadata.config.test_prefix):
env = dict(os.environ.copy())
env.update(environ.get_dict(m=metadata, prefix=metadata.config.test_prefix))
env["CONDA_BUILD_STATE"] = "TEST"
if env_path_backup_var_exists:
env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]
if config.test_run_post:
from conda_build.utils import get_installed_packages
installed = get_installed_packages(metadata.config.test_prefix)
files = installed[metadata.meta["package"]["name"]]["files"]
replacements = get_all_replacements(metadata.config)
try_download(metadata, False, True)
create_info_files(metadata, replacements, files, metadata.config.test_prefix)
post_build(metadata, files, None, metadata.config.test_prefix, True)
# when workdir is removed, the source files are unavailable. There's the test/source_files
# entry that lets people keep these files around. The files are copied into test_dir for
# intuitive relative path behavior, though, not work_dir, so we need to adjust where
# SRC_DIR points. The initial CWD during tests is test_dir.
if metadata.config.remove_work_dir:
env["SRC_DIR"] = metadata.config.test_dir
test_script, _ = write_test_scripts(
metadata, env, py_files, pl_files, lua_files, r_files, shell_files, trace
)
if utils.on_win:
cmd = [os.environ.get("COMSPEC", "cmd.exe"), "/d", "/c", test_script]
else:
cmd = (
[shell_path]
+ (["-x"] if metadata.config.debug else [])
+ ["-o", "errexit", test_script]
)
try:
test_stats = {}
if not provision_only:
# rewrite long paths in stdout back to their env variables
if metadata.config.debug or metadata.config.no_rewrite_stdout_env:
rewrite_env = None
else:
rewrite_env = {k: env[k] for k in ["PREFIX", "SRC_DIR"] if k in env}
if metadata.config.verbose:
for k, v in rewrite_env.items():
print(
"{0} {1}={2}".format(
"set" if test_script.endswith(".bat") else "export",
k,
v,
)
)
utils.check_call_env(
cmd,
env=env,
cwd=metadata.config.test_dir,
stats=test_stats,
rewrite_stdout_env=rewrite_env,
)
log_stats(test_stats, "testing {}".format(metadata.name()))
# TODO need to implement metadata.get_used_loop_vars
# if stats is not None and metadata.config.variants:
# stats[
# stats_key(metadata, "test_{}".format(metadata.name()))
# ] = test_stats
if os.path.exists(join(metadata.config.test_dir, "TEST_FAILED")):
raise subprocess.CalledProcessError(-1, "")
print("TEST END:", test_package_name)
except subprocess.CalledProcessError as _: # noqa
tests_failed(
metadata,
move_broken=move_broken,
broken_dir=metadata.config.broken_dir,
config=metadata.config,
)
raise
if config.need_cleanup and config.recipe_dir is not None and not provision_only:
utils.rm_rf(config.recipe_dir)
return True
def _construct_metadata_for_test_from_package(package, config):
recipe_dir, need_cleanup = utils.get_recipe_abspath(package)
config.need_cleanup = need_cleanup
config.recipe_dir = recipe_dir
hash_input = {}
info_dir = os.path.normpath(os.path.join(recipe_dir, "info"))
with open(os.path.join(info_dir, "index.json")) as f:
package_data = json.load(f)
if package_data["subdir"] != "noarch":
config.host_subdir = package_data["subdir"]
# We may be testing an (old) package built without filename hashing.
hash_input = os.path.join(info_dir, "hash_input.json")
if os.path.isfile(hash_input):
with open(os.path.join(info_dir, "hash_input.json")) as f:
hash_input = json.load(f)
else:
config.filename_hashing = False
hash_input = {}
# not actually used as a variant, since metadata will have been finalized.
# This is still necessary for computing the hash correctly though
config.variant = hash_input
log = utils.get_logger(__name__)
# get absolute file location
local_pkg_location = os.path.normpath(os.path.abspath(os.path.dirname(package)))
# get last part of the path
last_element = os.path.basename(local_pkg_location)
is_channel = False
for platform in ("win-", "linux-", "osx-", "noarch"):
if last_element.startswith(platform):
is_channel = True
if not is_channel:
log.warn(
"Copying package to conda-build croot. No packages otherwise alongside yours will"
" be available unless you specify -c local. To avoid this warning, your package "
"must reside in a channel structure with platform-subfolders. See more info on "
"what a valid channel is at "
"https://conda.io/docs/user-guide/tasks/create-custom-channels.html"
)
local_dir = os.path.join(config.croot, config.host_subdir)
mkdir_p(local_dir)
local_pkg_location = os.path.join(local_dir, os.path.basename(package))
utils.copy_into(package, local_pkg_location)
local_pkg_location = local_dir
local_channel = os.path.dirname(local_pkg_location)
# update indices in the channel
update_index(local_channel, verbose=config.debug, threads=1)
try:
# raise IOError()
# metadata = render_recipe(
# os.path.join(info_dir, "recipe"), config=config, reset_build_id=False
# )[0][0]
metadata = get_metadata(os.path.join(info_dir, "recipe", "recipe.yaml"), config)
# with open(os.path.join(info_dir, "recipe", "recipe.yaml")) as fi:
# metadata = yaml.load(fi)
# no recipe in package. Fudge metadata
except SystemExit:
# force the build string to line up - recomputing it would
# yield a different result
metadata = MetaData.fromdict(
{
"package": {
"name": package_data["name"],
"version": package_data["version"],
},
"build": {
"number": int(package_data["build_number"]),
"string": package_data["build"],
},
"requirements": {"run": package_data["depends"]},
},
config=config,
)
# HACK: because the recipe is fully baked, detecting "used" variables no longer works. The set
# of variables in the hash_input suffices, though.
if metadata.noarch:
metadata.config.variant["target_platform"] = "noarch"
metadata.config.used_vars = list(hash_input.keys())
urls = list(utils.ensure_list(metadata.config.channel_urls))
local_path = url_path(local_channel)
# replace local with the appropriate real channel. Order is maintained.
urls = [url if url != "local" else local_path for url in urls]
if local_path not in urls:
urls.insert(0, local_path)
metadata.config.channel_urls = urls
utils.rm_rf(metadata.config.test_dir)
return metadata, hash_input
def render_recipe(recipe_path,
config,
no_download_source=False,
variants=None,
permit_unsatisfiable_variants=True,
reset_build_id=True,
bypass_env_check=False):
"""Returns a list of tuples, each consisting of
(metadata-object, needs_download, needs_render_in_env)
You get one tuple per variant. Outputs are not factored in here (subpackages won't affect these
results returned here.)
"""
arg = recipe_path
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
elif arg.endswith('.yaml'):
recipe_dir = os.path.dirname(arg)
need_cleanup = False
else:
print("Ignoring non-recipe: %s" % arg)
return None, None
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir, config=config)
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
rendered_metadata = {}
# important: set build id *before* downloading source. Otherwise source goes into a different
# build folder.
if config.set_build_id:
m.config.compute_build_id(m.name(), m.version(), reset=reset_build_id)
# this source may go into a folder that doesn't match the eventual build folder.
# There's no way around it AFAICT. We must download the source to be able to render
# the recipe (from anything like GIT_FULL_HASH), but we can't know the final build
# folder until rendering is complete, because package names can have variant jinja2 in them.
if m.needs_source_for_render and not m.source_provided:
try_download(m, no_download_source=no_download_source)
if m.final:
if not hasattr(m.config, 'variants') or not m.config.variant:
m.config.ignore_system_variants = True
if os.path.isfile(os.path.join(m.path, 'conda_build_config.yaml')):
m.config.variant_config_files = [
os.path.join(m.path, 'conda_build_config.yaml')
]
m.config.variants = get_package_variants(m, variants=variants)
m.config.variant = m.config.variants[0]
rendered_metadata = [
(m, False, False),
]
else:
# merge any passed-in variants with any files found
variants = get_package_variants(m, variants=variants)
# when building, we don't want to fully expand all outputs into metadata, only expand
# whatever variants we have (i.e. expand top-level variants, not output-only variants)
rendered_metadata = distribute_variants(
m,
variants,
permit_unsatisfiable_variants=permit_unsatisfiable_variants,
allow_no_other_outputs=True,
bypass_env_check=bypass_env_check)
if need_cleanup:
utils.rm_rf(recipe_dir)
return rendered_metadata
def run_setuppy(src_dir, temp_dir, python_version, extra_specs, config,
setup_options):
'''
Patch distutils and then run setup.py in a subprocess.
:param src_dir: Directory containing the source code
:type src_dir: str
:param temp_dir: Temporary directory for doing for storing pkginfo.yaml
:type temp_dir: str
'''
# TODO: we could make everyone's lives easier if we include packaging here, because setuptools
# needs it in recent versions. At time of writing, it is not a package in defaults, so this
# actually breaks conda-build right now. Omit it until packaging is on defaults.
# specs = ['python %s*' % python_version, 'pyyaml', 'setuptools', 'six', 'packaging', 'appdirs']
specs = ['python %s*' % python_version, 'pyyaml', 'setuptools']
with open(os.path.join(src_dir, "setup.py")) as setup:
text = setup.read()
if 'import numpy' in text or 'from numpy' in text:
specs.append('numpy')
specs.extend(extra_specs)
rm_rf(config.host_prefix)
create_env(config.host_prefix,
specs_or_actions=specs,
env='host',
subdir=config.host_subdir,
clear_cache=False,
config=config)
stdlib_dir = join(
config.host_prefix,
'Lib' if sys.platform == 'win32' else 'lib/python%s' % python_version)
patch = join(temp_dir, 'pypi-distutils.patch')
with open(patch, 'w') as f:
f.write(DISTUTILS_PATCH.format(temp_dir.replace('\\', '\\\\')))
if exists(join(stdlib_dir, 'distutils', 'core.py-copy')):
rm_rf(join(stdlib_dir, 'distutils', 'core.py'))
copy2(join(stdlib_dir, 'distutils', 'core.py-copy'),
join(stdlib_dir, 'distutils', 'core.py'))
# Avoid race conditions. Invalidate the cache.
if PY3:
rm_rf(
join(stdlib_dir, 'distutils', '__pycache__',
'core.cpython-%s%s.pyc' % sys.version_info[:2]))
rm_rf(
join(stdlib_dir, 'distutils', '__pycache__',
'core.cpython-%s%s.pyo' % sys.version_info[:2]))
else:
rm_rf(join(stdlib_dir, 'distutils', 'core.pyc'))
rm_rf(join(stdlib_dir, 'distutils', 'core.pyo'))
else:
copy2(join(stdlib_dir, 'distutils', 'core.py'),
join(stdlib_dir, 'distutils', 'core.py-copy'))
apply_patch(join(stdlib_dir, 'distutils'), patch, config=config)
# Save PYTHONPATH for later
env = os.environ.copy()
if 'PYTHONPATH' in env:
env[str('PYTHONPATH')] = str(src_dir + ':' + env['PYTHONPATH'])
else:
env[str('PYTHONPATH')] = str(src_dir)
cwd = getcwd()
chdir(src_dir)
cmdargs = [config.host_python, 'setup.py', 'install']
cmdargs.extend(setup_options)
try:
check_call_env(cmdargs, env=env)
except subprocess.CalledProcessError:
print('$PYTHONPATH = %s' % env['PYTHONPATH'])
sys.exit('Error: command failed: %s' % ' '.join(cmdargs))
finally:
chdir(cwd)
def build(m, get_src=True, verbose=True):
'''
Build the package with the specified metadata.
:param m: Package metadata
:type m: Metadata
:param get_src: Should we download the source?
:type get_src: bool
'''
rm_rf(prefix)
print("BUILD START:", m.dist())
create_env(prefix, [ms.spec for ms in m.ms_depends('build')],
verbose=verbose)
if get_src:
source.provide(m.path, m.get_section('source'))
assert isdir(source.WORK_DIR)
if os.listdir(source.get_dir()):
print("source tree in:", source.get_dir())
else:
print("no source")
rm_rf(info_dir)
files1 = prefix_files()
if sys.platform == 'win32':
import conda_build.windows as windows
windows.build(m)
else:
env = environ.get_dict(m)
build_file = join(m.path, 'build.sh')
if exists(build_file):
script = m.get_value('build/script', None)
if script:
if isinstance(script, list):
script = '\n'.join(script)
with open(build_file, 'w', encoding='utf-8') as bf:
bf.write(script)
os.chmod(build_file, 0o766)
cmd = ['/bin/bash', '-x', '-e', build_file]
_check_call(cmd, env=env, cwd=source.get_dir())
get_build_metadata(m)
create_post_scripts(m)
create_entry_points(m.get_value('build/entry_points'))
post_process(preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))
assert not exists(info_dir)
files2 = prefix_files()
post_build(sorted(files2 - files1),
binary_relocation=bool(
m.get_value('build/binary_relocation', True)))
create_info_files(m, sorted(files2 - files1), include_recipe=bool(m.path))
files3 = prefix_files()
fix_permissions(files3 - files1)
path = bldpkg_path(m)
t = tarfile.open(path, 'w:bz2')
for f in sorted(files3 - files1):
t.add(join(prefix, f), f)
t.close()
print("BUILD END:", m.dist())
# we're done building, perform some checks
tarcheck.check_all(path)
update_index(config.bldpkgs_dir)
def clean_build(config, folders=None):
if not folders:
folders = get_build_folders(config.croot)
for folder in folders:
rm_rf(folder)
def cleanup():
# session scoped cleanup is called at end of the session
rm_rf(sp)
def run_setuppy(src_dir, temp_dir, python_version, config, setup_options):
'''
Patch distutils and then run setup.py in a subprocess.
:param src_dir: Directory containing the source code
:type src_dir: str
:param temp_dir: Temporary directory for doing for storing pkginfo.yaml
:type temp_dir: str
'''
specs = ['python %s*' % python_version, 'pyyaml', 'setuptools']
with open(os.path.join(src_dir, "setup.py")) as setup:
text = setup.read()
if 'import numpy' in text or 'from numpy' in text:
specs.append('numpy')
# Do everything in the build env in case the setup.py install goes
# haywire.
# TODO: Try with another version of Python if this one fails. Some
# packages are Python 2 or Python 3 only.
create_env(config.build_prefix, specs=specs,
clear_cache=False,
config=config)
stdlib_dir = join(config.build_prefix,
'Lib' if sys.platform == 'win32'
else 'lib/python%s' % python_version)
patch = join(temp_dir, 'pypi-distutils.patch')
with open(patch, 'w') as f:
f.write(DISTUTILS_PATCH.format(temp_dir.replace('\\', '\\\\')))
if exists(join(stdlib_dir, 'distutils', 'core.py-copy')):
rm_rf(join(stdlib_dir, 'distutils', 'core.py'))
copy2(join(stdlib_dir, 'distutils', 'core.py-copy'),
join(stdlib_dir, 'distutils', 'core.py'))
# Avoid race conditions. Invalidate the cache.
if PY3:
rm_rf(join(stdlib_dir, 'distutils', '__pycache__',
'core.cpython-%s%s.pyc' % sys.version_info[:2]))
rm_rf(join(stdlib_dir, 'distutils', '__pycache__',
'core.cpython-%s%s.pyo' % sys.version_info[:2]))
else:
rm_rf(join(stdlib_dir, 'distutils', 'core.pyc'))
rm_rf(join(stdlib_dir, 'distutils', 'core.pyo'))
else:
copy2(join(stdlib_dir, 'distutils', 'core.py'), join(stdlib_dir,
'distutils', 'core.py-copy'))
apply_patch(join(stdlib_dir, 'distutils'), patch, config=config)
# Save PYTHONPATH for later
env = os.environ.copy()
if 'PYTHONPATH' in env:
env[str('PYTHONPATH')] = str(src_dir + ':' + env['PYTHONPATH'])
else:
env[str('PYTHONPATH')] = str(src_dir)
cwd = getcwd()
chdir(src_dir)
cmdargs = [config.build_python, 'setup.py', 'install']
cmdargs.extend(setup_options)
try:
subprocess.check_call(cmdargs, env=env)
except subprocess.CalledProcessError:
print('$PYTHONPATH = %s' % env['PYTHONPATH'])
sys.exit('Error: command failed: %s' % ' '.join(cmdargs))
finally:
chdir(cwd)
