def expand_outputs(metadata_tuples):
"""Obtain all metadata objects for all outputs from recipe. Useful for outputting paths."""
expanded_outputs = OrderedDict()
for (_m, download, reparse) in metadata_tuples:
from conda_build.build import get_all_replacements
get_all_replacements(_m.config)
from copy import deepcopy
for (output_dict, m) in deepcopy(_m).get_output_metadata_set(
permit_unsatisfiable_variants=False):
get_all_replacements(m.config)
expanded_outputs[m.dist()] = (output_dict, m)
return list(expanded_outputs.values())
def execute(args, print_results=True):
p, args = parse_args(args)
config = get_or_merge_config(None, **args.__dict__)
variants = get_package_variants(args.recipe, config, variants=args.variants)
from conda_build.build import get_all_replacements
get_all_replacements(variants)
set_language_env_vars(variants)
config.channel_urls = get_channel_urls(args.__dict__)
config.override_channels = args.override_channels
if args.output:
config.verbose = False
config.debug = False
metadata_tuples = api.render(args.recipe, config=config,
no_download_source=args.no_source,
variants=args.variants)
if args.file and len(metadata_tuples) > 1:
log.warning("Multiple variants rendered. "
"Only one will be written to the file you specified ({}).".format(args.file))
if print_results:
if args.output:
with LoggingContext(logging.CRITICAL + 1):
paths = api.get_output_file_paths(metadata_tuples, config=config)
print('\n'.join(sorted(paths)))
if args.file:
m = metadata_tuples[-1][0]
api.output_yaml(m, args.file, suppress_outputs=True)
else:
logging.basicConfig(level=logging.INFO)
for (m, _, _) in metadata_tuples:
print("--------------")
print("Hash contents:")
print("--------------")
pprint(m.get_hash_contents())
print("----------")
print("meta.yaml:")
print("----------")
print(api.output_yaml(m, args.file, suppress_outputs=True))
else:
return metadata_tuples
def distribute_variants(metadata,
variants,
permit_unsatisfiable_variants=False,
allow_no_other_outputs=False,
bypass_env_check=False):
rendered_metadata = {}
need_source_download = True
# don't bother distributing python if it's a noarch package, and figure out
# which python version we prefer. `python_age` can use used to tweak which
# python gets used here.
if metadata.noarch or metadata.noarch_python:
from .conda_interface import VersionOrder
age = int(
metadata.get_value('build/noarch_python_build_age',
metadata.config.noarch_python_build_age))
versions = []
for variant in variants:
if 'python' in variant:
vo = variant['python']
if vo not in versions:
versions.append(vo)
version_indices = sorted(
range(len(versions)),
key=lambda k: VersionOrder(versions[k].split(' ')[0]))
if age < 0:
age = 0
elif age > len(versions) - 1:
age = len(versions) - 1
build_ver = versions[version_indices[len(versions) - 1 - age]]
variants = filter_by_key_value(variants, 'python', build_ver,
'noarch_python_reduction')
# store these for reference later
metadata.config.variants = variants
# These are always the full set. just 'variants' is the one that gets
# used mostly, and can be reduced
metadata.config.input_variants = variants
recipe_requirements = metadata.extract_requirements_text()
recipe_package_and_build_text = metadata.extract_package_and_build_text()
recipe_text = recipe_package_and_build_text + recipe_requirements
if PY3 and hasattr(recipe_text, 'decode'):
recipe_text = recipe_text.decode()
elif not PY3 and hasattr(recipe_text, 'encode'):
recipe_text = recipe_text.encode()
metadata.config.variant = variants[0]
used_variables = metadata.get_used_loop_vars(force_global=False)
top_loop = metadata.get_reduced_variant_set(used_variables)
for variant in top_loop:
from conda_build.build import get_all_replacements
get_all_replacements(variant)
mv = metadata.copy()
mv.config.variant = variant
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
conform_dict = {}
for key in used_variables:
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
conform_dict[key] = variant[key]
for key, values in conform_dict.items():
mv.config.variants = (filter_by_key_value(
mv.config.variants, key, values,
'distribute_variants_reduction') or mv.config.variants)
get_all_replacements(mv.config.variants)
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
numpy_pinned_variants = []
for _variant in mv.config.variants:
_variant['pin_run_as_build'] = pin_run_as_build
numpy_pinned_variants.append(_variant)
mv.config.variants = numpy_pinned_variants
mv.config.squished_variants = list_of_dicts_to_dict_of_lists(
mv.config.variants)
if mv.needs_source_for_render and mv.variant_in_source:
mv.parse_again()
utils.rm_rf(mv.config.work_dir)
source.provide(mv)
mv.parse_again()
try:
mv.parse_until_resolved(
allow_no_other_outputs=allow_no_other_outputs,
bypass_env_check=bypass_env_check)
except SystemExit:
pass
need_source_download = (not mv.needs_source_for_render
or not mv.source_provided)
rendered_metadata[(mv.dist(),
mv.config.variant.get('target_platform', mv.config.subdir),
tuple((var, mv.config.variant.get(var))
for var in mv.get_used_vars()))] = \
(mv, need_source_download, None)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
def run_test(
recipedir_or_package_or_metadata,
config,
stats,
move_broken=True,
provision_only=False,
solver=None,
):
"""
Execute any test scripts for the given package.
:param m: Package's metadata.
:type m: Metadata
"""
# we want to know if we're dealing with package input. If so, we can move the input on success.
hash_input = {}
# store this name to keep it consistent. By changing files, we change the hash later.
# It matches the build hash now, so let's keep it around.
test_package_name = (
recipedir_or_package_or_metadata.dist()
if hasattr(recipedir_or_package_or_metadata, "dist")
else recipedir_or_package_or_metadata
)
if not provision_only:
print("TEST START:", test_package_name)
if hasattr(recipedir_or_package_or_metadata, "config"):
metadata = recipedir_or_package_or_metadata
utils.rm_rf(metadata.config.test_dir)
else:
metadata, hash_input = construct_metadata_for_test(
recipedir_or_package_or_metadata, config
)
trace = "-x " if metadata.config.debug else ""
# Must download *after* computing build id, or else computing build id will change
# folder destination
_extract_test_files_from_package(metadata)
# When testing a .tar.bz2 in the pkgs dir, clean_pkg_cache() will remove it.
# Prevent this. When https://github.com/conda/conda/issues/5708 gets fixed
# I think we can remove this call to clean_pkg_cache().
in_pkg_cache = (
not hasattr(recipedir_or_package_or_metadata, "config")
and os.path.isfile(recipedir_or_package_or_metadata)
and recipedir_or_package_or_metadata.endswith(CONDA_PACKAGE_EXTENSIONS)
and os.path.dirname(recipedir_or_package_or_metadata) in pkgs_dirs[0]
)
if not in_pkg_cache:
environ.clean_pkg_cache(metadata.dist(), metadata.config)
copy_test_source_files(metadata, metadata.config.test_dir)
# this is also copying tests/source_files from work_dir to testing workdir
_, pl_files, py_files, r_files, lua_files, shell_files = create_all_test_files(
metadata
)
if (
not any([py_files, shell_files, pl_files, lua_files, r_files])
and not metadata.config.test_run_post
):
print("Nothing to test for:", test_package_name)
return True
if metadata.config.remove_work_dir:
for name, prefix in (
("host", metadata.config.host_prefix),
("build", metadata.config.build_prefix),
):
if os.path.isdir(prefix):
# move host folder to force hardcoded paths to host env to break during tests
# (so that they can be properly addressed by recipe author)
dest = os.path.join(
os.path.dirname(prefix),
"_".join(
(
"%s_prefix_moved" % name,
metadata.dist(),
getattr(metadata.config, "%s_subdir" % name),
)
),
)
# Needs to come after create_files in case there's test/source_files
shutil_move_more_retrying(prefix, dest, "{} prefix".format(prefix))
# nested if so that there's no warning when we just leave the empty workdir in place
if metadata.source_provided:
dest = os.path.join(
os.path.dirname(metadata.config.work_dir),
"_".join(("work_moved", metadata.dist(), metadata.config.host_subdir)),
)
# Needs to come after create_files in case there's test/source_files
shutil_move_more_retrying(config.work_dir, dest, "work")
else:
log.warn(
"Not moving work directory after build. Your package may depend on files "
"in the work directory that are not included with your package"
)
# looks like a dead function to me
# get_build_metadata(metadata)
specs = metadata.get_test_deps(py_files, pl_files, lua_files, r_files)
with utils.path_prepended(metadata.config.test_prefix):
env = dict(os.environ.copy())
env.update(environ.get_dict(m=metadata, prefix=config.test_prefix))
env["CONDA_BUILD_STATE"] = "TEST"
env["CONDA_BUILD"] = "1"
if env_path_backup_var_exists:
env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]
if not metadata.config.activate or metadata.name() == "conda":
# prepend bin (or Scripts) directory
env = utils.prepend_bin_path(
env, metadata.config.test_prefix, prepend_prefix=True
)
if utils.on_win:
env["PATH"] = metadata.config.test_prefix + os.pathsep + env["PATH"]
env["PREFIX"] = metadata.config.test_prefix
if "BUILD_PREFIX" in env:
del env["BUILD_PREFIX"]
# In the future, we will need to support testing cross compiled
# packages on physical hardware. until then it is expected that
# something like QEMU or Wine will be used on the build machine,
# therefore, for now, we use host_subdir.
# ensure that the test prefix isn't kept between variants
utils.rm_rf(metadata.config.test_prefix)
if solver is None:
solver, pkg_cache_path = get_solver(metadata.config.host_subdir)
else:
pkg_cache_path = PackageCacheData.first_writable().pkgs_dir
solver.replace_channels()
transaction = solver.solve(specs, [pkg_cache_path])
downloaded = transaction.fetch_extract_packages(
pkg_cache_path, solver.repos + list(solver.local_repos.values()),
)
if not downloaded:
raise RuntimeError("Did not succeed in downloading packages.")
mkdir_p(metadata.config.test_prefix)
transaction.execute(
PrefixData(metadata.config.test_prefix), pkg_cache_path,
)
with utils.path_prepended(metadata.config.test_prefix):
env = dict(os.environ.copy())
env.update(environ.get_dict(m=metadata, prefix=metadata.config.test_prefix))
env["CONDA_BUILD_STATE"] = "TEST"
if env_path_backup_var_exists:
env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]
if config.test_run_post:
from conda_build.utils import get_installed_packages
installed = get_installed_packages(metadata.config.test_prefix)
files = installed[metadata.meta["package"]["name"]]["files"]
replacements = get_all_replacements(metadata.config)
try_download(metadata, False, True)
create_info_files(metadata, replacements, files, metadata.config.test_prefix)
post_build(metadata, files, None, metadata.config.test_prefix, True)
# when workdir is removed, the source files are unavailable. There's the test/source_files
# entry that lets people keep these files around. The files are copied into test_dir for
# intuitive relative path behavior, though, not work_dir, so we need to adjust where
# SRC_DIR points. The initial CWD during tests is test_dir.
if metadata.config.remove_work_dir:
env["SRC_DIR"] = metadata.config.test_dir
test_script, _ = write_test_scripts(
metadata, env, py_files, pl_files, lua_files, r_files, shell_files, trace
)
if utils.on_win:
cmd = [os.environ.get("COMSPEC", "cmd.exe"), "/d", "/c", test_script]
else:
cmd = (
[shell_path]
+ (["-x"] if metadata.config.debug else [])
+ ["-o", "errexit", test_script]
)
try:
test_stats = {}
if not provision_only:
# rewrite long paths in stdout back to their env variables
if metadata.config.debug or metadata.config.no_rewrite_stdout_env:
rewrite_env = None
else:
rewrite_env = {k: env[k] for k in ["PREFIX", "SRC_DIR"] if k in env}
if metadata.config.verbose:
for k, v in rewrite_env.items():
print(
"{0} {1}={2}".format(
"set" if test_script.endswith(".bat") else "export",
k,
v,
)
)
utils.check_call_env(
cmd,
env=env,
cwd=metadata.config.test_dir,
stats=test_stats,
rewrite_stdout_env=rewrite_env,
)
log_stats(test_stats, "testing {}".format(metadata.name()))
# TODO need to implement metadata.get_used_loop_vars
# if stats is not None and metadata.config.variants:
# stats[
# stats_key(metadata, "test_{}".format(metadata.name()))
# ] = test_stats
if os.path.exists(join(metadata.config.test_dir, "TEST_FAILED")):
raise subprocess.CalledProcessError(-1, "")
print("TEST END:", test_package_name)
except subprocess.CalledProcessError as _: # noqa
tests_failed(
metadata,
move_broken=move_broken,
broken_dir=metadata.config.broken_dir,
config=metadata.config,
)
raise
if config.need_cleanup and config.recipe_dir is not None and not provision_only:
utils.rm_rf(config.recipe_dir)
return True
