def expand_outputs(metadata_tuples): """Obtain all metadata objects for all outputs from recipe. Useful for outputting paths.""" expanded_outputs = OrderedDict() for (_m, download, reparse) in metadata_tuples: from conda_build.build import get_all_replacements get_all_replacements(_m.config) from copy import deepcopy for (output_dict, m) in deepcopy(_m).get_output_metadata_set( permit_unsatisfiable_variants=False): get_all_replacements(m.config) expanded_outputs[m.dist()] = (output_dict, m) return list(expanded_outputs.values())
def execute(args, print_results=True): p, args = parse_args(args) config = get_or_merge_config(None, **args.__dict__) variants = get_package_variants(args.recipe, config, variants=args.variants) from conda_build.build import get_all_replacements get_all_replacements(variants) set_language_env_vars(variants) config.channel_urls = get_channel_urls(args.__dict__) config.override_channels = args.override_channels if args.output: config.verbose = False config.debug = False metadata_tuples = api.render(args.recipe, config=config, no_download_source=args.no_source, variants=args.variants) if args.file and len(metadata_tuples) > 1: log.warning("Multiple variants rendered. " "Only one will be written to the file you specified ({}).".format(args.file)) if print_results: if args.output: with LoggingContext(logging.CRITICAL + 1): paths = api.get_output_file_paths(metadata_tuples, config=config) print('\n'.join(sorted(paths))) if args.file: m = metadata_tuples[-1][0] api.output_yaml(m, args.file, suppress_outputs=True) else: logging.basicConfig(level=logging.INFO) for (m, _, _) in metadata_tuples: print("--------------") print("Hash contents:") print("--------------") pprint(m.get_hash_contents()) print("----------") print("meta.yaml:") print("----------") print(api.output_yaml(m, args.file, suppress_outputs=True)) else: return metadata_tuples
def distribute_variants(metadata, variants, permit_unsatisfiable_variants=False, allow_no_other_outputs=False, bypass_env_check=False): rendered_metadata = {} need_source_download = True # don't bother distributing python if it's a noarch package, and figure out # which python version we prefer. `python_age` can use used to tweak which # python gets used here. if metadata.noarch or metadata.noarch_python: from .conda_interface import VersionOrder age = int( metadata.get_value('build/noarch_python_build_age', metadata.config.noarch_python_build_age)) versions = [] for variant in variants: if 'python' in variant: vo = variant['python'] if vo not in versions: versions.append(vo) version_indices = sorted( range(len(versions)), key=lambda k: VersionOrder(versions[k].split(' ')[0])) if age < 0: age = 0 elif age > len(versions) - 1: age = len(versions) - 1 build_ver = versions[version_indices[len(versions) - 1 - age]] variants = filter_by_key_value(variants, 'python', build_ver, 'noarch_python_reduction') # store these for reference later metadata.config.variants = variants # These are always the full set. just 'variants' is the one that gets # used mostly, and can be reduced metadata.config.input_variants = variants recipe_requirements = metadata.extract_requirements_text() recipe_package_and_build_text = metadata.extract_package_and_build_text() recipe_text = recipe_package_and_build_text + recipe_requirements if PY3 and hasattr(recipe_text, 'decode'): recipe_text = recipe_text.decode() elif not PY3 and hasattr(recipe_text, 'encode'): recipe_text = recipe_text.encode() metadata.config.variant = variants[0] used_variables = metadata.get_used_loop_vars(force_global=False) top_loop = metadata.get_reduced_variant_set(used_variables) for variant in top_loop: from conda_build.build import get_all_replacements get_all_replacements(variant) mv = metadata.copy() mv.config.variant = variant pin_run_as_build = variant.get('pin_run_as_build', {}) if mv.numpy_xx and 'numpy' not in pin_run_as_build: pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'} conform_dict = {} for key in used_variables: # We use this variant in the top-level recipe. # constrain the stored variants to only this version in the output # variant mapping conform_dict[key] = variant[key] for key, values in conform_dict.items(): mv.config.variants = (filter_by_key_value( mv.config.variants, key, values, 'distribute_variants_reduction') or mv.config.variants) get_all_replacements(mv.config.variants) pin_run_as_build = variant.get('pin_run_as_build', {}) if mv.numpy_xx and 'numpy' not in pin_run_as_build: pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'} numpy_pinned_variants = [] for _variant in mv.config.variants: _variant['pin_run_as_build'] = pin_run_as_build numpy_pinned_variants.append(_variant) mv.config.variants = numpy_pinned_variants mv.config.squished_variants = list_of_dicts_to_dict_of_lists( mv.config.variants) if mv.needs_source_for_render and mv.variant_in_source: mv.parse_again() utils.rm_rf(mv.config.work_dir) source.provide(mv) mv.parse_again() try: mv.parse_until_resolved( allow_no_other_outputs=allow_no_other_outputs, bypass_env_check=bypass_env_check) except SystemExit: pass need_source_download = (not mv.needs_source_for_render or not mv.source_provided) rendered_metadata[(mv.dist(), mv.config.variant.get('target_platform', mv.config.subdir), tuple((var, mv.config.variant.get(var)) for var in mv.get_used_vars()))] = \ (mv, need_source_download, None) # list of tuples. # each tuple item is a tuple of 3 items: # metadata, need_download, need_reparse_in_env return list(rendered_metadata.values())
def run_test( recipedir_or_package_or_metadata, config, stats, move_broken=True, provision_only=False, solver=None, ): """ Execute any test scripts for the given package. :param m: Package's metadata. :type m: Metadata """ # we want to know if we're dealing with package input. If so, we can move the input on success. hash_input = {} # store this name to keep it consistent. By changing files, we change the hash later. # It matches the build hash now, so let's keep it around. test_package_name = ( recipedir_or_package_or_metadata.dist() if hasattr(recipedir_or_package_or_metadata, "dist") else recipedir_or_package_or_metadata ) if not provision_only: print("TEST START:", test_package_name) if hasattr(recipedir_or_package_or_metadata, "config"): metadata = recipedir_or_package_or_metadata utils.rm_rf(metadata.config.test_dir) else: metadata, hash_input = construct_metadata_for_test( recipedir_or_package_or_metadata, config ) trace = "-x " if metadata.config.debug else "" # Must download *after* computing build id, or else computing build id will change # folder destination _extract_test_files_from_package(metadata) # When testing a .tar.bz2 in the pkgs dir, clean_pkg_cache() will remove it. # Prevent this. When https://github.com/conda/conda/issues/5708 gets fixed # I think we can remove this call to clean_pkg_cache(). in_pkg_cache = ( not hasattr(recipedir_or_package_or_metadata, "config") and os.path.isfile(recipedir_or_package_or_metadata) and recipedir_or_package_or_metadata.endswith(CONDA_PACKAGE_EXTENSIONS) and os.path.dirname(recipedir_or_package_or_metadata) in pkgs_dirs[0] ) if not in_pkg_cache: environ.clean_pkg_cache(metadata.dist(), metadata.config) copy_test_source_files(metadata, metadata.config.test_dir) # this is also copying tests/source_files from work_dir to testing workdir _, pl_files, py_files, r_files, lua_files, shell_files = create_all_test_files( metadata ) if ( not any([py_files, shell_files, pl_files, lua_files, r_files]) and not metadata.config.test_run_post ): print("Nothing to test for:", test_package_name) return True if metadata.config.remove_work_dir: for name, prefix in ( ("host", metadata.config.host_prefix), ("build", metadata.config.build_prefix), ): if os.path.isdir(prefix): # move host folder to force hardcoded paths to host env to break during tests # (so that they can be properly addressed by recipe author) dest = os.path.join( os.path.dirname(prefix), "_".join( ( "%s_prefix_moved" % name, metadata.dist(), getattr(metadata.config, "%s_subdir" % name), ) ), ) # Needs to come after create_files in case there's test/source_files shutil_move_more_retrying(prefix, dest, "{} prefix".format(prefix)) # nested if so that there's no warning when we just leave the empty workdir in place if metadata.source_provided: dest = os.path.join( os.path.dirname(metadata.config.work_dir), "_".join(("work_moved", metadata.dist(), metadata.config.host_subdir)), ) # Needs to come after create_files in case there's test/source_files shutil_move_more_retrying(config.work_dir, dest, "work") else: log.warn( "Not moving work directory after build. Your package may depend on files " "in the work directory that are not included with your package" ) # looks like a dead function to me # get_build_metadata(metadata) specs = metadata.get_test_deps(py_files, pl_files, lua_files, r_files) with utils.path_prepended(metadata.config.test_prefix): env = dict(os.environ.copy()) env.update(environ.get_dict(m=metadata, prefix=config.test_prefix)) env["CONDA_BUILD_STATE"] = "TEST" env["CONDA_BUILD"] = "1" if env_path_backup_var_exists: env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] if not metadata.config.activate or metadata.name() == "conda": # prepend bin (or Scripts) directory env = utils.prepend_bin_path( env, metadata.config.test_prefix, prepend_prefix=True ) if utils.on_win: env["PATH"] = metadata.config.test_prefix + os.pathsep + env["PATH"] env["PREFIX"] = metadata.config.test_prefix if "BUILD_PREFIX" in env: del env["BUILD_PREFIX"] # In the future, we will need to support testing cross compiled # packages on physical hardware. until then it is expected that # something like QEMU or Wine will be used on the build machine, # therefore, for now, we use host_subdir. # ensure that the test prefix isn't kept between variants utils.rm_rf(metadata.config.test_prefix) if solver is None: solver, pkg_cache_path = get_solver(metadata.config.host_subdir) else: pkg_cache_path = PackageCacheData.first_writable().pkgs_dir solver.replace_channels() transaction = solver.solve(specs, [pkg_cache_path]) downloaded = transaction.fetch_extract_packages( pkg_cache_path, solver.repos + list(solver.local_repos.values()), ) if not downloaded: raise RuntimeError("Did not succeed in downloading packages.") mkdir_p(metadata.config.test_prefix) transaction.execute( PrefixData(metadata.config.test_prefix), pkg_cache_path, ) with utils.path_prepended(metadata.config.test_prefix): env = dict(os.environ.copy()) env.update(environ.get_dict(m=metadata, prefix=metadata.config.test_prefix)) env["CONDA_BUILD_STATE"] = "TEST" if env_path_backup_var_exists: env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] if config.test_run_post: from conda_build.utils import get_installed_packages installed = get_installed_packages(metadata.config.test_prefix) files = installed[metadata.meta["package"]["name"]]["files"] replacements = get_all_replacements(metadata.config) try_download(metadata, False, True) create_info_files(metadata, replacements, files, metadata.config.test_prefix) post_build(metadata, files, None, metadata.config.test_prefix, True) # when workdir is removed, the source files are unavailable. There's the test/source_files # entry that lets people keep these files around. The files are copied into test_dir for # intuitive relative path behavior, though, not work_dir, so we need to adjust where # SRC_DIR points. The initial CWD during tests is test_dir. if metadata.config.remove_work_dir: env["SRC_DIR"] = metadata.config.test_dir test_script, _ = write_test_scripts( metadata, env, py_files, pl_files, lua_files, r_files, shell_files, trace ) if utils.on_win: cmd = [os.environ.get("COMSPEC", "cmd.exe"), "/d", "/c", test_script] else: cmd = ( [shell_path] + (["-x"] if metadata.config.debug else []) + ["-o", "errexit", test_script] ) try: test_stats = {} if not provision_only: # rewrite long paths in stdout back to their env variables if metadata.config.debug or metadata.config.no_rewrite_stdout_env: rewrite_env = None else: rewrite_env = {k: env[k] for k in ["PREFIX", "SRC_DIR"] if k in env} if metadata.config.verbose: for k, v in rewrite_env.items(): print( "{0} {1}={2}".format( "set" if test_script.endswith(".bat") else "export", k, v, ) ) utils.check_call_env( cmd, env=env, cwd=metadata.config.test_dir, stats=test_stats, rewrite_stdout_env=rewrite_env, ) log_stats(test_stats, "testing {}".format(metadata.name())) # TODO need to implement metadata.get_used_loop_vars # if stats is not None and metadata.config.variants: # stats[ # stats_key(metadata, "test_{}".format(metadata.name())) # ] = test_stats if os.path.exists(join(metadata.config.test_dir, "TEST_FAILED")): raise subprocess.CalledProcessError(-1, "") print("TEST END:", test_package_name) except subprocess.CalledProcessError as _: # noqa tests_failed( metadata, move_broken=move_broken, broken_dir=metadata.config.broken_dir, config=metadata.config, ) raise if config.need_cleanup and config.recipe_dir is not None and not provision_only: utils.rm_rf(config.recipe_dir) return True