def system_vars(env_dict, prefix, config): d = dict() compiler_vars = defaultdict(text_type) # note the dictionary is passed in here - variables are set in that dict if they are non-null get_default = lambda key, default='': set_from_os_or_variant( d, key, config.variant, default) get_default('CPU_COUNT', get_cpu_count()) get_default('LANG') get_default('LC_ALL') get_default('MAKEFLAGS') d['SHLIB_EXT'] = get_shlib_ext() d['PATH'] = os.environ.copy()['PATH'] if not config.activate: d = prepend_bin_path(d, prefix) if sys.platform == 'win32': windows_vars(prefix, config, get_default) else: unix_vars(prefix, get_default) if sys.platform == 'darwin': osx_vars(compiler_vars, config, get_default) elif sys.platform.startswith('linux'): linux_vars(compiler_vars, config, get_default) # make sure compiler_vars get appended to anything already set, including build/script_env for key in compiler_vars: if key in env_dict and env_dict[key]: compiler_vars[key] += env_dict[key] d.update(compiler_vars) return d
def system_vars(env_dict, m, prefix): d = dict() # note the dictionary is passed in here - variables are set in that dict if they are non-null get_default = lambda key, default='': set_from_os_or_variant( d, key, m.config.variant, default) get_default('CPU_COUNT', get_cpu_count()) get_default('LANG') get_default('LC_ALL') get_default('MAKEFLAGS') d['SHLIB_EXT'] = get_shlib_ext(m.config.host_platform) d['PATH'] = os.environ.copy()['PATH'] if not m.config.activate: d = prepend_bin_path(d, m.config.host_prefix) if sys.platform == 'win32': windows_vars(m, get_default, prefix) else: unix_vars(m, get_default, prefix) if m.config.host_platform == "osx": osx_vars(m, get_default, prefix) elif m.config.host_platform == "linux": linux_vars(m, get_default, prefix) return d
def system_vars(env_dict, prefix, config): d = dict() compiler_vars = defaultdict(text_type) # note the dictionary is passed in here - variables are set in that dict if they are non-null get_default = lambda key, default='': set_from_os_or_variant(d, key, config.variant, default) get_default('CPU_COUNT', get_cpu_count()) get_default('LANG') get_default('LC_ALL') get_default('MAKEFLAGS') d['SHLIB_EXT'] = get_shlib_ext() d['PATH'] = os.environ.copy()['PATH'] if not config.activate: d = prepend_bin_path(d, prefix) if sys.platform == 'win32': windows_vars(prefix, config, get_default) else: unix_vars(prefix, get_default) if sys.platform == 'darwin': osx_vars(compiler_vars, config, get_default) elif sys.platform.startswith('linux'): linux_vars(compiler_vars, config, get_default) # make sure compiler_vars get appended to anything already set, including build/script_env for key in compiler_vars: if key in env_dict and env_dict[key]: compiler_vars[key] += env_dict[key] d.update(compiler_vars) return d
def system_vars(env_dict, prefix, config): d = dict() compiler_vars = defaultdict(text_type) if 'MAKEFLAGS' in os.environ: d['MAKEFLAGS'] = os.environ['MAKEFLAGS'] if 'CPU_COUNT' in os.environ: d['CPU_COUNT'] = os.environ['CPU_COUNT'] else: d['CPU_COUNT'] = get_cpu_count() if "LANG" in os.environ: d['LANG'] = os.environ['LANG'] d['PATH'] = os.environ.copy()['PATH'] if not config.activate: d = prepend_bin_path(d, prefix) if sys.platform == 'win32': d.update(windows_vars(prefix)) else: d.update(unix_vars(prefix)) if sys.platform == 'darwin': d.update(osx_vars(compiler_vars, config)) elif sys.platform.startswith('linux'): d.update(linux_vars(compiler_vars, prefix, config)) # make sure compiler_vars get appended to anything already set, including build/script_env for key in compiler_vars: if key in env_dict: compiler_vars[key] += env_dict[key] d.update(compiler_vars) return d
def write_test_scripts( metadata, env_vars, py_files, pl_files, lua_files, r_files, shell_files, trace="" ): if not metadata.config.activate or metadata.name() == "conda": # prepend bin (or Scripts) directory env_vars = utils.prepend_bin_path( env_vars, metadata.config.test_prefix, prepend_prefix=True ) if utils.on_win: env_vars["PATH"] = ( metadata.config.test_prefix + os.pathsep + env_vars["PATH"] ) # set variables like CONDA_PY in the test environment env_vars.update(set_language_env_vars(metadata.config.variant)) # Python 2 Windows requires that envs variables be string, not unicode env_vars = {str(key): str(value) for key, value in env_vars.items()} suffix = "bat" if utils.on_win else "sh" test_env_script = join( metadata.config.test_dir, "conda_test_env_vars.{suffix}".format(suffix=suffix) ) test_run_script = join( metadata.config.test_dir, "conda_test_runner.{suffix}".format(suffix=suffix) ) with open(test_env_script, "w") as tf: if not utils.on_win: tf.write("set {trace}-e\n".format(trace=trace)) if metadata.config.activate and not metadata.name() == "conda": ext = ".bat" if utils.on_win else "" tf.write( '{source} "{conda_root}activate{ext}" "{test_env}"\n'.format( conda_root=utils.root_script_dir + os.path.sep, source="call" if utils.on_win else "source", ext=ext, test_env=metadata.config.test_prefix, ) ) if utils.on_win: tf.write("IF %ERRORLEVEL% NEQ 0 exit /B 1\n") # In-case people source this, it's essential errors are not fatal in an interactive shell. if not utils.on_win: tf.write("set +e\n") _write_test_run_script( metadata, test_run_script, test_env_script, py_files, pl_files, lua_files, r_files, shell_files, trace, ) return test_run_script, test_env_script
def testing_env(testing_workdir, request, monkeypatch): env_path = os.path.join(testing_workdir, 'env') check_call_env(['conda', 'create', '-yq', '-p', env_path, 'python={0}'.format(".".join(sys.version.split('.')[:2]))]) monkeypatch.setenv('PATH', prepend_bin_path(os.environ.copy(), env_path, prepend_prefix=True)['PATH']) # cleanup is done by just cleaning up the testing_workdir return env_path
def testing_env(testing_workdir, request): env_path = os.path.join(testing_workdir, 'env') subprocess.check_call(['conda', 'create', '-yq', '-p', env_path, 'python={0}'.format(".".join(sys.version.split('.')[:2]))]) path_backup = os.environ['PATH'] os.environ['PATH'] = prepend_bin_path(os.environ.copy(), env_path, prepend_prefix=True)['PATH'] # cleanup is done by just cleaning up the testing_workdir def reset_path(): os.environ['PATH'] = path_backup request.addfinalizer(reset_path) return env_path
def system_vars(env_dict, prefix, config): d = dict() compiler_vars = defaultdict(text_type) if "MAKEFLAGS" in os.environ: d["MAKEFLAGS"] = os.environ["MAKEFLAGS"] if "CPU_COUNT" in os.environ: d["CPU_COUNT"] = os.environ["CPU_COUNT"] else: d["CPU_COUNT"] = get_cpu_count() d["SHLIB_EXT"] = get_shlib_ext() if "LANG" in os.environ: d["LANG"] = os.environ["LANG"] d["PATH"] = os.environ.copy()["PATH"] if not config.activate: d = prepend_bin_path(d, prefix) if sys.platform == "win32": d.update(windows_vars(prefix)) else: d.update(unix_vars(prefix)) if sys.platform == "darwin": d.update(osx_vars(compiler_vars, config)) elif sys.platform.startswith("linux"): d.update(linux_vars(compiler_vars, prefix, config)) # make sure compiler_vars get appended to anything already set, including build/script_env for key in compiler_vars: if key in env_dict: compiler_vars[key] += env_dict[key] d.update(compiler_vars) return d
def test(m, config, move_broken=True): ''' Execute any test scripts for the given package. :param m: Package's metadata. :type m: Metadata ''' if not os.path.isdir(config.build_folder): os.makedirs(config.build_folder) clean_pkg_cache(m.dist(), config.timeout) with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"), timeout=config.timeout): tmp_dir = config.test_dir if not isdir(tmp_dir): os.makedirs(tmp_dir) create_files(tmp_dir, m, config) # Make Perl or Python-specific test files if m.name().startswith('perl-'): pl_files = create_pl_files(tmp_dir, m) py_files = False lua_files = False else: py_files = create_py_files(tmp_dir, m) pl_files = False lua_files = False shell_files = create_shell_files(tmp_dir, m, config) if not (py_files or shell_files or pl_files or lua_files): print("Nothing to test for:", m.dist()) return print("TEST START:", m.dist()) get_build_metadata(m, config=config) specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())] # add packages listed in the run environment and test/requires specs.extend(ms.spec for ms in m.ms_depends('run')) specs += m.get_value('test/requires', []) if py_files: # as the tests are run by python, ensure that python is installed. # (If they already provided python as a run or test requirement, # this won't hurt anything.) specs += ['python %s*' % environ.get_py_ver(config)] if pl_files: # as the tests are run by perl, we need to specify it specs += ['perl %s*' % environ.get_perl_ver(config)] if lua_files: # not sure how this shakes out specs += ['lua %s*' % environ.get_lua_ver(config)] create_env(config.test_prefix, specs, config=config) with path_prepended(config.test_prefix): env = dict(os.environ.copy()) env.update( environ.get_dict(config=config, m=m, prefix=config.test_prefix)) env["CONDA_BUILD_STATE"] = "TEST" if not config.activate: # prepend bin (or Scripts) directory env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True) if on_win: env['PATH'] = config.test_prefix + os.pathsep + env['PATH'] for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA': env[varname] = str(getattr(config, varname) or '') # Python 2 Windows requires that envs variables be string, not unicode env = {str(key): str(value) for key, value in env.items()} suffix = "bat" if on_win else "sh" test_script = join(tmp_dir, "conda_test_runner.{suffix}".format(suffix=suffix)) with open(test_script, 'w') as tf: if config.activate: ext = ".bat" if on_win else "" tf.write( "{source} {conda_root}activate{ext} {test_env} {squelch}\n" .format(conda_root=root_script_dir + os.path.sep, source="call" if on_win else "source", ext=ext, test_env=config.test_prefix, squelch=">nul 2>&1" if on_win else "&> /dev/null")) if on_win: tf.write("if errorlevel 1 exit 1\n") if py_files: tf.write("{python} -s {test_file}\n".format( python=config.test_python, test_file=join(tmp_dir, 'run_test.py'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if pl_files: tf.write("{perl} {test_file}\n".format(perl=config.test_perl, test_file=join( tmp_dir, 'run_test.pl'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if lua_files: tf.write("{lua} {test_file}\n".format(lua=config.test_lua, test_file=join( tmp_dir, 'run_test.lua'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if shell_files: test_file = join(tmp_dir, 'run_test.' + suffix) if on_win: tf.write("call {test_file}\n".format(test_file=test_file)) if on_win: tf.write("if errorlevel 1 exit 1\n") else: # TODO: Run the test/commands here instead of in run_test.py tf.write("{shell_path} -x -e {test_file}\n".format( shell_path=shell_path, test_file=test_file)) if on_win: cmd = ['cmd.exe', "/d", "/c", test_script] else: cmd = [shell_path, '-x', '-e', test_script] try: subprocess.check_call(cmd, env=env, cwd=tmp_dir) except subprocess.CalledProcessError: tests_failed(m, move_broken=move_broken, broken_dir=config.broken_dir, config=config) print("TEST END:", m.dist())
def run_test( recipedir_or_package_or_metadata, config, stats, move_broken=True, provision_only=False, solver=None, ): """ Execute any test scripts for the given package. :param m: Package's metadata. :type m: Metadata """ # we want to know if we're dealing with package input. If so, we can move the input on success. hash_input = {} # store this name to keep it consistent. By changing files, we change the hash later. # It matches the build hash now, so let's keep it around. test_package_name = ( recipedir_or_package_or_metadata.dist() if hasattr(recipedir_or_package_or_metadata, "dist") else recipedir_or_package_or_metadata ) if not provision_only: print("TEST START:", test_package_name) if hasattr(recipedir_or_package_or_metadata, "config"): metadata = recipedir_or_package_or_metadata utils.rm_rf(metadata.config.test_dir) else: metadata, hash_input = construct_metadata_for_test( recipedir_or_package_or_metadata, config ) trace = "-x " if metadata.config.debug else "" # Must download *after* computing build id, or else computing build id will change # folder destination _extract_test_files_from_package(metadata) # When testing a .tar.bz2 in the pkgs dir, clean_pkg_cache() will remove it. # Prevent this. When https://github.com/conda/conda/issues/5708 gets fixed # I think we can remove this call to clean_pkg_cache(). in_pkg_cache = ( not hasattr(recipedir_or_package_or_metadata, "config") and os.path.isfile(recipedir_or_package_or_metadata) and recipedir_or_package_or_metadata.endswith(CONDA_PACKAGE_EXTENSIONS) and os.path.dirname(recipedir_or_package_or_metadata) in pkgs_dirs[0] ) if not in_pkg_cache: environ.clean_pkg_cache(metadata.dist(), metadata.config) copy_test_source_files(metadata, metadata.config.test_dir) # this is also copying tests/source_files from work_dir to testing workdir _, pl_files, py_files, r_files, lua_files, shell_files = create_all_test_files( metadata ) if ( not any([py_files, shell_files, pl_files, lua_files, r_files]) and not metadata.config.test_run_post ): print("Nothing to test for:", test_package_name) return True if metadata.config.remove_work_dir: for name, prefix in ( ("host", metadata.config.host_prefix), ("build", metadata.config.build_prefix), ): if os.path.isdir(prefix): # move host folder to force hardcoded paths to host env to break during tests # (so that they can be properly addressed by recipe author) dest = os.path.join( os.path.dirname(prefix), "_".join( ( "%s_prefix_moved" % name, metadata.dist(), getattr(metadata.config, "%s_subdir" % name), ) ), ) # Needs to come after create_files in case there's test/source_files shutil_move_more_retrying(prefix, dest, "{} prefix".format(prefix)) # nested if so that there's no warning when we just leave the empty workdir in place if metadata.source_provided: dest = os.path.join( os.path.dirname(metadata.config.work_dir), "_".join(("work_moved", metadata.dist(), metadata.config.host_subdir)), ) # Needs to come after create_files in case there's test/source_files shutil_move_more_retrying(config.work_dir, dest, "work") else: log.warn( "Not moving work directory after build. Your package may depend on files " "in the work directory that are not included with your package" ) # looks like a dead function to me # get_build_metadata(metadata) specs = metadata.get_test_deps(py_files, pl_files, lua_files, r_files) with utils.path_prepended(metadata.config.test_prefix): env = dict(os.environ.copy()) env.update(environ.get_dict(m=metadata, prefix=config.test_prefix)) env["CONDA_BUILD_STATE"] = "TEST" env["CONDA_BUILD"] = "1" if env_path_backup_var_exists: env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] if not metadata.config.activate or metadata.name() == "conda": # prepend bin (or Scripts) directory env = utils.prepend_bin_path( env, metadata.config.test_prefix, prepend_prefix=True ) if utils.on_win: env["PATH"] = metadata.config.test_prefix + os.pathsep + env["PATH"] env["PREFIX"] = metadata.config.test_prefix if "BUILD_PREFIX" in env: del env["BUILD_PREFIX"] # In the future, we will need to support testing cross compiled # packages on physical hardware. until then it is expected that # something like QEMU or Wine will be used on the build machine, # therefore, for now, we use host_subdir. # ensure that the test prefix isn't kept between variants utils.rm_rf(metadata.config.test_prefix) if solver is None: solver, pkg_cache_path = get_solver(metadata.config.host_subdir) else: pkg_cache_path = PackageCacheData.first_writable().pkgs_dir solver.replace_channels() transaction = solver.solve(specs, [pkg_cache_path]) downloaded = transaction.fetch_extract_packages( pkg_cache_path, solver.repos + list(solver.local_repos.values()), ) if not downloaded: raise RuntimeError("Did not succeed in downloading packages.") mkdir_p(metadata.config.test_prefix) transaction.execute( PrefixData(metadata.config.test_prefix), pkg_cache_path, ) with utils.path_prepended(metadata.config.test_prefix): env = dict(os.environ.copy()) env.update(environ.get_dict(m=metadata, prefix=metadata.config.test_prefix)) env["CONDA_BUILD_STATE"] = "TEST" if env_path_backup_var_exists: env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"] if config.test_run_post: from conda_build.utils import get_installed_packages installed = get_installed_packages(metadata.config.test_prefix) files = installed[metadata.meta["package"]["name"]]["files"] replacements = get_all_replacements(metadata.config) try_download(metadata, False, True) create_info_files(metadata, replacements, files, metadata.config.test_prefix) post_build(metadata, files, None, metadata.config.test_prefix, True) # when workdir is removed, the source files are unavailable. There's the test/source_files # entry that lets people keep these files around. The files are copied into test_dir for # intuitive relative path behavior, though, not work_dir, so we need to adjust where # SRC_DIR points. The initial CWD during tests is test_dir. if metadata.config.remove_work_dir: env["SRC_DIR"] = metadata.config.test_dir test_script, _ = write_test_scripts( metadata, env, py_files, pl_files, lua_files, r_files, shell_files, trace ) if utils.on_win: cmd = [os.environ.get("COMSPEC", "cmd.exe"), "/d", "/c", test_script] else: cmd = ( [shell_path] + (["-x"] if metadata.config.debug else []) + ["-o", "errexit", test_script] ) try: test_stats = {} if not provision_only: # rewrite long paths in stdout back to their env variables if metadata.config.debug or metadata.config.no_rewrite_stdout_env: rewrite_env = None else: rewrite_env = {k: env[k] for k in ["PREFIX", "SRC_DIR"] if k in env} if metadata.config.verbose: for k, v in rewrite_env.items(): print( "{0} {1}={2}".format( "set" if test_script.endswith(".bat") else "export", k, v, ) ) utils.check_call_env( cmd, env=env, cwd=metadata.config.test_dir, stats=test_stats, rewrite_stdout_env=rewrite_env, ) log_stats(test_stats, "testing {}".format(metadata.name())) # TODO need to implement metadata.get_used_loop_vars # if stats is not None and metadata.config.variants: # stats[ # stats_key(metadata, "test_{}".format(metadata.name())) # ] = test_stats if os.path.exists(join(metadata.config.test_dir, "TEST_FAILED")): raise subprocess.CalledProcessError(-1, "") print("TEST END:", test_package_name) except subprocess.CalledProcessError as _: # noqa tests_failed( metadata, move_broken=move_broken, broken_dir=metadata.config.broken_dir, config=metadata.config, ) raise if config.need_cleanup and config.recipe_dir is not None and not provision_only: utils.rm_rf(config.recipe_dir) return True
def test(m, config, move_broken=True): ''' Execute any test scripts for the given package. :param m: Package's metadata. :type m: Metadata ''' if not os.path.isdir(config.build_folder): os.makedirs(config.build_folder) clean_pkg_cache(m.dist(), config.timeout) with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"), timeout=config.timeout): tmp_dir = config.test_dir if not isdir(tmp_dir): os.makedirs(tmp_dir) create_files(tmp_dir, m, config) # Make Perl or Python-specific test files if m.name().startswith('perl-'): pl_files = create_pl_files(tmp_dir, m) py_files = False lua_files = False else: py_files = create_py_files(tmp_dir, m) pl_files = False lua_files = False shell_files = create_shell_files(tmp_dir, m, config) if not (py_files or shell_files or pl_files or lua_files): print("Nothing to test for:", m.dist()) return print("TEST START:", m.dist()) get_build_metadata(m, config=config) specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())] # add packages listed in the run environment and test/requires specs.extend(ms.spec for ms in m.ms_depends('run')) specs += m.get_value('test/requires', []) if py_files: # as the tests are run by python, ensure that python is installed. # (If they already provided python as a run or test requirement, # this won't hurt anything.) specs += ['python %s*' % environ.get_py_ver(config)] if pl_files: # as the tests are run by perl, we need to specify it specs += ['perl %s*' % environ.get_perl_ver(config)] if lua_files: # not sure how this shakes out specs += ['lua %s*' % environ.get_lua_ver(config)] create_env(config.test_prefix, specs, config=config) with path_prepended(config.test_prefix): env = dict(os.environ.copy()) env.update(environ.get_dict(config=config, m=m, prefix=config.test_prefix)) env["CONDA_BUILD_STATE"] = "TEST" if not config.activate: # prepend bin (or Scripts) directory env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True) if on_win: env['PATH'] = config.test_prefix + os.pathsep + env['PATH'] for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA': env[varname] = str(getattr(config, varname) or '') # Python 2 Windows requires that envs variables be string, not unicode env = {str(key): str(value) for key, value in env.items()} suffix = "bat" if on_win else "sh" test_script = join(tmp_dir, "conda_test_runner.{suffix}".format(suffix=suffix)) with open(test_script, 'w') as tf: if config.activate: ext = ".bat" if on_win else "" tf.write("{source} {conda_root}activate{ext} {test_env} {squelch}\n".format( conda_root=root_script_dir + os.path.sep, source="call" if on_win else "source", ext=ext, test_env=config.test_prefix, squelch=">nul 2>&1" if on_win else "&> /dev/null")) if on_win: tf.write("if errorlevel 1 exit 1\n") if py_files: tf.write("{python} -s {test_file}\n".format( python=config.test_python, test_file=join(tmp_dir, 'run_test.py'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if pl_files: tf.write("{perl} {test_file}\n".format( perl=config.test_perl, test_file=join(tmp_dir, 'run_test.pl'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if lua_files: tf.write("{lua} {test_file}\n".format( lua=config.test_lua, test_file=join(tmp_dir, 'run_test.lua'))) if on_win: tf.write("if errorlevel 1 exit 1\n") if shell_files: test_file = join(tmp_dir, 'run_test.' + suffix) if on_win: tf.write("call {test_file}\n".format(test_file=test_file)) if on_win: tf.write("if errorlevel 1 exit 1\n") else: # TODO: Run the test/commands here instead of in run_test.py tf.write("{shell_path} -x -e {test_file}\n".format(shell_path=shell_path, test_file=test_file)) if on_win: cmd = ['cmd.exe', "/d", "/c", test_script] else: cmd = [shell_path, '-x', '-e', test_script] try: subprocess.check_call(cmd, env=env, cwd=tmp_dir) except subprocess.CalledProcessError: tests_failed(m, move_broken=move_broken, broken_dir=config.broken_dir, config=config) print("TEST END:", m.dist())