def setup_package(pkg, dirty): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, spack_env) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False, deptype=('build', 'test')): spkg = dspec.package set_module_variables_for_package(spkg) # Allow dependencies to modify the module dpkg = dspec.package dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) if (not dirty) and (not spack_env.is_unset('CPATH')): tty.debug("A dependency has updated CPATH, this may lead pkg-config" " to assume that the package is part of the system" " includes and omit it when invoked with '--cflags'.") set_module_variables_for_package(pkg) pkg.setup_environment(spack_env, run_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the spack_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) # Make sure nothing's strange about the Spack environment. validate(spack_env, tty.warn) spack_env.apply_modifications()
def deactivate(): """ Deactivate an environment and collect corresponding environment modifications. Note: unloads the environment in its current state, not in the state it was loaded in, meaning that specs that were removed from the spack environment after activation are not unloaded. Returns: spack.util.environment.EnvironmentModifications: Environment variables modifications to activate environment. """ env_mods = EnvironmentModifications() active = ev.active_environment() if active is None: return env_mods if ev.default_view_name in active.views: try: with spack.store.db.read_transaction(): active.rm_default_view_from_env(env_mods) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.warn(e) tty.warn('Could not fully deactivate view due to missing package ' 'or repo, shell environment may be corrupt.') ev.deactivate() return env_mods
def test_external_prefixes_last(mutable_config, mock_packages, working_env, monkeypatch): # Sanity check: under normal circumstances paths associated with # dt-diamond-left would appear first. We'll mark it as external in # the test to check if the associated paths are placed last. assert 'dt-diamond-left' < 'dt-diamond-right' cfg_data = syaml.load_config("""\ dt-diamond-left: externals: - spec: [email protected] prefix: /fake/path1 buildable: false """) spack.config.set("packages", cfg_data) top = spack.spec.Spec('dt-diamond').concretized() def _trust_me_its_a_dir(path): return True monkeypatch.setattr(os.path, 'isdir', _trust_me_its_a_dir) env_mods = EnvironmentModifications() spack.build_environment.set_wrapper_variables(top.package, env_mods) env_mods.apply_modifications() link_dir_var = os.environ['SPACK_LINK_DIRS'] link_dirs = link_dir_var.split(':') external_lib_paths = set(['/fake/path1/lib', '/fake/path1/lib64']) # The external lib paths should be the last two entries of the list and # should not appear anywhere before the last two entries assert (set(os.path.normpath(x) for x in link_dirs[-2:]) == external_lib_paths) assert not (set(os.path.normpath(x) for x in link_dirs[:-2]) & external_lib_paths)
def modifications_from_dependencies(spec, context): """Returns the environment modifications that are required by the dependencies of a spec and also applies modifications to this spec's package at module scope, if need be. Args: spec (Spec): spec for which we want the modifications context (str): either 'build' for build-time modifications or 'run' for run-time modifications """ env = EnvironmentModifications() pkg = spec.package # Maps the context to deptype and method to be called deptype_and_method = { 'build': (('build', 'link', 'test'), 'setup_dependent_build_environment'), 'run': (('link', 'run'), 'setup_dependent_run_environment'), 'test': (('link', 'run', 'test'), 'setup_dependent_run_environment') } deptype, method = deptype_and_method[context] root = context == 'test' for dspec in spec.traverse(order='post', root=root, deptype=deptype): dpkg = dspec.package set_module_variables_for_package(dpkg) # Allow dependencies to modify the module dpkg.setup_dependent_package(pkg.module, spec) getattr(dpkg, method)(env, spec) return env
def clean_environment(): # Stuff in here sanitizes the build environment to eliminate # anything the user has set that may interfere. We apply it immediately # unlike the other functions so it doesn't overwrite what the modules load. env = EnvironmentModifications() # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') env.unset('LIBRARY_PATH') env.unset('CPATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') build_lang = spack.config.get('config:build_language') if build_lang: # Override language-related variables. This can be used to force # English compiler messages etc., which allows parse_log_events to # show useful matches. env.set('LC_ALL', build_lang) # Remove any macports installs from the PATH. The macports ld can # cause conflicts with the built-in linker on el capitan. Solves # assembler issues, e.g.: # suffix or operands invalid for `movq'" path = get_path('PATH') for p in path: if '/macports/' in p: env.remove_path('PATH', p) env.apply_modifications()
def deactivate(): """ Deactivate an environment and collect corresponding environment modifications Returns: spack.util.environment.EnvironmentModifications: Environment variables modifications to activate environment. """ env_mods = EnvironmentModifications() active = ev.active_environment() if active is None: return env_mods if ev.default_view_name in active.views: try: with spack.store.db.read_transaction(): active.rm_default_view_from_env(env_mods) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.warn(e) tty.warn('Could not fully deactivate view due to missing package ' 'or repo, shell environment may be corrupt.') ev.deactivate() return env_mods
def test_set_build_environment_variables(config, mock_packages, working_env, monkeypatch, installation_dir_with_headers): """Check that build_environment supplies the needed library/include directories via the SPACK_LINK_DIRS and SPACK_INCLUDE_DIRS environment variables. """ root = spack.spec.Spec('dt-diamond') root.concretize() for s in root.traverse(): s.prefix = '/{0}-prefix/'.format(s.name) dep_pkg = root['dt-diamond-left'].package dep_lib_paths = ['/test/path/to/ex1.so', '/test/path/to/subdir/ex2.so'] dep_lib_dirs = ['/test/path/to', '/test/path/to/subdir'] dep_libs = LibraryList(dep_lib_paths) dep2_pkg = root['dt-diamond-right'].package dep2_pkg.spec.prefix = str(installation_dir_with_headers) setattr(dep_pkg, 'libs', dep_libs) try: pkg = root.package env_mods = EnvironmentModifications() spack.build_environment.set_build_environment_variables(pkg, env_mods, dirty=False) env_mods.apply_modifications() def normpaths(paths): return list(os.path.normpath(p) for p in paths) link_dir_var = os.environ['SPACK_LINK_DIRS'] assert (normpaths(link_dir_var.split(':')) == normpaths(dep_lib_dirs)) root_libdirs = ['/dt-diamond-prefix/lib', '/dt-diamond-prefix/lib64'] rpath_dir_var = os.environ['SPACK_RPATH_DIRS'] # The 'lib' and 'lib64' subdirectories of the root package prefix # should always be rpathed and should be the first rpaths assert (normpaths(rpath_dir_var.split(':')) == normpaths(root_libdirs + dep_lib_dirs)) header_dir_var = os.environ['SPACK_INCLUDE_DIRS'] # The default implementation looks for header files only # in <prefix>/include and subdirectories prefix = str(installation_dir_with_headers) include_dirs = normpaths(header_dir_var.split(':')) assert os.path.join(prefix, 'include') in include_dirs assert os.path.join(prefix, 'include', 'boost') not in include_dirs assert os.path.join(prefix, 'path', 'to') not in include_dirs assert os.path.join(prefix, 'path', 'to', 'subdir') not in include_dirs finally: delattr(dep_pkg, 'libs')
def setup_package(pkg, dirty): """Execute all environment setup routines.""" build_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, build_env) set_build_environment_variables(pkg, build_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, build_env) build_env.extend( modifications_from_dependencies(pkg.spec, context='build') ) if (not dirty) and (not build_env.is_unset('CPATH')): tty.debug("A dependency has updated CPATH, this may lead pkg-config" " to assume that the package is part of the system" " includes and omit it when invoked with '--cflags'.") set_module_variables_for_package(pkg) pkg.setup_build_environment(build_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the build_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) # kludge to handle cray libsci being automatically loaded by PrgEnv # modules on cray platform. Module unload does no damage when # unnecessary module('unload', 'cray-libsci') if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) implicit_rpaths = pkg.compiler.implicit_rpaths() if implicit_rpaths: build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS', ':'.join(implicit_rpaths)) # Make sure nothing's strange about the Spack environment. validate(build_env, tty.warn) build_env.apply_modifications()
def __init__(self, name): self.exe = shlex.split(str(name)) self.default_env = {} from spack.util.environment import EnvironmentModifications # no cycle self.default_envmod = EnvironmentModifications() self.returncode = None if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name)
def test_extend(env): """Tests that we can construct a list of environment modifications starting from another list. """ env.set('A', 'dummy value') env.set('B', 3) copy_construct = EnvironmentModifications(env) assert len(copy_construct) == 2 for x, y in zip(env, copy_construct): assert x is y
def __init__(self, name): # necesary here for the shlex call to succeed name = format_os_path(name, mode=Path.unix) self.exe = shlex.split(str(name)) # filter back to platform dependent path self.exe = path_to_os_path(*self.exe) self.default_env = {} from spack.util.environment import EnvironmentModifications # no cycle self.default_envmod = EnvironmentModifications() self.returncode = None if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name)
def test_source_files(files_to_be_sourced): """Tests the construction of a list of environment modifications that are the result of sourcing a file. """ env = EnvironmentModifications() for filename in files_to_be_sourced: if filename.endswith('sourceme_parameters.sh'): env.extend( EnvironmentModifications.from_sourcing_file( filename, 'intel64')) else: env.extend(EnvironmentModifications.from_sourcing_file(filename)) modifications = env.group_by_name() # This is sensitive to the user's environment; can include # spurious entries for things like PS1 # # TODO: figure out how to make a bit more robust. assert len(modifications) >= 5 # Set new variables assert len(modifications['NEW_VAR']) == 1 assert isinstance(modifications['NEW_VAR'][0], SetEnv) assert modifications['NEW_VAR'][0].value == 'new' assert len(modifications['FOO']) == 1 assert isinstance(modifications['FOO'][0], SetEnv) assert modifications['FOO'][0].value == 'intel64' # Unset variables assert len(modifications['EMPTY_PATH_LIST']) == 1 assert isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv) # Modified variables assert len(modifications['UNSET_ME']) == 1 assert isinstance(modifications['UNSET_ME'][0], SetEnv) assert modifications['UNSET_ME'][0].value == 'overridden' assert len(modifications['PATH_LIST']) == 3 assert isinstance(modifications['PATH_LIST'][0], RemovePath) assert modifications['PATH_LIST'][0].value == '/path/third' assert isinstance(modifications['PATH_LIST'][1], AppendPath) assert modifications['PATH_LIST'][1].value == '/path/fourth' assert isinstance(modifications['PATH_LIST'][2], PrependPath) assert modifications['PATH_LIST'][2].value == '/path/first'
def test_setting_dtags_based_on_config(config_setting, expected_flag, config, mock_packages): # Pick a random package to be able to set compiler's variables s = spack.spec.Spec('cmake') s.concretize() pkg = s.package env = EnvironmentModifications() with spack.config.override('config:shared_linking', config_setting): spack.build_environment.set_compiler_environment_variables(pkg, env) modifications = env.group_by_name() assert 'SPACK_DTAGS_TO_STRIP' in modifications assert 'SPACK_DTAGS_TO_ADD' in modifications assert len(modifications['SPACK_DTAGS_TO_ADD']) == 1 assert len(modifications['SPACK_DTAGS_TO_STRIP']) == 1 dtags_to_add = modifications['SPACK_DTAGS_TO_ADD'][0] assert dtags_to_add.value == expected_flag
def activate(env, use_env_repo=False, add_view=True): """ Activate an environment and append environment modifications To activate an environment, we add its configuration scope to the existing Spack configuration, and we set active to the current environment. Arguments: env (spack.environment.Environment): the environment to activate use_env_repo (bool): use the packages exactly as they appear in the environment's repository add_view (bool): generate commands to add view to path variables Returns: spack.util.environment.EnvironmentModifications: Environment variables modifications to activate environment. """ ev.activate(env, use_env_repo=use_env_repo) env_mods = EnvironmentModifications() # # NOTE in the fish-shell: Path variables are a special kind of variable # used to support colon-delimited path lists including PATH, CDPATH, # MANPATH, PYTHONPATH, etc. All variables that end in PATH (case-sensitive) # become PATH variables. # try: if add_view and ev.default_view_name in env.views: with spack.store.db.read_transaction(): env.add_default_view_to_env(env_mods) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.error(e) tty.die( 'Environment view is broken due to a missing package or repo.\n', ' To activate without views enabled, activate with:\n', ' spack env activate -V {0}\n'.format(env.name), ' To remove it and resolve the issue, ' 'force concretize with the command:\n', ' spack -e {0} concretize --force'.format(env.name)) return env_mods
def env_activate(args): if not args.activate_env and not args.dir and not args.temp: tty.die('spack env activate requires an environment name, directory, or --temp') if not args.shell: spack.cmd.common.shell_init_instructions( "spack env activate", " eval `spack env activate {sh_arg} [...]`", ) return 1 # Error out when -e, -E, -D flags are given, cause they are ambiguous. if args.env or args.no_env or args.env_dir: tty.die('Calling spack env activate with --env, --env-dir and --no-env ' 'is ambiguous') env_name_or_dir = args.activate_env or args.dir # Temporary environment if args.temp: env = create_temp_env_directory() env_path = os.path.abspath(env) short_name = os.path.basename(env_path) ev.Environment(env).write(regenerate=False) # Named environment elif ev.exists(env_name_or_dir) and not args.dir: env_path = ev.root(env_name_or_dir) short_name = env_name_or_dir # Environment directory elif ev.is_env_dir(env_name_or_dir): env_path = os.path.abspath(env_name_or_dir) short_name = os.path.basename(env_path) else: tty.die("No such environment: '%s'" % env_name_or_dir) env_prompt = '[%s]' % short_name # We only support one active environment at a time, so deactivate the current one. if ev.active_environment() is None: cmds = '' env_mods = EnvironmentModifications() else: cmds = spack.environment.shell.deactivate_header(shell=args.shell) env_mods = spack.environment.shell.deactivate() # Activate new environment active_env = ev.Environment(env_path) cmds += spack.environment.shell.activate_header( env=active_env, shell=args.shell, prompt=env_prompt if args.prompt else None ) env_mods.extend(spack.environment.shell.activate( env=active_env, add_view=args.with_view )) cmds += env_mods.shell_modifications(args.shell) sys.stdout.write(cmds)
def get_executable(exe, spec=None, install=False): """Find an executable named exe, either in PATH or in Spack Args: exe (str): needed executable name spec (Spec or str): spec to search for exe in (default exe) install (bool): install spec if not available When ``install`` is True, Spack will use the python used to run Spack as an external. The ``install`` option should only be used with packages that install quickly (when using external python) or are guaranteed by Spack organization to be in a binary mirror (clingo).""" # Search the system first runner = spack.util.executable.which(exe) if runner: return runner # Check whether it's already installed spec = spack.spec.Spec(spec or exe) installed_specs = spack.store.db.query(spec, installed=True) for ispec in installed_specs: # filter out directories of the same name as the executable exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe) if fs.is_exe(exe_p)] if exe_path: ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in ispec.traverse(root=True, order='post'): envmod.extend(uenv.environment_modifications_for_spec(dep)) ret.add_default_envmod(envmod) return ret else: tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix)) def _raise_error(executable, exe_spec): error_msg = 'cannot find the executable "{0}"'.format(executable) if exe_spec: error_msg += ' from spec "{0}'.format(exe_spec) raise RuntimeError(error_msg) # If we're not allowed to install this for ourselves, we can't find it if not install: _raise_error(exe, spec) with spack_python_interpreter(): # We will install for ourselves, using this python if needed # Concretize the spec spec.concretize() spec.package.do_install() # filter out directories of the same name as the executable exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe) if fs.is_exe(exe_p)] if exe_path: ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in spec.traverse(root=True, order='post'): envmod.extend(uenv.environment_modifications_for_spec(dep)) ret.add_default_envmod(envmod) return ret _raise_error(exe, spec)
def setup_package(pkg, dirty): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, spack_env) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False, deptype=('build', 'test')): # If a user makes their own package repo, e.g. # spack.pkg.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.pkg.original.libelf.Libelf, # then set the module variables for both classes so the # parent class can still use them if it gets called. spkg = dspec.package modules = parent_class_modules(spkg.__class__) for mod in modules: set_module_variables_for_package(spkg, mod) set_module_variables_for_package(spkg, spkg.module) # Allow dependencies to modify the module dpkg = dspec.package dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) set_module_variables_for_package(pkg, pkg.module) pkg.setup_environment(spack_env, run_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the spack_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) # Make sure nothing's strange about the Spack environment. validate(spack_env, tty.warn) spack_env.apply_modifications()
def clean_environment(): # Stuff in here sanitizes the build environment to eliminate # anything the user has set that may interfere. We apply it immediately # unlike the other functions so it doesn't overwrite what the modules load. env = EnvironmentModifications() # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') env.unset('DYLD_FALLBACK_LIBRARY_PATH') # These vars affect how the compiler finds libraries and include dirs. env.unset('LIBRARY_PATH') env.unset('CPATH') env.unset('C_INCLUDE_PATH') env.unset('CPLUS_INCLUDE_PATH') env.unset('OBJC_INCLUDE_PATH') # On Cray "cluster" systems, unset CRAY_LD_LIBRARY_PATH to avoid # interference with Spack dependencies. # CNL requires these variables to be set (or at least some of them, # depending on the CNL version). hostarch = arch.Arch(arch.platform(), 'default_os', 'default_target') on_cray = str(hostarch.platform) == 'cray' using_cnl = re.match(r'cnl\d+', str(hostarch.os)) if on_cray and not using_cnl: env.unset('CRAY_LD_LIBRARY_PATH') for varname in os.environ.keys(): if 'PKGCONF' in varname: env.unset(varname) # Unset the following variables because they can affect installation of # Autotools and CMake packages. build_system_vars = [ 'CC', 'CFLAGS', 'CPP', 'CPPFLAGS', # C variables 'CXX', 'CCC', 'CXXFLAGS', 'CXXCPP', # C++ variables 'F77', 'FFLAGS', 'FLIBS', # Fortran77 variables 'FC', 'FCFLAGS', 'FCLIBS', # Fortran variables 'LDFLAGS', 'LIBS' # linker variables ] for v in build_system_vars: env.unset(v) # Unset mpi environment vars. These flags should only be set by # mpi providers for packages with mpi dependencies mpi_vars = ['MPICC', 'MPICXX', 'MPIFC', 'MPIF77', 'MPIF90'] for v in mpi_vars: env.unset(v) build_lang = spack.config.get('config:build_language') if build_lang: # Override language-related variables. This can be used to force # English compiler messages etc., which allows parse_log_events to # show useful matches. env.set('LC_ALL', build_lang) # Remove any macports installs from the PATH. The macports ld can # cause conflicts with the built-in linker on el capitan. Solves # assembler issues, e.g.: # suffix or operands invalid for `movq'" path = get_path('PATH') for p in path: if '/macports/' in p: env.remove_path('PATH', p) env.apply_modifications()
def env(prepare_environment_for_tests): """Returns an empty EnvironmentModifications object.""" return EnvironmentModifications()
def add_default_view_to_shell(self, shell): env_mod = EnvironmentModifications() for var, paths in self._shell_vars(): for path in paths: env_mod.prepend_path(var, path) return env_mod.shell_modifications(shell)
def rm_default_view_from_shell(self, shell): env_mod = EnvironmentModifications() for var, paths in self._shell_vars(): for path in paths: env_mod.remove_path(var, path) return env_mod.shell_modifications(shell)