def test_external_prefixes_last(mutable_config, mock_packages, working_env, monkeypatch): # Sanity check: under normal circumstances paths associated with # dt-diamond-left would appear first. We'll mark it as external in # the test to check if the associated paths are placed last. assert 'dt-diamond-left' < 'dt-diamond-right' cfg_data = syaml.load_config("""\ dt-diamond-left: externals: - spec: [email protected] prefix: /fake/path1 buildable: false """) spack.config.set("packages", cfg_data) top = spack.spec.Spec('dt-diamond').concretized() def _trust_me_its_a_dir(path): return True monkeypatch.setattr(os.path, 'isdir', _trust_me_its_a_dir) env_mods = EnvironmentModifications() spack.build_environment.set_wrapper_variables(top.package, env_mods) env_mods.apply_modifications() link_dir_var = os.environ['SPACK_LINK_DIRS'] link_dirs = link_dir_var.split(':') external_lib_paths = set(['/fake/path1/lib', '/fake/path1/lib64']) # The external lib paths should be the last two entries of the list and # should not appear anywhere before the last two entries assert (set(os.path.normpath(x) for x in link_dirs[-2:]) == external_lib_paths) assert not (set(os.path.normpath(x) for x in link_dirs[:-2]) & external_lib_paths)
def test_set_build_environment_variables(config, mock_packages, working_env, monkeypatch, installation_dir_with_headers): """Check that build_environment supplies the needed library/include directories via the SPACK_LINK_DIRS and SPACK_INCLUDE_DIRS environment variables. """ root = spack.spec.Spec('dt-diamond') root.concretize() for s in root.traverse(): s.prefix = '/{0}-prefix/'.format(s.name) dep_pkg = root['dt-diamond-left'].package dep_lib_paths = ['/test/path/to/ex1.so', '/test/path/to/subdir/ex2.so'] dep_lib_dirs = ['/test/path/to', '/test/path/to/subdir'] dep_libs = LibraryList(dep_lib_paths) dep2_pkg = root['dt-diamond-right'].package dep2_pkg.spec.prefix = str(installation_dir_with_headers) setattr(dep_pkg, 'libs', dep_libs) try: pkg = root.package env_mods = EnvironmentModifications() spack.build_environment.set_build_environment_variables(pkg, env_mods, dirty=False) env_mods.apply_modifications() def normpaths(paths): return list(os.path.normpath(p) for p in paths) link_dir_var = os.environ['SPACK_LINK_DIRS'] assert (normpaths(link_dir_var.split(':')) == normpaths(dep_lib_dirs)) root_libdirs = ['/dt-diamond-prefix/lib', '/dt-diamond-prefix/lib64'] rpath_dir_var = os.environ['SPACK_RPATH_DIRS'] # The 'lib' and 'lib64' subdirectories of the root package prefix # should always be rpathed and should be the first rpaths assert (normpaths(rpath_dir_var.split(':')) == normpaths(root_libdirs + dep_lib_dirs)) header_dir_var = os.environ['SPACK_INCLUDE_DIRS'] # The default implementation looks for header files only # in <prefix>/include and subdirectories prefix = str(installation_dir_with_headers) include_dirs = normpaths(header_dir_var.split(':')) assert os.path.join(prefix, 'include') in include_dirs assert os.path.join(prefix, 'include', 'boost') not in include_dirs assert os.path.join(prefix, 'path', 'to') not in include_dirs assert os.path.join(prefix, 'path', 'to', 'subdir') not in include_dirs finally: delattr(dep_pkg, 'libs')
def __init__(self, name): self.exe = shlex.split(str(name)) self.default_env = {} from spack.util.environment import EnvironmentModifications # no cycle self.default_envmod = EnvironmentModifications() self.returncode = None if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name)
def setup_package(pkg, dirty): """Execute all environment setup routines.""" build_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, build_env) set_build_environment_variables(pkg, build_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, build_env) build_env.extend( modifications_from_dependencies(pkg.spec, context='build') ) if (not dirty) and (not build_env.is_unset('CPATH')): tty.debug("A dependency has updated CPATH, this may lead pkg-config" " to assume that the package is part of the system" " includes and omit it when invoked with '--cflags'.") set_module_variables_for_package(pkg) pkg.setup_build_environment(build_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the build_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) # kludge to handle cray libsci being automatically loaded by PrgEnv # modules on cray platform. Module unload does no damage when # unnecessary module('unload', 'cray-libsci') if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) implicit_rpaths = pkg.compiler.implicit_rpaths() if implicit_rpaths: build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS', ':'.join(implicit_rpaths)) # Make sure nothing's strange about the Spack environment. validate(build_env, tty.warn) build_env.apply_modifications()
def __init__(self, name): # necesary here for the shlex call to succeed name = format_os_path(name, mode=Path.unix) self.exe = shlex.split(str(name)) # filter back to platform dependent path self.exe = path_to_os_path(*self.exe) self.default_env = {} from spack.util.environment import EnvironmentModifications # no cycle self.default_envmod = EnvironmentModifications() self.returncode = None if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name)
def deactivate(): """ Deactivate an environment and collect corresponding environment modifications. Note: unloads the environment in its current state, not in the state it was loaded in, meaning that specs that were removed from the spack environment after activation are not unloaded. Returns: spack.util.environment.EnvironmentModifications: Environment variables modifications to activate environment. """ env_mods = EnvironmentModifications() active = ev.active_environment() if active is None: return env_mods if ev.default_view_name in active.views: try: with spack.store.db.read_transaction(): active.rm_default_view_from_env(env_mods) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.warn(e) tty.warn('Could not fully deactivate view due to missing package ' 'or repo, shell environment may be corrupt.') ev.deactivate() return env_mods
def setup_environment(self, spack_env, run_env): if not self.stage.source_path: self.stage.fetch() self.stage.expand_archive() spack_env.set('FSLDIR', self.stage.source_path) # Here, run-time environment variables are being set manually. # Normally these would be added to the modulefile at build-time # by sourcing fsl.sh, but incorrect paths were being set, pointing to # the staging directory rather than the install directory. run_env.set('FSLDIR', self.prefix) run_env.set('FSLOUTPUTTYPE', 'NIFTI_GZ') run_env.set('FSLMULTIFILEQUIT', 'TRUE') run_env.set('FSLTCLSH', self.prefix.bin.fsltclsh) run_env.set('FSLWISH', self.prefix.bin.fslwish) run_env.set('FSLLOCKDIR', '') run_env.set('FSLMACHINELIST', '') run_env.set('FSLREMOTECALL', '') run_env.set('FSLGECUDAQ', 'cuda.q') run_env.prepend_path('PATH', self.prefix) # Below is for sourcing purposes during building fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf', 'fsl.sh') if os.path.isfile(fslsetup): spack_env.extend(EnvironmentModifications.from_sourcing_file( fslsetup))
def setup_environment(self, spack_env, run_env): if not self.stage.source_path: self.stage.fetch() self.stage.expand_archive() spack_env.set('FSLDIR', self.stage.source_path) # Here, run-time environment variables are being set manually. # Normally these would be added to the modulefile at build-time # by sourcing fsl.sh, but incorrect paths were being set, pointing to # the staging directory rather than the install directory. run_env.set('FSLDIR', self.prefix) run_env.set('FSLOUTPUTTYPE', 'NIFTI_GZ') run_env.set('FSLMULTIFILEQUIT', 'TRUE') run_env.set('FSLTCLSH', self.prefix.bin.fsltclsh) run_env.set('FSLWISH', self.prefix.bin.fslwish) run_env.set('FSLLOCKDIR', '') run_env.set('FSLMACHINELIST', '') run_env.set('FSLREMOTECALL', '') run_env.set('FSLGECUDAQ', 'cuda.q') run_env.prepend_path('PATH', self.prefix) # Below is for sourcing purposes during building fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf', 'fsl.sh') if os.path.isfile(fslsetup): spack_env.extend(EnvironmentModifications.from_sourcing_file( fslsetup))
def modifications_from_dependencies(spec, context): """Returns the environment modifications that are required by the dependencies of a spec and also applies modifications to this spec's package at module scope, if need be. Args: spec (Spec): spec for which we want the modifications context (str): either 'build' for build-time modifications or 'run' for run-time modifications """ env = EnvironmentModifications() pkg = spec.package # Maps the context to deptype and method to be called deptype_and_method = { 'build': (('build', 'link', 'test'), 'setup_dependent_build_environment'), 'run': (('link', 'run'), 'setup_dependent_run_environment'), 'test': (('link', 'run', 'test'), 'setup_dependent_run_environment') } deptype, method = deptype_and_method[context] root = context == 'test' for dspec in spec.traverse(order='post', root=root, deptype=deptype): dpkg = dspec.package set_module_variables_for_package(dpkg) # Allow dependencies to modify the module dpkg.setup_dependent_package(pkg.module, spec) getattr(dpkg, method)(env, spec) return env
def setup_run_environment(self, env): # Set the environment variables after copying tree env.set('FSLDIR', self.prefix) fslsetup = join_path(self.prefix, 'etc', 'fslconf', 'fsl.sh') if os.path.isfile(fslsetup): env.extend(EnvironmentModifications.from_sourcing_file(fslsetup))
def deactivate(): """ Deactivate an environment and collect corresponding environment modifications Returns: spack.util.environment.EnvironmentModifications: Environment variables modifications to activate environment. """ env_mods = EnvironmentModifications() active = ev.active_environment() if active is None: return env_mods if ev.default_view_name in active.views: try: with spack.store.db.read_transaction(): active.rm_default_view_from_env(env_mods) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.warn(e) tty.warn('Could not fully deactivate view due to missing package ' 'or repo, shell environment may be corrupt.') ev.deactivate() return env_mods
def test_blacklist_lmod_variables(): # Construct the list of environment modifications file = os.path.join(datadir, 'sourceme_lmod.sh') env = EnvironmentModifications.from_sourcing_file(file) # Check that variables related to lmod are not in there modifications = env.group_by_name() assert not any(x.startswith('LMOD_') for x in modifications)
def setup_run_environment(self, env): try: env.extend(EnvironmentModifications.from_sourcing_file( join_path(self.spec.prefix, 'headas-config_spack.sh'), clean=True )) except Exception as e: msg = 'unexpected error when sourcing HEASOFT setup [{0}]' tty.warn(msg.format(str(e)))
def test_setting_dtags_based_on_config(config_setting, expected_flag, config, mock_packages): # Pick a random package to be able to set compiler's variables s = spack.spec.Spec('cmake') s.concretize() pkg = s.package env = EnvironmentModifications() with spack.config.override('config:shared_linking', config_setting): spack.build_environment.set_compiler_environment_variables(pkg, env) modifications = env.group_by_name() assert 'SPACK_DTAGS_TO_STRIP' in modifications assert 'SPACK_DTAGS_TO_ADD' in modifications assert len(modifications['SPACK_DTAGS_TO_ADD']) == 1 assert len(modifications['SPACK_DTAGS_TO_STRIP']) == 1 dtags_to_add = modifications['SPACK_DTAGS_TO_ADD'][0] assert dtags_to_add.value == expected_flag
def setup_run_environment(self, env): bashrc = self.prefix.etc.bashrc try: env.extend(EnvironmentModifications.from_sourcing_file( bashrc, clean=True )) except Exception as e: msg = 'unexpected error when sourcing OpenFOAM bashrc [{0}]' tty.warn(msg.format(str(e)))
def setup_run_environment(self, env): """Sets the run environment (post-installation). The environment comes from running: .. code-block:: console $ . $WM_PROJECT_DIR/etc/bashrc """ bashrc = join_path(self.projectdir, 'etc', 'bashrc') minimal = True if os.path.isfile(bashrc): # post-install: source the installed bashrc try: mods = EnvironmentModifications.from_sourcing_file( bashrc, clean=True, # Remove duplicate entries blacklist=[ # Blacklist these # Inadvertent changes # ------------------- 'PS1', # Leave untouched 'MANPATH', # Leave untouched # Unneeded bits # ------------- # 'FOAM_SETTINGS', # Do not use with modules # 'FOAM_INST_DIR', # Old # 'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)', # 'FOAM_TUTORIALS', # May be useful # 'WM_OSTYPE', # Purely optional value # Third-party cruft - only used for orig compilation # ----------------- '[A-Z].*_ARCH_PATH', # '(KAHIP|METIS|SCOTCH)_VERSION', # User-specific # ------------- 'FOAM_RUN', '(FOAM|WM)_.*USER_.*', ], whitelist=[ # Whitelist these 'MPI_ARCH_PATH', # Can be required for compilation ]) env.extend(mods) minimal = False tty.info('OpenFOAM bashrc env: {0}'.format(bashrc)) except Exception: minimal = True if minimal: # pre-build or minimal environment self.setup_minimal_environment(env)
def test_source_files(files_to_be_sourced): """Tests the construction of a list of environment modifications that are the result of sourcing a file. """ env = EnvironmentModifications() for filename in files_to_be_sourced: if filename.endswith('sourceme_parameters.sh'): env.extend( EnvironmentModifications.from_sourcing_file( filename, 'intel64')) else: env.extend(EnvironmentModifications.from_sourcing_file(filename)) modifications = env.group_by_name() # This is sensitive to the user's environment; can include # spurious entries for things like PS1 # # TODO: figure out how to make a bit more robust. assert len(modifications) >= 5 # Set new variables assert len(modifications['NEW_VAR']) == 1 assert isinstance(modifications['NEW_VAR'][0], SetEnv) assert modifications['NEW_VAR'][0].value == 'new' assert len(modifications['FOO']) == 1 assert isinstance(modifications['FOO'][0], SetEnv) assert modifications['FOO'][0].value == 'intel64' # Unset variables assert len(modifications['EMPTY_PATH_LIST']) == 1 assert isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv) # Modified variables assert len(modifications['UNSET_ME']) == 1 assert isinstance(modifications['UNSET_ME'][0], SetEnv) assert modifications['UNSET_ME'][0].value == 'overridden' assert len(modifications['PATH_LIST']) == 3 assert isinstance(modifications['PATH_LIST'][0], RemovePath) assert modifications['PATH_LIST'][0].value == '/path/third' assert isinstance(modifications['PATH_LIST'][1], AppendPath) assert modifications['PATH_LIST'][1].value == '/path/fourth' assert isinstance(modifications['PATH_LIST'][2], PrependPath) assert modifications['PATH_LIST'][2].value == '/path/first'
def setup_run_environment(self, env): """Adds environment variables to the generated module file. These environment variables come from running: .. code-block:: console $ source {prefix}/setvars.sh --force """ env.extend( EnvironmentModifications.from_sourcing_file( join(self.prefix, self.component_dir, 'latest/env/vars.sh')))
def test_extend(env): """Tests that we can construct a list of environment modifications starting from another list. """ env.set('A', 'dummy value') env.set('B', 3) copy_construct = EnvironmentModifications(env) assert len(copy_construct) == 2 for x, y in zip(env, copy_construct): assert x is y
def setup_run_environment(self, env): """Adds environment variables to the generated module file. These environment variables come from running: .. code-block:: console $ source {prefix}/{component}/{version}/env/vars.sh """ env.extend( EnvironmentModifications.from_sourcing_file( join_path(self.component_path, 'env', 'vars.sh')))
def setup_build_environment(self, env): if not self.stage.source_path: self.stage.fetch() self.stage.expand_archive() env.set('FSLDIR', self.stage.source_path) # Below is for sourcing purposes during building fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf', 'fsl.sh') if os.path.isfile(fslsetup): env.extend(EnvironmentModifications.from_sourcing_file(fslsetup))
def setup_package(pkg, dirty): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, spack_env) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False, deptype=('build', 'test')): spkg = dspec.package set_module_variables_for_package(spkg) # Allow dependencies to modify the module dpkg = dspec.package dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) if (not dirty) and (not spack_env.is_unset('CPATH')): tty.debug("A dependency has updated CPATH, this may lead pkg-config" " to assume that the package is part of the system" " includes and omit it when invoked with '--cflags'.") set_module_variables_for_package(pkg) pkg.setup_environment(spack_env, run_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the spack_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) # Make sure nothing's strange about the Spack environment. validate(spack_env, tty.warn) spack_env.apply_modifications()
def _read_environment_file(self, filename): """ Read and parse the environment file. Given an environment file, we want to read it, split by semicolons and new lines, and then parse down to the subset of SPACK_* variables. We assume that all spack prefix variables are not secrets, and unlike the install_manifest.json, we don't (at least to start) parse the values to remove path prefixes specific to user systems. """ if not os.path.exists(filename): return mods = EnvironmentModifications.from_sourcing_file(filename) env = {} mods.apply_modifications(env) return env
def activate(env, use_env_repo=False, add_view=True): """ Activate an environment and append environment modifications To activate an environment, we add its configuration scope to the existing Spack configuration, and we set active to the current environment. Arguments: env (spack.environment.Environment): the environment to activate use_env_repo (bool): use the packages exactly as they appear in the environment's repository add_view (bool): generate commands to add view to path variables Returns: spack.util.environment.EnvironmentModifications: Environment variables modifications to activate environment. """ ev.activate(env, use_env_repo=use_env_repo) env_mods = EnvironmentModifications() # # NOTE in the fish-shell: Path variables are a special kind of variable # used to support colon-delimited path lists including PATH, CDPATH, # MANPATH, PYTHONPATH, etc. All variables that end in PATH (case-sensitive) # become PATH variables. # try: if add_view and ev.default_view_name in env.views: with spack.store.db.read_transaction(): env.add_default_view_to_env(env_mods) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.error(e) tty.die( 'Environment view is broken due to a missing package or repo.\n', ' To activate without views enabled, activate with:\n', ' spack env activate -V {0}\n'.format(env.name), ' To remove it and resolve the issue, ' 'force concretize with the command:\n', ' spack -e {0} concretize --force'.format(env.name)) return env_mods
def setup_environment(self, spack_env, run_env): """Add environment variables to the generated module file. These environment variables come from running: .. code-block:: console $ . $WM_PROJECT_DIR/etc/bashrc """ # NOTE: Spack runs setup_environment twice. # 1) pre-build to set up the build environment # 2) post-install to determine runtime environment variables # The etc/bashrc is only available (with corrrect content) # post-installation. bashrc = join_path(self.projectdir, 'etc', 'bashrc') minimal = True if os.path.isfile(bashrc): # post-install: source the installed bashrc try: mods = EnvironmentModifications.from_sourcing_file( bashrc, clean=True, # Remove duplicate entries blacklist=[ # Blacklist these # Inadvertent changes # ------------------- 'PS1', # Leave unaffected 'MANPATH', # Leave unaffected # Unneeded bits # ------------- 'FOAM_INST_DIR', # Possibly incorrect 'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)', 'FOAM_TEST_.*_DIR', 'WM_NCOMPPROCS', # 'FOAM_TUTORIALS', # can be useful # Lots of third-party cruft # ------------------------- '[A-Z].*_(BIN|LIB|INCLUDE)_DIR', '[A-Z].*_SYSTEM', 'WM_THIRD_PARTY_.*', '(BISON|FLEX|CMAKE|ZLIB)_DIR', '(METIS|PARMETIS|PARMGRIDGEN|SCOTCH)_DIR', # User-specific # ------------- 'FOAM_RUN', '(FOAM|WM)_.*USER_.*', ], whitelist=[ # Whitelist these 'MPI_ARCH_PATH', # Can be needed for compilation 'PYTHON_BIN_DIR', ]) run_env.extend(mods) minimal = False tty.info('foam-extend env: {0}'.format(bashrc)) except Exception: minimal = True if minimal: # pre-build or minimal environment tty.info('foam-extend minimal env {0}'.format(self.prefix)) run_env.set('FOAM_INST_DIR', os.path.dirname(self.projectdir)), run_env.set('FOAM_PROJECT_DIR', self.projectdir) run_env.set('WM_PROJECT_DIR', self.projectdir) for d in ['wmake', self.archbin]: # bin added automatically run_env.prepend_path('PATH', join_path(self.projectdir, d))
def get_executable(exe, spec=None, install=False): """Find an executable named exe, either in PATH or in Spack Args: exe (str): needed executable name spec (Spec or str): spec to search for exe in (default exe) install (bool): install spec if not available When ``install`` is True, Spack will use the python used to run Spack as an external. The ``install`` option should only be used with packages that install quickly (when using external python) or are guaranteed by Spack organization to be in a binary mirror (clingo).""" # Search the system first runner = spack.util.executable.which(exe) if runner: return runner # Check whether it's already installed spec = spack.spec.Spec(spec or exe) installed_specs = spack.store.db.query(spec, installed=True) for ispec in installed_specs: # filter out directories of the same name as the executable exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe) if fs.is_exe(exe_p)] if exe_path: ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in ispec.traverse(root=True, order='post'): envmod.extend(uenv.environment_modifications_for_spec(dep)) ret.add_default_envmod(envmod) return ret else: tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix)) def _raise_error(executable, exe_spec): error_msg = 'cannot find the executable "{0}"'.format(executable) if exe_spec: error_msg += ' from spec "{0}'.format(exe_spec) raise RuntimeError(error_msg) # If we're not allowed to install this for ourselves, we can't find it if not install: _raise_error(exe, spec) with spack_python_interpreter(): # We will install for ourselves, using this python if needed # Concretize the spec spec.concretize() spec.package.do_install() # filter out directories of the same name as the executable exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe) if fs.is_exe(exe_p)] if exe_path: ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in spec.traverse(root=True, order='post'): envmod.extend(uenv.environment_modifications_for_spec(dep)) ret.add_default_envmod(envmod) return ret _raise_error(exe, spec)
def setup_environment(self, spack_env, run_env): """Add environment variables to the generated module file. These environment variables come from running: .. code-block:: console $ . $WM_PROJECT_DIR/etc/bashrc """ # NOTE: Spack runs setup_environment twice. # 1) pre-build to set up the build environment # 2) post-install to determine runtime environment variables # The etc/bashrc is only available (with corrrect content) # post-installation. bashrc = join_path(self.projectdir, 'etc', 'bashrc') minimal = True if os.path.isfile(bashrc): # post-install: source the installed bashrc try: mods = EnvironmentModifications.from_sourcing_file( bashrc, clean=True, # Remove duplicate entries blacklist=[ # Blacklist these # Inadvertent changes # ------------------- 'PS1', # Leave unaffected 'MANPATH', # Leave unaffected # Unneeded bits # ------------- 'FOAM_INST_DIR', # Possibly incorrect 'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)', 'FOAM_TEST_.*_DIR', 'WM_NCOMPPROCS', # 'FOAM_TUTORIALS', # can be useful # Lots of third-party cruft # ------------------------- '[A-Z].*_(BIN|LIB|INCLUDE)_DIR', '[A-Z].*_SYSTEM', 'WM_THIRD_PARTY_.*', '(BISON|FLEX|CMAKE|ZLIB)_DIR', '(METIS|PARMETIS|PARMGRIDGEN|SCOTCH)_DIR', # User-specific # ------------- 'FOAM_RUN', '(FOAM|WM)_.*USER_.*', ], whitelist=[ # Whitelist these 'MPI_ARCH_PATH', # Can be needed for compilation 'PYTHON_BIN_DIR', ]) run_env.extend(mods) minimal = False tty.info('foam-extend env: {0}'.format(bashrc)) except Exception: minimal = True if minimal: # pre-build or minimal environment tty.info('foam-extend minimal env {0}'.format(self.prefix)) run_env.set('FOAM_INST_DIR', os.path.dirname(self.projectdir)), run_env.set('FOAM_PROJECT_DIR', self.projectdir) run_env.set('WM_PROJECT_DIR', self.projectdir) for d in ['wmake', self.archbin]: # bin added automatically run_env.prepend_path('PATH', join_path(self.projectdir, d))
def clean_environment(): # Stuff in here sanitizes the build environment to eliminate # anything the user has set that may interfere. We apply it immediately # unlike the other functions so it doesn't overwrite what the modules load. env = EnvironmentModifications() # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') env.unset('DYLD_FALLBACK_LIBRARY_PATH') # These vars affect how the compiler finds libraries and include dirs. env.unset('LIBRARY_PATH') env.unset('CPATH') env.unset('C_INCLUDE_PATH') env.unset('CPLUS_INCLUDE_PATH') env.unset('OBJC_INCLUDE_PATH') # On Cray "cluster" systems, unset CRAY_LD_LIBRARY_PATH to avoid # interference with Spack dependencies. # CNL requires these variables to be set (or at least some of them, # depending on the CNL version). hostarch = arch.Arch(arch.platform(), 'default_os', 'default_target') on_cray = str(hostarch.platform) == 'cray' using_cnl = re.match(r'cnl\d+', str(hostarch.os)) if on_cray and not using_cnl: env.unset('CRAY_LD_LIBRARY_PATH') for varname in os.environ.keys(): if 'PKGCONF' in varname: env.unset(varname) # Unset the following variables because they can affect installation of # Autotools and CMake packages. build_system_vars = [ 'CC', 'CFLAGS', 'CPP', 'CPPFLAGS', # C variables 'CXX', 'CCC', 'CXXFLAGS', 'CXXCPP', # C++ variables 'F77', 'FFLAGS', 'FLIBS', # Fortran77 variables 'FC', 'FCFLAGS', 'FCLIBS', # Fortran variables 'LDFLAGS', 'LIBS' # linker variables ] for v in build_system_vars: env.unset(v) # Unset mpi environment vars. These flags should only be set by # mpi providers for packages with mpi dependencies mpi_vars = ['MPICC', 'MPICXX', 'MPIFC', 'MPIF77', 'MPIF90'] for v in mpi_vars: env.unset(v) build_lang = spack.config.get('config:build_language') if build_lang: # Override language-related variables. This can be used to force # English compiler messages etc., which allows parse_log_events to # show useful matches. env.set('LC_ALL', build_lang) # Remove any macports installs from the PATH. The macports ld can # cause conflicts with the built-in linker on el capitan. Solves # assembler issues, e.g.: # suffix or operands invalid for `movq'" path = get_path('PATH') for p in path: if '/macports/' in p: env.remove_path('PATH', p) env.apply_modifications()
def env(prepare_environment_for_tests): """Returns an empty EnvironmentModifications object.""" return EnvironmentModifications()
def setup_package(pkg, dirty): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, spack_env) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False, deptype=('build', 'test')): # If a user makes their own package repo, e.g. # spack.pkg.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.pkg.original.libelf.Libelf, # then set the module variables for both classes so the # parent class can still use them if it gets called. spkg = dspec.package modules = parent_class_modules(spkg.__class__) for mod in modules: set_module_variables_for_package(spkg, mod) set_module_variables_for_package(spkg, spkg.module) # Allow dependencies to modify the module dpkg = dspec.package dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) set_module_variables_for_package(pkg, pkg.module) pkg.setup_environment(spack_env, run_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the spack_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) # Make sure nothing's strange about the Spack environment. validate(spack_env, tty.warn) spack_env.apply_modifications()
class Executable(object): """Class representing a program that can be run on the command line.""" def __init__(self, name): self.exe = shlex.split(str(name)) self.default_env = {} from spack.util.environment import EnvironmentModifications # no cycle self.default_envmod = EnvironmentModifications() self.returncode = None if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name) def add_default_arg(self, arg): """Add a default argument to the command.""" self.exe.append(arg) def add_default_env(self, key, value): """Set an environment variable when the command is run. Parameters: key: The environment variable to set value: The value to set it to """ self.default_env[key] = value def add_default_envmod(self, envmod): """Set an EnvironmentModifications to use when the command is run.""" self.default_envmod.extend(envmod) @property def command(self): """The command-line string. Returns: str: The executable and default arguments """ return ' '.join(self.exe) @property def name(self): """The executable name. Returns: str: The basename of the executable """ return os.path.basename(self.path) @property def path(self): """The path to the executable. Returns: str: The path to the executable """ return self.exe[0] def __call__(self, *args, **kwargs): """Run this executable in a subprocess. Parameters: *args (str): Command-line arguments to the executable to run Keyword Arguments: _dump_env (dict): Dict to be set to the environment actually used (envisaged for testing purposes only) env (dict or EnvironmentModifications): The environment with which to run the executable extra_env (dict or EnvironmentModifications): Extra items to add to the environment (neither requires nor precludes env) fail_on_error (bool): Raise an exception if the subprocess returns an error. Default is True. The return code is available as ``exe.returncode`` ignore_errors (int or list): A list of error codes to ignore. If these codes are returned, this process will not raise an exception even if ``fail_on_error`` is set to ``True`` ignore_quotes (bool): If False, warn users that quotes are not needed as Spack does not use a shell. Defaults to False. input: Where to read stdin from output: Where to send stdout error: Where to send stderr Accepted values for input, output, and error: * python streams, e.g. open Python file objects, or ``os.devnull`` * filenames, which will be automatically opened for writing * ``str``, as in the Python string type. If you set these to ``str``, output and error will be written to pipes and returned as a string. If both ``output`` and ``error`` are set to ``str``, then one string is returned containing output concatenated with error. Not valid for ``input`` * ``str.split``, as in the ``split`` method of the Python string type. Behaves the same as ``str``, except that value is also written to ``stdout`` or ``stderr``. By default, the subprocess inherits the parent's file descriptors. """ # Environment env_arg = kwargs.get('env', None) # Setup default environment env = os.environ.copy() if env_arg is None else {} self.default_envmod.apply_modifications(env) env.update(self.default_env) from spack.util.environment import EnvironmentModifications # no cycle # Apply env argument if isinstance(env_arg, EnvironmentModifications): env_arg.apply_modifications(env) elif env_arg: env.update(env_arg) # Apply extra env extra_env = kwargs.get('extra_env', {}) if isinstance(extra_env, EnvironmentModifications): extra_env.apply_modifications(env) else: env.update(extra_env) if '_dump_env' in kwargs: kwargs['_dump_env'].clear() kwargs['_dump_env'].update(env) fail_on_error = kwargs.pop('fail_on_error', True) ignore_errors = kwargs.pop('ignore_errors', ()) ignore_quotes = kwargs.pop('ignore_quotes', False) # If they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): ignore_errors = (ignore_errors, ) input = kwargs.pop('input', None) output = kwargs.pop('output', None) error = kwargs.pop('error', None) if input is str: raise ValueError('Cannot use `str` as input stream.') def streamify(arg, mode): if isinstance(arg, string_types): return open(arg, mode), True elif arg in (str, str.split): return subprocess.PIPE, False else: return arg, False ostream, close_ostream = streamify(output, 'w') estream, close_estream = streamify(error, 'w') istream, close_istream = streamify(input, 'r') if not ignore_quotes: quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] if quoted_args: tty.warn( "Quotes in command arguments can confuse scripts like" " configure.", "The following arguments may cause problems when executed:", str("\n".join([" " + arg for arg in quoted_args])), "Quotes aren't needed because spack doesn't use a shell. " "Consider removing them.", "If multiple levels of quotation are required, use " "`ignore_quotes=True`.") cmd = self.exe + list(args) cmd_line = "'%s'" % "' '".join( map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) tty.debug(cmd_line) try: proc = subprocess.Popen( cmd, stdin=istream, stderr=estream, stdout=ostream, env=env) out, err = proc.communicate() result = None if output in (str, str.split) or error in (str, str.split): result = '' if output in (str, str.split): outstr = text_type(out.decode('utf-8')) result += outstr if output is str.split: sys.stdout.write(outstr) if error in (str, str.split): errstr = text_type(err.decode('utf-8')) result += errstr if error is str.split: sys.stderr.write(errstr) rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): long_msg = cmd_line if result: # If the output is not captured in the result, it will have # been stored either in the specified files (e.g. if # 'output' specifies a file) or written to the parent's # stdout/stderr (e.g. if 'output' is not specified) long_msg += '\n' + result raise ProcessError('Command exited with status %d:' % proc.returncode, long_msg) return result except OSError as e: raise ProcessError( '%s: %s' % (self.exe[0], e.strerror), 'Command: ' + cmd_line) except subprocess.CalledProcessError as e: if fail_on_error: raise ProcessError( str(e), '\nExit status %d when invoking command: %s' % (proc.returncode, cmd_line)) finally: if close_ostream: ostream.close() if close_estream: estream.close() if close_istream: istream.close() def __eq__(self, other): return hasattr(other, 'exe') and self.exe == other.exe def __neq__(self, other): return not (self == other) def __hash__(self): return hash((type(self), ) + tuple(self.exe)) def __repr__(self): return '<exe: %s>' % self.exe def __str__(self): return ' '.join(self.exe)
def setup_run_environment(self, env): filename = self.prefix.etc.join('profile.d').join('conda.sh') env.extend(EnvironmentModifications.from_sourcing_file(filename))
def setup_environment(self, spack_env, run_env): """Add environment variables to the generated module file. These environment variables come from running: .. code-block:: console $ . $WM_PROJECT_DIR/etc/bashrc """ # NOTE: Spack runs setup_environment twice. # 1) pre-build to set up the build environment # 2) post-install to determine runtime environment variables # The etc/bashrc is only available (with corrrect content) # post-installation. bashrc = join_path(self.projectdir, 'etc', 'bashrc') minimal = True if os.path.isfile(bashrc): # post-install: source the installed bashrc try: mods = EnvironmentModifications.from_sourcing_file( bashrc, clean=True, # Remove duplicate entries blacklist=[ # Blacklist these # Inadvertent changes # ------------------- 'PS1', # Leave unaffected 'MANPATH', # Leave unaffected # Unneeded bits # ------------- # 'FOAM_SETTINGS', # Do not use with modules # 'FOAM_INST_DIR', # Old # 'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)', # 'FOAM_TUTORIALS', # can be useful # 'WM_OSTYPE', # Purely optional value # Third-party cruft - only used for orig compilation # ----------------- '[A-Z].*_ARCH_PATH', # '(KAHIP|METIS|SCOTCH)_VERSION', # User-specific # ------------- 'FOAM_RUN', '(FOAM|WM)_.*USER_.*', ], whitelist=[ # Whitelist these 'MPI_ARCH_PATH', # Can be needed for compilation ]) run_env.extend(mods) spack_env.extend(mods) minimal = False tty.info('OpenFOAM bashrc env: {0}'.format(bashrc)) except Exception: minimal = True if minimal: # pre-build or minimal environment tty.info('OpenFOAM minimal env {0}'.format(self.prefix)) run_env.set('FOAM_PROJECT_DIR', self.projectdir) run_env.set('WM_PROJECT_DIR', self.projectdir) spack_env.set('FOAM_PROJECT_DIR', self.projectdir) spack_env.set('WM_PROJECT_DIR', self.projectdir) for d in ['wmake', self.archbin]: # bin added automatically run_env.prepend_path('PATH', join_path(self.projectdir, d)) spack_env.prepend_path('PATH', join_path(self.projectdir, d))
def clean_environment(): # Stuff in here sanitizes the build environment to eliminate # anything the user has set that may interfere. We apply it immediately # unlike the other functions so it doesn't overwrite what the modules load. env = EnvironmentModifications() # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') env.unset('LIBRARY_PATH') env.unset('CPATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') build_lang = spack.config.get('config:build_language') if build_lang: # Override language-related variables. This can be used to force # English compiler messages etc., which allows parse_log_events to # show useful matches. env.set('LC_ALL', build_lang) # Remove any macports installs from the PATH. The macports ld can # cause conflicts with the built-in linker on el capitan. Solves # assembler issues, e.g.: # suffix or operands invalid for `movq'" path = get_path('PATH') for p in path: if '/macports/' in p: env.remove_path('PATH', p) env.apply_modifications()