def test_external_prefixes_last(mutable_config, mock_packages, working_env,
monkeypatch):
# Sanity check: under normal circumstances paths associated with
# dt-diamond-left would appear first. We'll mark it as external in
# the test to check if the associated paths are placed last.
assert 'dt-diamond-left' < 'dt-diamond-right'
cfg_data = syaml.load_config("""\
dt-diamond-left:
externals:
- spec: [email protected]
prefix: /fake/path1
buildable: false
""")
spack.config.set("packages", cfg_data)
top = spack.spec.Spec('dt-diamond').concretized()
def _trust_me_its_a_dir(path):
return True
monkeypatch.setattr(os.path, 'isdir', _trust_me_its_a_dir)
env_mods = EnvironmentModifications()
spack.build_environment.set_wrapper_variables(top.package, env_mods)
env_mods.apply_modifications()
link_dir_var = os.environ['SPACK_LINK_DIRS']
link_dirs = link_dir_var.split(':')
external_lib_paths = set(['/fake/path1/lib', '/fake/path1/lib64'])
# The external lib paths should be the last two entries of the list and
# should not appear anywhere before the last two entries
assert (set(os.path.normpath(x)
for x in link_dirs[-2:]) == external_lib_paths)
assert not (set(os.path.normpath(x)
for x in link_dirs[:-2]) & external_lib_paths)
def test_set_build_environment_variables(config, mock_packages, working_env,
monkeypatch,
installation_dir_with_headers):
"""Check that build_environment supplies the needed library/include
directories via the SPACK_LINK_DIRS and SPACK_INCLUDE_DIRS environment
variables.
"""
root = spack.spec.Spec('dt-diamond')
root.concretize()
for s in root.traverse():
s.prefix = '/{0}-prefix/'.format(s.name)
dep_pkg = root['dt-diamond-left'].package
dep_lib_paths = ['/test/path/to/ex1.so', '/test/path/to/subdir/ex2.so']
dep_lib_dirs = ['/test/path/to', '/test/path/to/subdir']
dep_libs = LibraryList(dep_lib_paths)
dep2_pkg = root['dt-diamond-right'].package
dep2_pkg.spec.prefix = str(installation_dir_with_headers)
setattr(dep_pkg, 'libs', dep_libs)
try:
pkg = root.package
env_mods = EnvironmentModifications()
spack.build_environment.set_build_environment_variables(pkg,
env_mods,
dirty=False)
env_mods.apply_modifications()
def normpaths(paths):
return list(os.path.normpath(p) for p in paths)
link_dir_var = os.environ['SPACK_LINK_DIRS']
assert (normpaths(link_dir_var.split(':')) == normpaths(dep_lib_dirs))
root_libdirs = ['/dt-diamond-prefix/lib', '/dt-diamond-prefix/lib64']
rpath_dir_var = os.environ['SPACK_RPATH_DIRS']
# The 'lib' and 'lib64' subdirectories of the root package prefix
# should always be rpathed and should be the first rpaths
assert (normpaths(rpath_dir_var.split(':')) == normpaths(root_libdirs +
dep_lib_dirs))
header_dir_var = os.environ['SPACK_INCLUDE_DIRS']
# The default implementation looks for header files only
# in <prefix>/include and subdirectories
prefix = str(installation_dir_with_headers)
include_dirs = normpaths(header_dir_var.split(':'))
assert os.path.join(prefix, 'include') in include_dirs
assert os.path.join(prefix, 'include', 'boost') not in include_dirs
assert os.path.join(prefix, 'path', 'to') not in include_dirs
assert os.path.join(prefix, 'path', 'to', 'subdir') not in include_dirs
finally:
delattr(dep_pkg, 'libs')
def __init__(self, name):
self.exe = shlex.split(str(name))
self.default_env = {}
from spack.util.environment import EnvironmentModifications # no cycle
self.default_envmod = EnvironmentModifications()
self.returncode = None
if not self.exe:
raise ProcessError("Cannot construct executable for '%s'" % name)
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
build_env = EnvironmentModifications()
if not dirty:
clean_environment()
set_compiler_environment_variables(pkg, build_env)
set_build_environment_variables(pkg, build_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, build_env)
build_env.extend(
modifications_from_dependencies(pkg.spec, context='build')
)
if (not dirty) and (not build_env.is_unset('CPATH')):
tty.debug("A dependency has updated CPATH, this may lead pkg-config"
" to assume that the package is part of the system"
" includes and omit it when invoked with '--cflags'.")
set_module_variables_for_package(pkg)
pkg.setup_build_environment(build_env)
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
# build of packages. To avoid a polluted environment, preserve the
# value of a few, selected, environment variables
# With the current ordering of environment modifications, this is strictly
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the build_env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
# kludge to handle cray libsci being automatically loaded by PrgEnv
# modules on cray platform. Module unload does no damage when
# unnecessary
module('unload', 'cray-libsci')
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
implicit_rpaths = pkg.compiler.implicit_rpaths()
if implicit_rpaths:
build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS',
':'.join(implicit_rpaths))
# Make sure nothing's strange about the Spack environment.
validate(build_env, tty.warn)
build_env.apply_modifications()
def __init__(self, name):
# necesary here for the shlex call to succeed
name = format_os_path(name, mode=Path.unix)
self.exe = shlex.split(str(name))
# filter back to platform dependent path
self.exe = path_to_os_path(*self.exe)
self.default_env = {}
from spack.util.environment import EnvironmentModifications # no cycle
self.default_envmod = EnvironmentModifications()
self.returncode = None
if not self.exe:
raise ProcessError("Cannot construct executable for '%s'" % name)
def deactivate():
"""
Deactivate an environment and collect corresponding environment modifications.
Note: unloads the environment in its current state, not in the state it was
loaded in, meaning that specs that were removed from the spack environment
after activation are not unloaded.
Returns:
spack.util.environment.EnvironmentModifications: Environment variables
modifications to activate environment.
"""
env_mods = EnvironmentModifications()
active = ev.active_environment()
if active is None:
return env_mods
if ev.default_view_name in active.views:
try:
with spack.store.db.read_transaction():
active.rm_default_view_from_env(env_mods)
except (spack.repo.UnknownPackageError,
spack.repo.UnknownNamespaceError) as e:
tty.warn(e)
tty.warn('Could not fully deactivate view due to missing package '
'or repo, shell environment may be corrupt.')
ev.deactivate()
return env_mods
def setup_environment(self, spack_env, run_env):
if not self.stage.source_path:
self.stage.fetch()
self.stage.expand_archive()
spack_env.set('FSLDIR', self.stage.source_path)
# Here, run-time environment variables are being set manually.
# Normally these would be added to the modulefile at build-time
# by sourcing fsl.sh, but incorrect paths were being set, pointing to
# the staging directory rather than the install directory.
run_env.set('FSLDIR', self.prefix)
run_env.set('FSLOUTPUTTYPE', 'NIFTI_GZ')
run_env.set('FSLMULTIFILEQUIT', 'TRUE')
run_env.set('FSLTCLSH', self.prefix.bin.fsltclsh)
run_env.set('FSLWISH', self.prefix.bin.fslwish)
run_env.set('FSLLOCKDIR', '')
run_env.set('FSLMACHINELIST', '')
run_env.set('FSLREMOTECALL', '')
run_env.set('FSLGECUDAQ', 'cuda.q')
run_env.prepend_path('PATH', self.prefix)
# Below is for sourcing purposes during building
fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf',
'fsl.sh')
if os.path.isfile(fslsetup):
spack_env.extend(EnvironmentModifications.from_sourcing_file(
fslsetup))
def setup_environment(self, spack_env, run_env):
if not self.stage.source_path:
self.stage.fetch()
self.stage.expand_archive()
spack_env.set('FSLDIR', self.stage.source_path)
# Here, run-time environment variables are being set manually.
# Normally these would be added to the modulefile at build-time
# by sourcing fsl.sh, but incorrect paths were being set, pointing to
# the staging directory rather than the install directory.
run_env.set('FSLDIR', self.prefix)
run_env.set('FSLOUTPUTTYPE', 'NIFTI_GZ')
run_env.set('FSLMULTIFILEQUIT', 'TRUE')
run_env.set('FSLTCLSH', self.prefix.bin.fsltclsh)
run_env.set('FSLWISH', self.prefix.bin.fslwish)
run_env.set('FSLLOCKDIR', '')
run_env.set('FSLMACHINELIST', '')
run_env.set('FSLREMOTECALL', '')
run_env.set('FSLGECUDAQ', 'cuda.q')
run_env.prepend_path('PATH', self.prefix)
# Below is for sourcing purposes during building
fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf',
'fsl.sh')
if os.path.isfile(fslsetup):
spack_env.extend(EnvironmentModifications.from_sourcing_file(
fslsetup))
def modifications_from_dependencies(spec, context):
"""Returns the environment modifications that are required by
the dependencies of a spec and also applies modifications
to this spec's package at module scope, if need be.
Args:
spec (Spec): spec for which we want the modifications
context (str): either 'build' for build-time modifications or 'run'
for run-time modifications
"""
env = EnvironmentModifications()
pkg = spec.package
# Maps the context to deptype and method to be called
deptype_and_method = {
'build':
(('build', 'link', 'test'), 'setup_dependent_build_environment'),
'run': (('link', 'run'), 'setup_dependent_run_environment'),
'test': (('link', 'run', 'test'), 'setup_dependent_run_environment')
}
deptype, method = deptype_and_method[context]
root = context == 'test'
for dspec in spec.traverse(order='post', root=root, deptype=deptype):
dpkg = dspec.package
set_module_variables_for_package(dpkg)
# Allow dependencies to modify the module
dpkg.setup_dependent_package(pkg.module, spec)
getattr(dpkg, method)(env, spec)
return env
def setup_run_environment(self, env):
# Set the environment variables after copying tree
env.set('FSLDIR', self.prefix)
fslsetup = join_path(self.prefix, 'etc', 'fslconf', 'fsl.sh')
if os.path.isfile(fslsetup):
env.extend(EnvironmentModifications.from_sourcing_file(fslsetup))
def deactivate():
"""
Deactivate an environment and collect corresponding environment modifications
Returns:
spack.util.environment.EnvironmentModifications: Environment variables
modifications to activate environment.
"""
env_mods = EnvironmentModifications()
active = ev.active_environment()
if active is None:
return env_mods
if ev.default_view_name in active.views:
try:
with spack.store.db.read_transaction():
active.rm_default_view_from_env(env_mods)
except (spack.repo.UnknownPackageError,
spack.repo.UnknownNamespaceError) as e:
tty.warn(e)
tty.warn('Could not fully deactivate view due to missing package '
'or repo, shell environment may be corrupt.')
ev.deactivate()
return env_mods
def test_blacklist_lmod_variables():
# Construct the list of environment modifications
file = os.path.join(datadir, 'sourceme_lmod.sh')
env = EnvironmentModifications.from_sourcing_file(file)
# Check that variables related to lmod are not in there
modifications = env.group_by_name()
assert not any(x.startswith('LMOD_') for x in modifications)
def setup_run_environment(self, env):
try:
env.extend(EnvironmentModifications.from_sourcing_file(
join_path(self.spec.prefix, 'headas-config_spack.sh'), clean=True
))
except Exception as e:
msg = 'unexpected error when sourcing HEASOFT setup [{0}]'
tty.warn(msg.format(str(e)))
def test_setting_dtags_based_on_config(config_setting, expected_flag, config,
mock_packages):
# Pick a random package to be able to set compiler's variables
s = spack.spec.Spec('cmake')
s.concretize()
pkg = s.package
env = EnvironmentModifications()
with spack.config.override('config:shared_linking', config_setting):
spack.build_environment.set_compiler_environment_variables(pkg, env)
modifications = env.group_by_name()
assert 'SPACK_DTAGS_TO_STRIP' in modifications
assert 'SPACK_DTAGS_TO_ADD' in modifications
assert len(modifications['SPACK_DTAGS_TO_ADD']) == 1
assert len(modifications['SPACK_DTAGS_TO_STRIP']) == 1
dtags_to_add = modifications['SPACK_DTAGS_TO_ADD'][0]
assert dtags_to_add.value == expected_flag
def setup_run_environment(self, env):
bashrc = self.prefix.etc.bashrc
try:
env.extend(EnvironmentModifications.from_sourcing_file(
bashrc, clean=True
))
except Exception as e:
msg = 'unexpected error when sourcing OpenFOAM bashrc [{0}]'
tty.warn(msg.format(str(e)))
def setup_run_environment(self, env):
"""Sets the run environment (post-installation).
The environment comes from running:
.. code-block:: console
$ . $WM_PROJECT_DIR/etc/bashrc
"""
bashrc = join_path(self.projectdir, 'etc', 'bashrc')
minimal = True
if os.path.isfile(bashrc):
# post-install: source the installed bashrc
try:
mods = EnvironmentModifications.from_sourcing_file(
bashrc,
clean=True, # Remove duplicate entries
blacklist=[ # Blacklist these
# Inadvertent changes
# -------------------
'PS1', # Leave untouched
'MANPATH', # Leave untouched
# Unneeded bits
# -------------
# 'FOAM_SETTINGS', # Do not use with modules
# 'FOAM_INST_DIR', # Old
# 'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)',
# 'FOAM_TUTORIALS', # May be useful
# 'WM_OSTYPE', # Purely optional value
# Third-party cruft - only used for orig compilation
# -----------------
'[A-Z].*_ARCH_PATH',
# '(KAHIP|METIS|SCOTCH)_VERSION',
# User-specific
# -------------
'FOAM_RUN',
'(FOAM|WM)_.*USER_.*',
],
whitelist=[ # Whitelist these
'MPI_ARCH_PATH', # Can be required for compilation
])
env.extend(mods)
minimal = False
tty.info('OpenFOAM bashrc env: {0}'.format(bashrc))
except Exception:
minimal = True
if minimal:
# pre-build or minimal environment
self.setup_minimal_environment(env)
def test_source_files(files_to_be_sourced):
"""Tests the construction of a list of environment modifications that are
the result of sourcing a file.
"""
env = EnvironmentModifications()
for filename in files_to_be_sourced:
if filename.endswith('sourceme_parameters.sh'):
env.extend(
EnvironmentModifications.from_sourcing_file(
filename, 'intel64'))
else:
env.extend(EnvironmentModifications.from_sourcing_file(filename))
modifications = env.group_by_name()
# This is sensitive to the user's environment; can include
# spurious entries for things like PS1
#
# TODO: figure out how to make a bit more robust.
assert len(modifications) >= 5
# Set new variables
assert len(modifications['NEW_VAR']) == 1
assert isinstance(modifications['NEW_VAR'][0], SetEnv)
assert modifications['NEW_VAR'][0].value == 'new'
assert len(modifications['FOO']) == 1
assert isinstance(modifications['FOO'][0], SetEnv)
assert modifications['FOO'][0].value == 'intel64'
# Unset variables
assert len(modifications['EMPTY_PATH_LIST']) == 1
assert isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv)
# Modified variables
assert len(modifications['UNSET_ME']) == 1
assert isinstance(modifications['UNSET_ME'][0], SetEnv)
assert modifications['UNSET_ME'][0].value == 'overridden'
assert len(modifications['PATH_LIST']) == 3
assert isinstance(modifications['PATH_LIST'][0], RemovePath)
assert modifications['PATH_LIST'][0].value == '/path/third'
assert isinstance(modifications['PATH_LIST'][1], AppendPath)
assert modifications['PATH_LIST'][1].value == '/path/fourth'
assert isinstance(modifications['PATH_LIST'][2], PrependPath)
assert modifications['PATH_LIST'][2].value == '/path/first'
def setup_run_environment(self, env):
"""Adds environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ source {prefix}/setvars.sh --force
"""
env.extend(
EnvironmentModifications.from_sourcing_file(
join(self.prefix, self.component_dir, 'latest/env/vars.sh')))
def test_extend(env):
"""Tests that we can construct a list of environment modifications
starting from another list.
"""
env.set('A', 'dummy value')
env.set('B', 3)
copy_construct = EnvironmentModifications(env)
assert len(copy_construct) == 2
for x, y in zip(env, copy_construct):
assert x is y
def setup_run_environment(self, env):
"""Adds environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ source {prefix}/{component}/{version}/env/vars.sh
"""
env.extend(
EnvironmentModifications.from_sourcing_file(
join_path(self.component_path, 'env', 'vars.sh')))
def setup_build_environment(self, env):
if not self.stage.source_path:
self.stage.fetch()
self.stage.expand_archive()
env.set('FSLDIR', self.stage.source_path)
# Below is for sourcing purposes during building
fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf',
'fsl.sh')
if os.path.isfile(fslsetup):
env.extend(EnvironmentModifications.from_sourcing_file(fslsetup))
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
if not dirty:
clean_environment()
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post',
root=False,
deptype=('build', 'test')):
spkg = dspec.package
set_module_variables_for_package(spkg)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
if (not dirty) and (not spack_env.is_unset('CPATH')):
tty.debug("A dependency has updated CPATH, this may lead pkg-config"
" to assume that the package is part of the system"
" includes and omit it when invoked with '--cflags'.")
set_module_variables_for_package(pkg)
pkg.setup_environment(spack_env, run_env)
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
# build of packages. To avoid a polluted environment, preserve the
# value of a few, selected, environment variables
# With the current ordering of environment modifications, this is strictly
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the spack_env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
def _read_environment_file(self, filename):
"""
Read and parse the environment file.
Given an environment file, we want to read it, split by semicolons
and new lines, and then parse down to the subset of SPACK_* variables.
We assume that all spack prefix variables are not secrets, and unlike
the install_manifest.json, we don't (at least to start) parse the values
to remove path prefixes specific to user systems.
"""
if not os.path.exists(filename):
return
mods = EnvironmentModifications.from_sourcing_file(filename)
env = {}
mods.apply_modifications(env)
return env
def activate(env, use_env_repo=False, add_view=True):
"""
Activate an environment and append environment modifications
To activate an environment, we add its configuration scope to the
existing Spack configuration, and we set active to the current
environment.
Arguments:
env (spack.environment.Environment): the environment to activate
use_env_repo (bool): use the packages exactly as they appear in the
environment's repository
add_view (bool): generate commands to add view to path variables
Returns:
spack.util.environment.EnvironmentModifications: Environment variables
modifications to activate environment.
"""
ev.activate(env, use_env_repo=use_env_repo)
env_mods = EnvironmentModifications()
#
# NOTE in the fish-shell: Path variables are a special kind of variable
# used to support colon-delimited path lists including PATH, CDPATH,
# MANPATH, PYTHONPATH, etc. All variables that end in PATH (case-sensitive)
# become PATH variables.
#
try:
if add_view and ev.default_view_name in env.views:
with spack.store.db.read_transaction():
env.add_default_view_to_env(env_mods)
except (spack.repo.UnknownPackageError,
spack.repo.UnknownNamespaceError) as e:
tty.error(e)
tty.die(
'Environment view is broken due to a missing package or repo.\n',
' To activate without views enabled, activate with:\n',
' spack env activate -V {0}\n'.format(env.name),
' To remove it and resolve the issue, '
'force concretize with the command:\n',
' spack -e {0} concretize --force'.format(env.name))
return env_mods
def setup_environment(self, spack_env, run_env):
"""Add environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ . $WM_PROJECT_DIR/etc/bashrc
"""
# NOTE: Spack runs setup_environment twice.
# 1) pre-build to set up the build environment
# 2) post-install to determine runtime environment variables
# The etc/bashrc is only available (with corrrect content)
# post-installation.
bashrc = join_path(self.projectdir, 'etc', 'bashrc')
minimal = True
if os.path.isfile(bashrc):
# post-install: source the installed bashrc
try:
mods = EnvironmentModifications.from_sourcing_file(
bashrc,
clean=True, # Remove duplicate entries
blacklist=[ # Blacklist these
# Inadvertent changes
# -------------------
'PS1', # Leave unaffected
'MANPATH', # Leave unaffected
# Unneeded bits
# -------------
'FOAM_INST_DIR', # Possibly incorrect
'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)',
'FOAM_TEST_.*_DIR',
'WM_NCOMPPROCS',
# 'FOAM_TUTORIALS', # can be useful
# Lots of third-party cruft
# -------------------------
'[A-Z].*_(BIN|LIB|INCLUDE)_DIR',
'[A-Z].*_SYSTEM',
'WM_THIRD_PARTY_.*',
'(BISON|FLEX|CMAKE|ZLIB)_DIR',
'(METIS|PARMETIS|PARMGRIDGEN|SCOTCH)_DIR',
# User-specific
# -------------
'FOAM_RUN',
'(FOAM|WM)_.*USER_.*',
],
whitelist=[ # Whitelist these
'MPI_ARCH_PATH', # Can be needed for compilation
'PYTHON_BIN_DIR',
])
run_env.extend(mods)
minimal = False
tty.info('foam-extend env: {0}'.format(bashrc))
except Exception:
minimal = True
if minimal:
# pre-build or minimal environment
tty.info('foam-extend minimal env {0}'.format(self.prefix))
run_env.set('FOAM_INST_DIR', os.path.dirname(self.projectdir)),
run_env.set('FOAM_PROJECT_DIR', self.projectdir)
run_env.set('WM_PROJECT_DIR', self.projectdir)
for d in ['wmake', self.archbin]: # bin added automatically
run_env.prepend_path('PATH', join_path(self.projectdir, d))
def get_executable(exe, spec=None, install=False):
"""Find an executable named exe, either in PATH or in Spack
Args:
exe (str): needed executable name
spec (Spec or str): spec to search for exe in (default exe)
install (bool): install spec if not available
When ``install`` is True, Spack will use the python used to run Spack as an
external. The ``install`` option should only be used with packages that
install quickly (when using external python) or are guaranteed by Spack
organization to be in a binary mirror (clingo)."""
# Search the system first
runner = spack.util.executable.which(exe)
if runner:
return runner
# Check whether it's already installed
spec = spack.spec.Spec(spec or exe)
installed_specs = spack.store.db.query(spec, installed=True)
for ispec in installed_specs:
# filter out directories of the same name as the executable
exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe)
if fs.is_exe(exe_p)]
if exe_path:
ret = spack.util.executable.Executable(exe_path[0])
envmod = EnvironmentModifications()
for dep in ispec.traverse(root=True, order='post'):
envmod.extend(uenv.environment_modifications_for_spec(dep))
ret.add_default_envmod(envmod)
return ret
else:
tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix))
def _raise_error(executable, exe_spec):
error_msg = 'cannot find the executable "{0}"'.format(executable)
if exe_spec:
error_msg += ' from spec "{0}'.format(exe_spec)
raise RuntimeError(error_msg)
# If we're not allowed to install this for ourselves, we can't find it
if not install:
_raise_error(exe, spec)
with spack_python_interpreter():
# We will install for ourselves, using this python if needed
# Concretize the spec
spec.concretize()
spec.package.do_install()
# filter out directories of the same name as the executable
exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe)
if fs.is_exe(exe_p)]
if exe_path:
ret = spack.util.executable.Executable(exe_path[0])
envmod = EnvironmentModifications()
for dep in spec.traverse(root=True, order='post'):
envmod.extend(uenv.environment_modifications_for_spec(dep))
ret.add_default_envmod(envmod)
return ret
_raise_error(exe, spec)
def setup_environment(self, spack_env, run_env):
"""Add environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ . $WM_PROJECT_DIR/etc/bashrc
"""
# NOTE: Spack runs setup_environment twice.
# 1) pre-build to set up the build environment
# 2) post-install to determine runtime environment variables
# The etc/bashrc is only available (with corrrect content)
# post-installation.
bashrc = join_path(self.projectdir, 'etc', 'bashrc')
minimal = True
if os.path.isfile(bashrc):
# post-install: source the installed bashrc
try:
mods = EnvironmentModifications.from_sourcing_file(
bashrc,
clean=True, # Remove duplicate entries
blacklist=[ # Blacklist these
# Inadvertent changes
# -------------------
'PS1', # Leave unaffected
'MANPATH', # Leave unaffected
# Unneeded bits
# -------------
'FOAM_INST_DIR', # Possibly incorrect
'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)',
'FOAM_TEST_.*_DIR',
'WM_NCOMPPROCS',
# 'FOAM_TUTORIALS', # can be useful
# Lots of third-party cruft
# -------------------------
'[A-Z].*_(BIN|LIB|INCLUDE)_DIR',
'[A-Z].*_SYSTEM',
'WM_THIRD_PARTY_.*',
'(BISON|FLEX|CMAKE|ZLIB)_DIR',
'(METIS|PARMETIS|PARMGRIDGEN|SCOTCH)_DIR',
# User-specific
# -------------
'FOAM_RUN',
'(FOAM|WM)_.*USER_.*',
],
whitelist=[ # Whitelist these
'MPI_ARCH_PATH', # Can be needed for compilation
'PYTHON_BIN_DIR',
])
run_env.extend(mods)
minimal = False
tty.info('foam-extend env: {0}'.format(bashrc))
except Exception:
minimal = True
if minimal:
# pre-build or minimal environment
tty.info('foam-extend minimal env {0}'.format(self.prefix))
run_env.set('FOAM_INST_DIR', os.path.dirname(self.projectdir)),
run_env.set('FOAM_PROJECT_DIR', self.projectdir)
run_env.set('WM_PROJECT_DIR', self.projectdir)
for d in ['wmake', self.archbin]: # bin added automatically
run_env.prepend_path('PATH', join_path(self.projectdir, d))
def clean_environment():
# Stuff in here sanitizes the build environment to eliminate
# anything the user has set that may interfere. We apply it immediately
# unlike the other functions so it doesn't overwrite what the modules load.
env = EnvironmentModifications()
# Remove these vars from the environment during build because they
# can affect how some packages find libraries. We want to make
# sure that builds never pull in unintended external dependencies.
env.unset('LD_LIBRARY_PATH')
env.unset('LD_RUN_PATH')
env.unset('DYLD_LIBRARY_PATH')
env.unset('DYLD_FALLBACK_LIBRARY_PATH')
# These vars affect how the compiler finds libraries and include dirs.
env.unset('LIBRARY_PATH')
env.unset('CPATH')
env.unset('C_INCLUDE_PATH')
env.unset('CPLUS_INCLUDE_PATH')
env.unset('OBJC_INCLUDE_PATH')
# On Cray "cluster" systems, unset CRAY_LD_LIBRARY_PATH to avoid
# interference with Spack dependencies.
# CNL requires these variables to be set (or at least some of them,
# depending on the CNL version).
hostarch = arch.Arch(arch.platform(), 'default_os', 'default_target')
on_cray = str(hostarch.platform) == 'cray'
using_cnl = re.match(r'cnl\d+', str(hostarch.os))
if on_cray and not using_cnl:
env.unset('CRAY_LD_LIBRARY_PATH')
for varname in os.environ.keys():
if 'PKGCONF' in varname:
env.unset(varname)
# Unset the following variables because they can affect installation of
# Autotools and CMake packages.
build_system_vars = [
'CC',
'CFLAGS',
'CPP',
'CPPFLAGS', # C variables
'CXX',
'CCC',
'CXXFLAGS',
'CXXCPP', # C++ variables
'F77',
'FFLAGS',
'FLIBS', # Fortran77 variables
'FC',
'FCFLAGS',
'FCLIBS', # Fortran variables
'LDFLAGS',
'LIBS' # linker variables
]
for v in build_system_vars:
env.unset(v)
# Unset mpi environment vars. These flags should only be set by
# mpi providers for packages with mpi dependencies
mpi_vars = ['MPICC', 'MPICXX', 'MPIFC', 'MPIF77', 'MPIF90']
for v in mpi_vars:
env.unset(v)
build_lang = spack.config.get('config:build_language')
if build_lang:
# Override language-related variables. This can be used to force
# English compiler messages etc., which allows parse_log_events to
# show useful matches.
env.set('LC_ALL', build_lang)
# Remove any macports installs from the PATH. The macports ld can
# cause conflicts with the built-in linker on el capitan. Solves
# assembler issues, e.g.:
# suffix or operands invalid for `movq'"
path = get_path('PATH')
for p in path:
if '/macports/' in p:
env.remove_path('PATH', p)
env.apply_modifications()
def env(prepare_environment_for_tests):
"""Returns an empty EnvironmentModifications object."""
return EnvironmentModifications()
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
if not dirty:
clean_environment()
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post', root=False,
deptype=('build', 'test')):
# If a user makes their own package repo, e.g.
# spack.pkg.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.pkg.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = dspec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
set_module_variables_for_package(pkg, pkg.module)
pkg.setup_environment(spack_env, run_env)
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
# build of packages. To avoid a polluted environment, preserve the
# value of a few, selected, environment variables
# With the current ordering of environment modifications, this is strictly
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the spack_env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
class Executable(object):
"""Class representing a program that can be run on the command line."""
def __init__(self, name):
self.exe = shlex.split(str(name))
self.default_env = {}
from spack.util.environment import EnvironmentModifications # no cycle
self.default_envmod = EnvironmentModifications()
self.returncode = None
if not self.exe:
raise ProcessError("Cannot construct executable for '%s'" % name)
def add_default_arg(self, arg):
"""Add a default argument to the command."""
self.exe.append(arg)
def add_default_env(self, key, value):
"""Set an environment variable when the command is run.
Parameters:
key: The environment variable to set
value: The value to set it to
"""
self.default_env[key] = value
def add_default_envmod(self, envmod):
"""Set an EnvironmentModifications to use when the command is run."""
self.default_envmod.extend(envmod)
@property
def command(self):
"""The command-line string.
Returns:
str: The executable and default arguments
"""
return ' '.join(self.exe)
@property
def name(self):
"""The executable name.
Returns:
str: The basename of the executable
"""
return os.path.basename(self.path)
@property
def path(self):
"""The path to the executable.
Returns:
str: The path to the executable
"""
return self.exe[0]
def __call__(self, *args, **kwargs):
"""Run this executable in a subprocess.
Parameters:
*args (str): Command-line arguments to the executable to run
Keyword Arguments:
_dump_env (dict): Dict to be set to the environment actually
used (envisaged for testing purposes only)
env (dict or EnvironmentModifications): The environment with which
to run the executable
extra_env (dict or EnvironmentModifications): Extra items to add to
the environment (neither requires nor precludes env)
fail_on_error (bool): Raise an exception if the subprocess returns
an error. Default is True. The return code is available as
``exe.returncode``
ignore_errors (int or list): A list of error codes to ignore.
If these codes are returned, this process will not raise
an exception even if ``fail_on_error`` is set to ``True``
ignore_quotes (bool): If False, warn users that quotes are not needed
as Spack does not use a shell. Defaults to False.
input: Where to read stdin from
output: Where to send stdout
error: Where to send stderr
Accepted values for input, output, and error:
* python streams, e.g. open Python file objects, or ``os.devnull``
* filenames, which will be automatically opened for writing
* ``str``, as in the Python string type. If you set these to ``str``,
output and error will be written to pipes and returned as a string.
If both ``output`` and ``error`` are set to ``str``, then one string
is returned containing output concatenated with error. Not valid
for ``input``
* ``str.split``, as in the ``split`` method of the Python string type.
Behaves the same as ``str``, except that value is also written to
``stdout`` or ``stderr``.
By default, the subprocess inherits the parent's file descriptors.
"""
# Environment
env_arg = kwargs.get('env', None)
# Setup default environment
env = os.environ.copy() if env_arg is None else {}
self.default_envmod.apply_modifications(env)
env.update(self.default_env)
from spack.util.environment import EnvironmentModifications # no cycle
# Apply env argument
if isinstance(env_arg, EnvironmentModifications):
env_arg.apply_modifications(env)
elif env_arg:
env.update(env_arg)
# Apply extra env
extra_env = kwargs.get('extra_env', {})
if isinstance(extra_env, EnvironmentModifications):
extra_env.apply_modifications(env)
else:
env.update(extra_env)
if '_dump_env' in kwargs:
kwargs['_dump_env'].clear()
kwargs['_dump_env'].update(env)
fail_on_error = kwargs.pop('fail_on_error', True)
ignore_errors = kwargs.pop('ignore_errors', ())
ignore_quotes = kwargs.pop('ignore_quotes', False)
# If they just want to ignore one error code, make it a tuple.
if isinstance(ignore_errors, int):
ignore_errors = (ignore_errors, )
input = kwargs.pop('input', None)
output = kwargs.pop('output', None)
error = kwargs.pop('error', None)
if input is str:
raise ValueError('Cannot use `str` as input stream.')
def streamify(arg, mode):
if isinstance(arg, string_types):
return open(arg, mode), True
elif arg in (str, str.split):
return subprocess.PIPE, False
else:
return arg, False
ostream, close_ostream = streamify(output, 'w')
estream, close_estream = streamify(error, 'w')
istream, close_istream = streamify(input, 'r')
if not ignore_quotes:
quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)]
if quoted_args:
tty.warn(
"Quotes in command arguments can confuse scripts like"
" configure.",
"The following arguments may cause problems when executed:",
str("\n".join([" " + arg for arg in quoted_args])),
"Quotes aren't needed because spack doesn't use a shell. "
"Consider removing them.",
"If multiple levels of quotation are required, use "
"`ignore_quotes=True`.")
cmd = self.exe + list(args)
cmd_line = "'%s'" % "' '".join(
map(lambda arg: arg.replace("'", "'\"'\"'"), cmd))
tty.debug(cmd_line)
try:
proc = subprocess.Popen(
cmd,
stdin=istream,
stderr=estream,
stdout=ostream,
env=env)
out, err = proc.communicate()
result = None
if output in (str, str.split) or error in (str, str.split):
result = ''
if output in (str, str.split):
outstr = text_type(out.decode('utf-8'))
result += outstr
if output is str.split:
sys.stdout.write(outstr)
if error in (str, str.split):
errstr = text_type(err.decode('utf-8'))
result += errstr
if error is str.split:
sys.stderr.write(errstr)
rc = self.returncode = proc.returncode
if fail_on_error and rc != 0 and (rc not in ignore_errors):
long_msg = cmd_line
if result:
# If the output is not captured in the result, it will have
# been stored either in the specified files (e.g. if
# 'output' specifies a file) or written to the parent's
# stdout/stderr (e.g. if 'output' is not specified)
long_msg += '\n' + result
raise ProcessError('Command exited with status %d:' %
proc.returncode, long_msg)
return result
except OSError as e:
raise ProcessError(
'%s: %s' % (self.exe[0], e.strerror), 'Command: ' + cmd_line)
except subprocess.CalledProcessError as e:
if fail_on_error:
raise ProcessError(
str(e), '\nExit status %d when invoking command: %s' %
(proc.returncode, cmd_line))
finally:
if close_ostream:
ostream.close()
if close_estream:
estream.close()
if close_istream:
istream.close()
def __eq__(self, other):
return hasattr(other, 'exe') and self.exe == other.exe
def __neq__(self, other):
return not (self == other)
def __hash__(self):
return hash((type(self), ) + tuple(self.exe))
def __repr__(self):
return '<exe: %s>' % self.exe
def __str__(self):
return ' '.join(self.exe)
def setup_run_environment(self, env):
filename = self.prefix.etc.join('profile.d').join('conda.sh')
env.extend(EnvironmentModifications.from_sourcing_file(filename))
def setup_environment(self, spack_env, run_env):
"""Add environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ . $WM_PROJECT_DIR/etc/bashrc
"""
# NOTE: Spack runs setup_environment twice.
# 1) pre-build to set up the build environment
# 2) post-install to determine runtime environment variables
# The etc/bashrc is only available (with corrrect content)
# post-installation.
bashrc = join_path(self.projectdir, 'etc', 'bashrc')
minimal = True
if os.path.isfile(bashrc):
# post-install: source the installed bashrc
try:
mods = EnvironmentModifications.from_sourcing_file(
bashrc,
clean=True, # Remove duplicate entries
blacklist=[ # Blacklist these
# Inadvertent changes
# -------------------
'PS1', # Leave unaffected
'MANPATH', # Leave unaffected
# Unneeded bits
# -------------
# 'FOAM_SETTINGS', # Do not use with modules
# 'FOAM_INST_DIR', # Old
# 'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)',
# 'FOAM_TUTORIALS', # can be useful
# 'WM_OSTYPE', # Purely optional value
# Third-party cruft - only used for orig compilation
# -----------------
'[A-Z].*_ARCH_PATH',
# '(KAHIP|METIS|SCOTCH)_VERSION',
# User-specific
# -------------
'FOAM_RUN',
'(FOAM|WM)_.*USER_.*',
],
whitelist=[ # Whitelist these
'MPI_ARCH_PATH', # Can be needed for compilation
])
run_env.extend(mods)
spack_env.extend(mods)
minimal = False
tty.info('OpenFOAM bashrc env: {0}'.format(bashrc))
except Exception:
minimal = True
if minimal:
# pre-build or minimal environment
tty.info('OpenFOAM minimal env {0}'.format(self.prefix))
run_env.set('FOAM_PROJECT_DIR', self.projectdir)
run_env.set('WM_PROJECT_DIR', self.projectdir)
spack_env.set('FOAM_PROJECT_DIR', self.projectdir)
spack_env.set('WM_PROJECT_DIR', self.projectdir)
for d in ['wmake', self.archbin]: # bin added automatically
run_env.prepend_path('PATH', join_path(self.projectdir, d))
spack_env.prepend_path('PATH', join_path(self.projectdir, d))
def clean_environment():
# Stuff in here sanitizes the build environment to eliminate
# anything the user has set that may interfere. We apply it immediately
# unlike the other functions so it doesn't overwrite what the modules load.
env = EnvironmentModifications()
# Remove these vars from the environment during build because they
# can affect how some packages find libraries. We want to make
# sure that builds never pull in unintended external dependencies.
env.unset('LD_LIBRARY_PATH')
env.unset('LIBRARY_PATH')
env.unset('CPATH')
env.unset('LD_RUN_PATH')
env.unset('DYLD_LIBRARY_PATH')
build_lang = spack.config.get('config:build_language')
if build_lang:
# Override language-related variables. This can be used to force
# English compiler messages etc., which allows parse_log_events to
# show useful matches.
env.set('LC_ALL', build_lang)
# Remove any macports installs from the PATH. The macports ld can
# cause conflicts with the built-in linker on el capitan. Solves
# assembler issues, e.g.:
# suffix or operands invalid for `movq'"
path = get_path('PATH')
for p in path:
if '/macports/' in p:
env.remove_path('PATH', p)
env.apply_modifications()
