def test_external_prefixes_last(mutable_config, mock_packages, working_env,
monkeypatch):
# Sanity check: under normal circumstances paths associated with
# dt-diamond-left would appear first. We'll mark it as external in
# the test to check if the associated paths are placed last.
assert 'dt-diamond-left' < 'dt-diamond-right'
cfg_data = syaml.load_config("""\
dt-diamond-left:
externals:
- spec: [email protected]
prefix: /fake/path1
buildable: false
""")
spack.config.set("packages", cfg_data)
top = spack.spec.Spec('dt-diamond').concretized()
def _trust_me_its_a_dir(path):
return True
monkeypatch.setattr(os.path, 'isdir', _trust_me_its_a_dir)
env_mods = EnvironmentModifications()
spack.build_environment.set_wrapper_variables(top.package, env_mods)
env_mods.apply_modifications()
link_dir_var = os.environ['SPACK_LINK_DIRS']
link_dirs = link_dir_var.split(':')
external_lib_paths = set(['/fake/path1/lib', '/fake/path1/lib64'])
# The external lib paths should be the last two entries of the list and
# should not appear anywhere before the last two entries
assert (set(os.path.normpath(x)
for x in link_dirs[-2:]) == external_lib_paths)
assert not (set(os.path.normpath(x)
for x in link_dirs[:-2]) & external_lib_paths)
def clean_environment():
# Stuff in here sanitizes the build environment to eliminate
# anything the user has set that may interfere. We apply it immediately
# unlike the other functions so it doesn't overwrite what the modules load.
env = EnvironmentModifications()
# Remove these vars from the environment during build because they
# can affect how some packages find libraries. We want to make
# sure that builds never pull in unintended external dependencies.
env.unset('LD_LIBRARY_PATH')
env.unset('LIBRARY_PATH')
env.unset('CPATH')
env.unset('LD_RUN_PATH')
env.unset('DYLD_LIBRARY_PATH')
build_lang = spack.config.get('config:build_language')
if build_lang:
# Override language-related variables. This can be used to force
# English compiler messages etc., which allows parse_log_events to
# show useful matches.
env.set('LC_ALL', build_lang)
# Remove any macports installs from the PATH. The macports ld can
# cause conflicts with the built-in linker on el capitan. Solves
# assembler issues, e.g.:
# suffix or operands invalid for `movq'"
path = get_path('PATH')
for p in path:
if '/macports/' in p:
env.remove_path('PATH', p)
env.apply_modifications()
def clean_environment():
# Stuff in here sanitizes the build environment to eliminate
# anything the user has set that may interfere. We apply it immediately
# unlike the other functions so it doesn't overwrite what the modules load.
env = EnvironmentModifications()
# Remove these vars from the environment during build because they
# can affect how some packages find libraries. We want to make
# sure that builds never pull in unintended external dependencies.
env.unset('LD_LIBRARY_PATH')
env.unset('LIBRARY_PATH')
env.unset('CPATH')
env.unset('LD_RUN_PATH')
env.unset('DYLD_LIBRARY_PATH')
build_lang = spack.config.get('config:build_language')
if build_lang:
# Override language-related variables. This can be used to force
# English compiler messages etc., which allows parse_log_events to
# show useful matches.
env.set('LC_ALL', build_lang)
# Remove any macports installs from the PATH. The macports ld can
# cause conflicts with the built-in linker on el capitan. Solves
# assembler issues, e.g.:
# suffix or operands invalid for `movq'"
path = get_path('PATH')
for p in path:
if '/macports/' in p:
env.remove_path('PATH', p)
env.apply_modifications()
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
if not dirty:
clean_environment()
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post',
root=False,
deptype=('build', 'test')):
spkg = dspec.package
set_module_variables_for_package(spkg)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
if (not dirty) and (not spack_env.is_unset('CPATH')):
tty.debug("A dependency has updated CPATH, this may lead pkg-config"
" to assume that the package is part of the system"
" includes and omit it when invoked with '--cflags'.")
set_module_variables_for_package(pkg)
pkg.setup_environment(spack_env, run_env)
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
# build of packages. To avoid a polluted environment, preserve the
# value of a few, selected, environment variables
# With the current ordering of environment modifications, this is strictly
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the spack_env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
def test_set_build_environment_variables(config, mock_packages, working_env,
monkeypatch,
installation_dir_with_headers):
"""Check that build_environment supplies the needed library/include
directories via the SPACK_LINK_DIRS and SPACK_INCLUDE_DIRS environment
variables.
"""
root = spack.spec.Spec('dt-diamond')
root.concretize()
for s in root.traverse():
s.prefix = '/{0}-prefix/'.format(s.name)
dep_pkg = root['dt-diamond-left'].package
dep_lib_paths = ['/test/path/to/ex1.so', '/test/path/to/subdir/ex2.so']
dep_lib_dirs = ['/test/path/to', '/test/path/to/subdir']
dep_libs = LibraryList(dep_lib_paths)
dep2_pkg = root['dt-diamond-right'].package
dep2_pkg.spec.prefix = str(installation_dir_with_headers)
setattr(dep_pkg, 'libs', dep_libs)
try:
pkg = root.package
env_mods = EnvironmentModifications()
spack.build_environment.set_build_environment_variables(pkg,
env_mods,
dirty=False)
env_mods.apply_modifications()
def normpaths(paths):
return list(os.path.normpath(p) for p in paths)
link_dir_var = os.environ['SPACK_LINK_DIRS']
assert (normpaths(link_dir_var.split(':')) == normpaths(dep_lib_dirs))
root_libdirs = ['/dt-diamond-prefix/lib', '/dt-diamond-prefix/lib64']
rpath_dir_var = os.environ['SPACK_RPATH_DIRS']
# The 'lib' and 'lib64' subdirectories of the root package prefix
# should always be rpathed and should be the first rpaths
assert (normpaths(rpath_dir_var.split(':')) == normpaths(root_libdirs +
dep_lib_dirs))
header_dir_var = os.environ['SPACK_INCLUDE_DIRS']
# The default implementation looks for header files only
# in <prefix>/include and subdirectories
prefix = str(installation_dir_with_headers)
include_dirs = normpaths(header_dir_var.split(':'))
assert os.path.join(prefix, 'include') in include_dirs
assert os.path.join(prefix, 'include', 'boost') not in include_dirs
assert os.path.join(prefix, 'path', 'to') not in include_dirs
assert os.path.join(prefix, 'path', 'to', 'subdir') not in include_dirs
finally:
delattr(dep_pkg, 'libs')
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
build_env = EnvironmentModifications()
if not dirty:
clean_environment()
set_compiler_environment_variables(pkg, build_env)
set_build_environment_variables(pkg, build_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, build_env)
build_env.extend(
modifications_from_dependencies(pkg.spec, context='build')
)
if (not dirty) and (not build_env.is_unset('CPATH')):
tty.debug("A dependency has updated CPATH, this may lead pkg-config"
" to assume that the package is part of the system"
" includes and omit it when invoked with '--cflags'.")
set_module_variables_for_package(pkg)
pkg.setup_build_environment(build_env)
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
# build of packages. To avoid a polluted environment, preserve the
# value of a few, selected, environment variables
# With the current ordering of environment modifications, this is strictly
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the build_env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
# kludge to handle cray libsci being automatically loaded by PrgEnv
# modules on cray platform. Module unload does no damage when
# unnecessary
module('unload', 'cray-libsci')
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
implicit_rpaths = pkg.compiler.implicit_rpaths()
if implicit_rpaths:
build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS',
':'.join(implicit_rpaths))
# Make sure nothing's strange about the Spack environment.
validate(build_env, tty.warn)
build_env.apply_modifications()
class Executable(object):
"""Class representing a program that can be run on the command line."""
def __init__(self, name):
self.exe = shlex.split(str(name))
self.default_env = {}
from spack.util.environment import EnvironmentModifications # no cycle
self.default_envmod = EnvironmentModifications()
self.returncode = None
if not self.exe:
raise ProcessError("Cannot construct executable for '%s'" % name)
def add_default_arg(self, arg):
"""Add a default argument to the command."""
self.exe.append(arg)
def add_default_env(self, key, value):
"""Set an environment variable when the command is run.
Parameters:
key: The environment variable to set
value: The value to set it to
"""
self.default_env[key] = value
def add_default_envmod(self, envmod):
"""Set an EnvironmentModifications to use when the command is run."""
self.default_envmod.extend(envmod)
@property
def command(self):
"""The command-line string.
Returns:
str: The executable and default arguments
"""
return ' '.join(self.exe)
@property
def name(self):
"""The executable name.
Returns:
str: The basename of the executable
"""
return os.path.basename(self.path)
@property
def path(self):
"""The path to the executable.
Returns:
str: The path to the executable
"""
return self.exe[0]
def __call__(self, *args, **kwargs):
"""Run this executable in a subprocess.
Parameters:
*args (str): Command-line arguments to the executable to run
Keyword Arguments:
_dump_env (dict): Dict to be set to the environment actually
used (envisaged for testing purposes only)
env (dict or EnvironmentModifications): The environment with which
to run the executable
extra_env (dict or EnvironmentModifications): Extra items to add to
the environment (neither requires nor precludes env)
fail_on_error (bool): Raise an exception if the subprocess returns
an error. Default is True. The return code is available as
``exe.returncode``
ignore_errors (int or list): A list of error codes to ignore.
If these codes are returned, this process will not raise
an exception even if ``fail_on_error`` is set to ``True``
ignore_quotes (bool): If False, warn users that quotes are not needed
as Spack does not use a shell. Defaults to False.
input: Where to read stdin from
output: Where to send stdout
error: Where to send stderr
Accepted values for input, output, and error:
* python streams, e.g. open Python file objects, or ``os.devnull``
* filenames, which will be automatically opened for writing
* ``str``, as in the Python string type. If you set these to ``str``,
output and error will be written to pipes and returned as a string.
If both ``output`` and ``error`` are set to ``str``, then one string
is returned containing output concatenated with error. Not valid
for ``input``
* ``str.split``, as in the ``split`` method of the Python string type.
Behaves the same as ``str``, except that value is also written to
``stdout`` or ``stderr``.
By default, the subprocess inherits the parent's file descriptors.
"""
# Environment
env_arg = kwargs.get('env', None)
# Setup default environment
env = os.environ.copy() if env_arg is None else {}
self.default_envmod.apply_modifications(env)
env.update(self.default_env)
from spack.util.environment import EnvironmentModifications # no cycle
# Apply env argument
if isinstance(env_arg, EnvironmentModifications):
env_arg.apply_modifications(env)
elif env_arg:
env.update(env_arg)
# Apply extra env
extra_env = kwargs.get('extra_env', {})
if isinstance(extra_env, EnvironmentModifications):
extra_env.apply_modifications(env)
else:
env.update(extra_env)
if '_dump_env' in kwargs:
kwargs['_dump_env'].clear()
kwargs['_dump_env'].update(env)
fail_on_error = kwargs.pop('fail_on_error', True)
ignore_errors = kwargs.pop('ignore_errors', ())
ignore_quotes = kwargs.pop('ignore_quotes', False)
# If they just want to ignore one error code, make it a tuple.
if isinstance(ignore_errors, int):
ignore_errors = (ignore_errors, )
input = kwargs.pop('input', None)
output = kwargs.pop('output', None)
error = kwargs.pop('error', None)
if input is str:
raise ValueError('Cannot use `str` as input stream.')
def streamify(arg, mode):
if isinstance(arg, string_types):
return open(arg, mode), True
elif arg in (str, str.split):
return subprocess.PIPE, False
else:
return arg, False
ostream, close_ostream = streamify(output, 'w')
estream, close_estream = streamify(error, 'w')
istream, close_istream = streamify(input, 'r')
if not ignore_quotes:
quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)]
if quoted_args:
tty.warn(
"Quotes in command arguments can confuse scripts like"
" configure.",
"The following arguments may cause problems when executed:",
str("\n".join([" " + arg for arg in quoted_args])),
"Quotes aren't needed because spack doesn't use a shell. "
"Consider removing them.",
"If multiple levels of quotation are required, use "
"`ignore_quotes=True`.")
cmd = self.exe + list(args)
cmd_line = "'%s'" % "' '".join(
map(lambda arg: arg.replace("'", "'\"'\"'"), cmd))
tty.debug(cmd_line)
try:
proc = subprocess.Popen(
cmd,
stdin=istream,
stderr=estream,
stdout=ostream,
env=env)
out, err = proc.communicate()
result = None
if output in (str, str.split) or error in (str, str.split):
result = ''
if output in (str, str.split):
outstr = text_type(out.decode('utf-8'))
result += outstr
if output is str.split:
sys.stdout.write(outstr)
if error in (str, str.split):
errstr = text_type(err.decode('utf-8'))
result += errstr
if error is str.split:
sys.stderr.write(errstr)
rc = self.returncode = proc.returncode
if fail_on_error and rc != 0 and (rc not in ignore_errors):
long_msg = cmd_line
if result:
# If the output is not captured in the result, it will have
# been stored either in the specified files (e.g. if
# 'output' specifies a file) or written to the parent's
# stdout/stderr (e.g. if 'output' is not specified)
long_msg += '\n' + result
raise ProcessError('Command exited with status %d:' %
proc.returncode, long_msg)
return result
except OSError as e:
raise ProcessError(
'%s: %s' % (self.exe[0], e.strerror), 'Command: ' + cmd_line)
except subprocess.CalledProcessError as e:
if fail_on_error:
raise ProcessError(
str(e), '\nExit status %d when invoking command: %s' %
(proc.returncode, cmd_line))
finally:
if close_ostream:
ostream.close()
if close_estream:
estream.close()
if close_istream:
istream.close()
def __eq__(self, other):
return hasattr(other, 'exe') and self.exe == other.exe
def __neq__(self, other):
return not (self == other)
def __hash__(self):
return hash((type(self), ) + tuple(self.exe))
def __repr__(self):
return '<exe: %s>' % self.exe
def __str__(self):
return ' '.join(self.exe)
def clean_environment():
# Stuff in here sanitizes the build environment to eliminate
# anything the user has set that may interfere. We apply it immediately
# unlike the other functions so it doesn't overwrite what the modules load.
env = EnvironmentModifications()
# Remove these vars from the environment during build because they
# can affect how some packages find libraries. We want to make
# sure that builds never pull in unintended external dependencies.
env.unset('LD_LIBRARY_PATH')
env.unset('LD_RUN_PATH')
env.unset('DYLD_LIBRARY_PATH')
env.unset('DYLD_FALLBACK_LIBRARY_PATH')
# These vars affect how the compiler finds libraries and include dirs.
env.unset('LIBRARY_PATH')
env.unset('CPATH')
env.unset('C_INCLUDE_PATH')
env.unset('CPLUS_INCLUDE_PATH')
env.unset('OBJC_INCLUDE_PATH')
# On Cray "cluster" systems, unset CRAY_LD_LIBRARY_PATH to avoid
# interference with Spack dependencies.
# CNL requires these variables to be set (or at least some of them,
# depending on the CNL version).
hostarch = arch.Arch(arch.platform(), 'default_os', 'default_target')
on_cray = str(hostarch.platform) == 'cray'
using_cnl = re.match(r'cnl\d+', str(hostarch.os))
if on_cray and not using_cnl:
env.unset('CRAY_LD_LIBRARY_PATH')
for varname in os.environ.keys():
if 'PKGCONF' in varname:
env.unset(varname)
# Unset the following variables because they can affect installation of
# Autotools and CMake packages.
build_system_vars = [
'CC',
'CFLAGS',
'CPP',
'CPPFLAGS', # C variables
'CXX',
'CCC',
'CXXFLAGS',
'CXXCPP', # C++ variables
'F77',
'FFLAGS',
'FLIBS', # Fortran77 variables
'FC',
'FCFLAGS',
'FCLIBS', # Fortran variables
'LDFLAGS',
'LIBS' # linker variables
]
for v in build_system_vars:
env.unset(v)
# Unset mpi environment vars. These flags should only be set by
# mpi providers for packages with mpi dependencies
mpi_vars = ['MPICC', 'MPICXX', 'MPIFC', 'MPIF77', 'MPIF90']
for v in mpi_vars:
env.unset(v)
build_lang = spack.config.get('config:build_language')
if build_lang:
# Override language-related variables. This can be used to force
# English compiler messages etc., which allows parse_log_events to
# show useful matches.
env.set('LC_ALL', build_lang)
# Remove any macports installs from the PATH. The macports ld can
# cause conflicts with the built-in linker on el capitan. Solves
# assembler issues, e.g.:
# suffix or operands invalid for `movq'"
path = get_path('PATH')
for p in path:
if '/macports/' in p:
env.remove_path('PATH', p)
env.apply_modifications()
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
if not dirty:
clean_environment()
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post',
root=False,
deptype=('build', 'test')):
# If a user makes their own package repo, e.g.
# spack.pkg.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.pkg.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = dspec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
set_module_variables_for_package(pkg, pkg.module)
pkg.setup_environment(spack_env, run_env)
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
# build of packages. To avoid a polluted environment, preserve the
# value of a few, selected, environment variables
# With the current ordering of environment modifications, this is strictly
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the spack_env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
if not dirty:
clean_environment()
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post', root=False,
deptype=('build', 'test')):
# If a user makes their own package repo, e.g.
# spack.pkg.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.pkg.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = dspec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
set_module_variables_for_package(pkg, pkg.module)
pkg.setup_environment(spack_env, run_env)
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
# build of packages. To avoid a polluted environment, preserve the
# value of a few, selected, environment variables
# With the current ordering of environment modifications, this is strictly
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the spack_env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
