def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
# Before proceeding, ensure that specs and packages are consistent
#
# This is a confusing behavior due to how packages are
# constructed. `setup_dependent_package` may set attributes on
# specs in the DAG for use by other packages' install
# method. However, spec.package will look up a package via
# spack.repo, which defensively copies specs into packages. This
# code ensures that all packages in the DAG have pieces of the
# same spec object at build time.
#
for s in pkg.spec.traverse():
assert s.package.spec is s
# Trap spack-tracked compiler flags as appropriate.
# Must be before set_compiler_environment_variables
# Current implementation of default flag handler relies on this being
# the first thing to affect the spack_env (so there is no appending), or
# on no other build_environment methods trying to affect these variables
# (CFLAGS, CXXFLAGS, etc). Currently both are true, either is sufficient.
for flag in spack.spec.FlagMap.valid_compiler_flags():
trap_func = getattr(
pkg, flag + '_handler',
getattr(pkg, 'default_flag_handler', lambda x, y: y[1]))
flag_val = pkg.spec.compiler_flags[flag]
pkg.spec.compiler_flags[flag] = trap_func(spack_env, (flag, flag_val))
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
load_external_modules(pkg)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post', root=False, deptype='build'):
# If a user makes their own package repo, e.g.
# spack.repos.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.repos.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = dspec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
set_module_variables_for_package(pkg, pkg.module)
pkg.setup_environment(spack_env, run_env)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
def test_extend(self):
env = EnvironmentModifications()
env.set('A', 'dummy value')
env.set('B', 3)
copy_construct = EnvironmentModifications(env)
self.assertEqual(len(copy_construct), 2)
for x, y in zip(env, copy_construct):
assert x is y
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post',
root=False,
deptype=('build', 'test')):
# If a user makes their own package repo, e.g.
# spack.pkg.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.pkg.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = dspec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
set_module_variables_for_package(pkg, pkg.module)
pkg.setup_environment(spack_env, run_env)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
# build of packages. To avoid a polluted environment, preserve the
# value of a few, selected, environment variables
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the spack_env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
def setup_package(pkg, dirty=False):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
# Before proceeding, ensure that specs and packages are consistent
#
# This is a confusing behavior due to how packages are
# constructed. `setup_dependent_package` may set attributes on
# specs in the DAG for use by other packages' install
# method. However, spec.package will look up a package via
# spack.repo, which defensively copies specs into packages. This
# code ensures that all packages in the DAG have pieces of the
# same spec object at build time.
#
# This is safe for the build process, b/c the build process is a
# throwaway environment, but it is kind of dirty.
#
# TODO: Think about how to avoid this fix and do something cleaner.
for s in pkg.spec.traverse():
s.package.spec = s
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
load_external_modules(pkg)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post', root=False, deptype='build'):
# If a user makes their own package repo, e.g.
# spack.repos.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.repos.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = dspec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
set_module_variables_for_package(pkg, pkg.module)
pkg.setup_environment(spack_env, run_env)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
def test_set(self):
env = EnvironmentModifications()
env.set('A', 'dummy value')
env.set('B', 3)
env.apply_modifications()
self.assertEqual('dummy value', os.environ['A'])
self.assertEqual(str(3), os.environ['B'])
def test_extra_arguments(self):
env = EnvironmentModifications()
env.set('A', 'dummy value', who='Pkg1')
for x in env:
assert 'who' in x.args
env.apply_modifications()
self.assertEqual('dummy value', os.environ['A'])
def test_path_manipulation(self):
env = EnvironmentModifications()
env.append_path('PATH_LIST', '/path/last')
env.prepend_path('PATH_LIST', '/path/first')
env.append_path('EMPTY_PATH_LIST', '/path/middle')
env.append_path('EMPTY_PATH_LIST', '/path/last')
env.prepend_path('EMPTY_PATH_LIST', '/path/first')
env.append_path('NEWLY_CREATED_PATH_LIST', '/path/middle')
env.append_path('NEWLY_CREATED_PATH_LIST', '/path/last')
env.prepend_path('NEWLY_CREATED_PATH_LIST', '/path/first')
env.remove_path('REMOVE_PATH_LIST', '/remove/this')
env.remove_path('REMOVE_PATH_LIST', '/duplicate/')
env.apply_modifications()
self.assertEqual(
'/path/first:/path/second:/path/third:/path/last',
os.environ['PATH_LIST']
)
self.assertEqual(
'/path/first:/path/middle:/path/last',
os.environ['EMPTY_PATH_LIST']
)
self.assertEqual(
'/path/first:/path/middle:/path/last',
os.environ['NEWLY_CREATED_PATH_LIST']
)
self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST'])
def clean_environment():
# Stuff in here sanitizes the build environment to eliminate
# anything the user has set that may interfere. We apply it immediately
# unlike the other functions so it doesn't overwrite what the modules load.
env = EnvironmentModifications()
# Remove these vars from the environment during build because they
# can affect how some packages find libraries. We want to make
# sure that builds never pull in unintended external dependencies.
env.unset('LD_LIBRARY_PATH')
env.unset('LIBRARY_PATH')
env.unset('CPATH')
env.unset('LD_RUN_PATH')
env.unset('DYLD_LIBRARY_PATH')
build_lang = spack.config.get('config:build_language')
if build_lang:
# Override language-related variables. This can be used to force
# English compiler messages etc., which allows parse_log_events to
# show useful matches.
env.set('LC_ALL', build_lang)
# Remove any macports installs from the PATH. The macports ld can
# cause conflicts with the built-in linker on el capitan. Solves
# assembler issues, e.g.:
# suffix or operands invalid for `movq'"
path = get_path('PATH')
for p in path:
if '/macports/' in p:
env.remove_path('PATH', p)
env.apply_modifications()
def clean_environment():
# Stuff in here sanitizes the build environment to eliminate
# anything the user has set that may interfere. We apply it immediately
# unlike the other functions so it doesn't overwrite what the modules load.
env = EnvironmentModifications()
# Remove these vars from the environment during build because they
# can affect how some packages find libraries. We want to make
# sure that builds never pull in unintended external dependencies.
env.unset('LD_LIBRARY_PATH')
env.unset('LIBRARY_PATH')
env.unset('CPATH')
env.unset('LD_RUN_PATH')
env.unset('DYLD_LIBRARY_PATH')
# Remove any macports installs from the PATH. The macports ld can
# cause conflicts with the built-in linker on el capitan. Solves
# assembler issues, e.g.:
# suffix or operands invalid for `movq'"
path = get_path('PATH')
for p in path:
if '/macports/' in p:
env.remove_path('PATH', p)
env.apply_modifications()
def test_extend(env):
"""Tests that we can construct a list of environment modifications
starting from another list.
"""
env.set('A', 'dummy value')
env.set('B', 3)
copy_construct = EnvironmentModifications(env)
assert len(copy_construct) == 2
for x, y in zip(env, copy_construct):
assert x is y
def test_source_files(files_to_be_sourced):
"""Tests the construction of a list of environment modifications that are
the result of sourcing a file.
"""
env = EnvironmentModifications()
for filename in files_to_be_sourced:
if filename.endswith('sourceme_parameters.sh'):
env.extend(
EnvironmentModifications.from_sourcing_file(
filename, 'intel64'))
else:
env.extend(EnvironmentModifications.from_sourcing_file(filename))
modifications = env.group_by_name()
# This is sensitive to the user's environment; can include
# spurious entries for things like PS1
#
# TODO: figure out how to make a bit more robust.
assert len(modifications) >= 5
# Set new variables
assert len(modifications['NEW_VAR']) == 1
assert isinstance(modifications['NEW_VAR'][0], SetEnv)
assert modifications['NEW_VAR'][0].value == 'new'
assert len(modifications['FOO']) == 1
assert isinstance(modifications['FOO'][0], SetEnv)
assert modifications['FOO'][0].value == 'intel64'
# Unset variables
assert len(modifications['EMPTY_PATH_LIST']) == 1
assert isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv)
# Modified variables
assert len(modifications['UNSET_ME']) == 1
assert isinstance(modifications['UNSET_ME'][0], SetEnv)
assert modifications['UNSET_ME'][0].value == 'overridden'
assert len(modifications['PATH_LIST']) == 3
assert isinstance(modifications['PATH_LIST'][0], RemovePath)
assert modifications['PATH_LIST'][0].value == '/path/third'
assert isinstance(modifications['PATH_LIST'][1], AppendPath)
assert modifications['PATH_LIST'][1].value == '/path/fourth'
assert isinstance(modifications['PATH_LIST'][2], PrependPath)
assert modifications['PATH_LIST'][2].value == '/path/first'
def env(prepare_environment_for_tests):
"""Returns an empty EnvironmentModifications object."""
return EnvironmentModifications()
def test_set_path(self):
env = EnvironmentModifications()
env.set_path('A', ['foo', 'bar', 'baz'])
env.apply_modifications()
self.assertEqual('foo:bar:baz', os.environ['A'])
def test_unset(self):
env = EnvironmentModifications()
self.assertEqual('foo', os.environ['UNSET_ME'])
env.unset('UNSET_ME')
env.apply_modifications()
self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME')
def setup_package(pkg, dirty):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
# Before proceeding, ensure that specs and packages are consistent
#
# This is a confusing behavior due to how packages are
# constructed. `setup_dependent_package` may set attributes on
# specs in the DAG for use by other packages' install
# method. However, spec.package will look up a package via
# spack.repo, which defensively copies specs into packages. This
# code ensures that all packages in the DAG have pieces of the
# same spec object at build time.
#
for s in pkg.spec.traverse():
assert s.package.spec is s
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, spack_env)
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dspec in pkg.spec.traverse(order='post', root=False, deptype='build'):
# If a user makes their own package repo, e.g.
# spack.repos.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.repos.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = dspec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
set_module_variables_for_package(pkg, pkg.module)
pkg.setup_environment(spack_env, run_env)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
# All module loads that otherwise would belong in previous functions
# have to occur after the spack_env object has its modifications applied.
# Otherwise the environment modifications could undo module changes, such
# as unsetting LD_LIBRARY_PATH after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
if pkg.architecture.target.module_name:
load_module(pkg.architecture.target.module_name)
load_external_modules(pkg)
