def load_external_modules(pkg): """Traverse a package's spec DAG and load any external modules. Traverse a package's dependencies and load any external modules associated with them. Args: pkg (PackageBase): package to load deps for """ for dep in list(pkg.spec.traverse()): if dep.external_module: load_module(dep.external_module)
def set_compiler_environment_variables(pkg, env): assert (pkg.spec.concrete) compiler = pkg.compiler flags = pkg.spec.compiler_flags # Set compiler variables used by CMake and autotools assert all(key in compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) # Populate an object with the list of environment modifications # and return it # TODO : add additional kwargs for better diagnostics, like requestor, # ttyout, ttyerr, etc. link_dir = spack.build_env_path # Set SPACK compiler variables so that our wrapper knows what to call if compiler.cc: env.set('SPACK_CC', compiler.cc) env.set('CC', join_path(link_dir, compiler.link_paths['cc'])) if compiler.cxx: env.set('SPACK_CXX', compiler.cxx) env.set('CXX', join_path(link_dir, compiler.link_paths['cxx'])) if compiler.f77: env.set('SPACK_F77', compiler.f77) env.set('F77', join_path(link_dir, compiler.link_paths['f77'])) if compiler.fc: env.set('SPACK_FC', compiler.fc) env.set('FC', join_path(link_dir, compiler.link_paths['fc'])) # Set SPACK compiler rpath flags so that our wrapper knows what to use env.set('SPACK_CC_RPATH_ARG', compiler.cc_rpath_arg) env.set('SPACK_CXX_RPATH_ARG', compiler.cxx_rpath_arg) env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg) env.set('SPACK_FC_RPATH_ARG', compiler.fc_rpath_arg) # Add every valid compiler flag to the environment, prefixed with "SPACK_" for flag in spack.spec.FlagMap.valid_compiler_flags(): # Concreteness guarantees key safety here if flags[flag] != []: env.set('SPACK_' + flag.upper(), ' '.join(f for f in flags[flag])) env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) for mod in compiler.modules: # Fixes issue https://github.com/LLNL/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) compiler.setup_custom_environment(pkg, env) return env
def setup_package(pkg, dirty): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, spack_env) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False, deptype=('build', 'test')): spkg = dspec.package set_module_variables_for_package(spkg) # Allow dependencies to modify the module dpkg = dspec.package dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) if (not dirty) and (not spack_env.is_unset('CPATH')): tty.debug("A dependency has updated CPATH, this may lead pkg-config" " to assume that the package is part of the system" " includes and omit it when invoked with '--cflags'.") set_module_variables_for_package(pkg) pkg.setup_environment(spack_env, run_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the spack_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) # Make sure nothing's strange about the Spack environment. validate(spack_env, tty.warn) spack_env.apply_modifications()
def setup_package(pkg, dirty): """Execute all environment setup routines.""" build_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, build_env) set_build_environment_variables(pkg, build_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, build_env) build_env.extend( modifications_from_dependencies(pkg.spec, context='build') ) if (not dirty) and (not build_env.is_unset('CPATH')): tty.debug("A dependency has updated CPATH, this may lead pkg-config" " to assume that the package is part of the system" " includes and omit it when invoked with '--cflags'.") set_module_variables_for_package(pkg) pkg.setup_build_environment(build_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the build_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) # kludge to handle cray libsci being automatically loaded by PrgEnv # modules on cray platform. Module unload does no damage when # unnecessary module('unload', 'cray-libsci') if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) implicit_rpaths = pkg.compiler.implicit_rpaths() if implicit_rpaths: build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS', ':'.join(implicit_rpaths)) # Make sure nothing's strange about the Spack environment. validate(build_env, tty.warn) build_env.apply_modifications()
def setup_package(pkg, dirty): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, spack_env) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False, deptype=('build', 'test')): # If a user makes their own package repo, e.g. # spack.pkg.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.pkg.original.libelf.Libelf, # then set the module variables for both classes so the # parent class can still use them if it gets called. spkg = dspec.package modules = parent_class_modules(spkg.__class__) for mod in modules: set_module_variables_for_package(spkg, mod) set_module_variables_for_package(spkg, spkg.module) # Allow dependencies to modify the module dpkg = dspec.package dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) set_module_variables_for_package(pkg, pkg.module) pkg.setup_environment(spack_env, run_env) # Make sure nothing's strange about the Spack environment. validate(spack_env, tty.warn) spack_env.apply_modifications() # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the spack_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg)
def setup_package(pkg, dirty): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, spack_env) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False, deptype=('build', 'test')): # If a user makes their own package repo, e.g. # spack.pkg.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.pkg.original.libelf.Libelf, # then set the module variables for both classes so the # parent class can still use them if it gets called. spkg = dspec.package modules = parent_class_modules(spkg.__class__) for mod in modules: set_module_variables_for_package(spkg, mod) set_module_variables_for_package(spkg, spkg.module) # Allow dependencies to modify the module dpkg = dspec.package dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) set_module_variables_for_package(pkg, pkg.module) pkg.setup_environment(spack_env, run_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the spack_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) # Make sure nothing's strange about the Spack environment. validate(spack_env, tty.warn) spack_env.apply_modifications()
def setup_package(pkg, dirty): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() # Before proceeding, ensure that specs and packages are consistent # # This is a confusing behavior due to how packages are # constructed. `setup_dependent_package` may set attributes on # specs in the DAG for use by other packages' install # method. However, spec.package will look up a package via # spack.repo, which defensively copies specs into packages. This # code ensures that all packages in the DAG have pieces of the # same spec object at build time. # for s in pkg.spec.traverse(): assert s.package.spec is s set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, spack_env) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False, deptype='build'): # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, # then set the module variables for both classes so the # parent class can still use them if it gets called. spkg = dspec.package modules = parent_class_modules(spkg.__class__) for mod in modules: set_module_variables_for_package(spkg, mod) set_module_variables_for_package(spkg, spkg.module) # Allow dependencies to modify the module dpkg = dspec.package dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) set_module_variables_for_package(pkg, pkg.module) pkg.setup_environment(spack_env, run_env) # Make sure nothing's strange about the Spack environment. validate(spack_env, tty.warn) spack_env.apply_modifications() # All module loads that otherwise would belong in previous functions # have to occur after the spack_env object has its modifications applied. # Otherwise the environment modifications could undo module changes, such # as unsetting LD_LIBRARY_PATH after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg)
def set_build_environment_variables(pkg, env, dirty): """Ensure a clean install environment when we build packages. This involves unsetting pesky environment variables that may affect the build. It also involves setting environment variables used by Spack's compiler wrappers. Args: pkg: The package we are building env: The build environment dirty (bool): Skip unsetting the user's environment settings """ # Gather information about various types of dependencies build_deps = pkg.spec.dependencies(deptype=('build', 'test')) link_deps = pkg.spec.traverse(root=False, deptype=('link')) build_link_deps = list(build_deps) + list(link_deps) rpath_deps = get_rpath_deps(pkg) build_prefixes = [dep.prefix for dep in build_deps] link_prefixes = [dep.prefix for dep in link_deps] build_link_prefixes = [dep.prefix for dep in build_link_deps] rpath_prefixes = [dep.prefix for dep in rpath_deps] # add run-time dependencies of direct build-time dependencies: for build_dep in build_deps: for run_dep in build_dep.traverse(deptype='run'): build_prefixes.append(run_dep.prefix) # Filter out system paths: ['/', '/usr', '/usr/local'] # These paths can be introduced into the build when an external package # is added as a dependency. The problem with these paths is that they often # contain hundreds of other packages installed in the same directory. # If these paths come first, they can overshadow Spack installations. build_prefixes = filter_system_paths(build_prefixes) link_prefixes = filter_system_paths(link_prefixes) build_link_prefixes = filter_system_paths(build_link_prefixes) rpath_prefixes = filter_system_paths(rpath_prefixes) # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES env.set_path(SPACK_DEPENDENCIES, build_link_prefixes) # These variables control compiler wrapper behavior env.set_path(SPACK_RPATH_DEPS, rpath_prefixes) env.set_path(SPACK_LINK_DEPS, link_prefixes) # Add dependencies to CMAKE_PREFIX_PATH env.set_path('CMAKE_PREFIX_PATH', build_link_prefixes) # Install prefix env.set(SPACK_PREFIX, pkg.prefix) # Install root prefix env.set(SPACK_INSTALL, spack.store.root) # Stuff in here sanitizes the build environment to eliminate # anything the user has set that may interfere. if not dirty: # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') env.unset('LIBRARY_PATH') env.unset('CPATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') # Remove any macports installs from the PATH. The macports ld can # cause conflicts with the built-in linker on el capitan. Solves # assembler issues, e.g.: # suffix or operands invalid for `movq'" path = get_path('PATH') for p in path: if '/macports/' in p: env.remove_path('PATH', p) # Set environment variables if specified for # the given compiler compiler = pkg.compiler environment = compiler.environment if 'set' in environment: env_to_set = environment['set'] for key, value in iteritems(env_to_set): env.set('SPACK_ENV_SET_%s' % key, value) env.set('%s' % key, value) # Let shell know which variables to set env_variables = ":".join(env_to_set.keys()) env.set('SPACK_ENV_TO_SET', env_variables) if compiler.extra_rpaths: extra_rpaths = ':'.join(compiler.extra_rpaths) env.set('SPACK_COMPILER_EXTRA_RPATHS', extra_rpaths) # Add bin directories from dependencies to the PATH for the build. for prefix in build_prefixes: for dirname in ['bin', 'bin64']: bin_dir = os.path.join(prefix, dirname) if os.path.isdir(bin_dir): env.prepend_path('PATH', bin_dir) # Add spack build environment path with compiler wrappers first in # the path. We add both spack.env_path, which includes default # wrappers (cc, c++, f77, f90), AND a subdirectory containing # compiler-specific symlinks. The latter ensures that builds that # are sensitive to the *name* of the compiler see the right name # when we're building with the wrappers. # # Conflicts on case-insensitive systems (like "CC" and "cc") are # handled by putting one in the <build_env_path>/case-insensitive # directory. Add that to the path too. env_paths = [] compiler_specific = join_path(spack.build_env_path, pkg.compiler.name) for item in [spack.build_env_path, compiler_specific]: env_paths.append(item) ci = join_path(item, 'case-insensitive') if os.path.isdir(ci): env_paths.append(ci) for item in reversed(env_paths): env.prepend_path('PATH', item) env.set_path(SPACK_ENV_PATH, env_paths) # Working directory for the spack command itself, for debug logs. if spack.debug: env.set(SPACK_DEBUG, 'TRUE') env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH for prefix in build_link_prefixes: for directory in ('lib', 'lib64', 'share'): pcdir = join_path(prefix, directory, 'pkgconfig') if os.path.isdir(pcdir): env.prepend_path('PKG_CONFIG_PATH', pcdir) if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) return env