def setup_platform_environment(cls, pkg, env): """ Change the linker to default dynamic to be more similar to linux/standard linker behavior """ # Unload these modules to prevent any silent linking or unnecessary # I/O profiling in the case of darshan. modules_to_unload = ["cray-mpich", "darshan", "cray-libsci", "altd"] for mod in modules_to_unload: module('unload', mod) env.set('CRAYPE_LINK_TYPE', 'dynamic') cray_wrapper_names = os.path.join(build_env_path, 'cray') if os.path.isdir(cray_wrapper_names): env.prepend_path('PATH', cray_wrapper_names) env.prepend_path('SPACK_ENV_PATH', cray_wrapper_names) # Makes spack installed pkg-config work on Crays env.append_path("PKG_CONFIG_PATH", "/usr/lib64/pkgconfig") env.append_path("PKG_CONFIG_PATH", "/usr/local/lib64/pkgconfig") # CRAY_LD_LIBRARY_PATH is used at build time by the cray compiler # wrappers to augment LD_LIBRARY_PATH. This is to avoid long load # times at runtime. This behavior is not always respected on cray # "cluster" systems, so we reproduce it here. if os.environ.get('CRAY_LD_LIBRARY_PATH'): env.prepend_path('LD_LIBRARY_PATH', os.environ['CRAY_LD_LIBRARY_PATH'])
def unload_programming_environment(): """Context manager that unloads Cray Programming Environments.""" env_bu = None # We rely on the fact that the PrgEnv-* modules set the PE_ENV # environment variable. if 'PE_ENV' in os.environ: # Copy environment variables to restore them after the compiler # detection. We expect that the only thing PrgEnv-* modules do is # the environment variables modifications. env_bu = os.environ.copy() # Get the name of the module from the environment variable. prg_env = 'PrgEnv-' + os.environ['PE_ENV'].lower() # Unload the PrgEnv-* module. By doing this we intentionally # provoke errors when the Cray's compiler wrappers are executed # (Error: A PrgEnv-* modulefile must be loaded.) so they will not # be detected as valid compilers by the overridden method. We also # expect that the modules that add the actual compilers' binaries # into the PATH environment variable (i.e. the following modules: # 'intel', 'cce', 'gcc', etc.) will also be unloaded since they are # specified as prerequisites in the PrgEnv-* modulefiles. module('unload', prg_env) yield # Restore the environment. if env_bu is not None: os.environ.clear() os.environ.update(env_bu)
def setup_package(pkg, dirty): """Execute all environment setup routines.""" build_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, build_env) set_build_environment_variables(pkg, build_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, build_env) build_env.extend( modifications_from_dependencies(pkg.spec, context='build') ) if (not dirty) and (not build_env.is_unset('CPATH')): tty.debug("A dependency has updated CPATH, this may lead pkg-config" " to assume that the package is part of the system" " includes and omit it when invoked with '--cflags'.") set_module_variables_for_package(pkg) pkg.setup_build_environment(build_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the build_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) # kludge to handle cray libsci being automatically loaded by PrgEnv # modules on cray platform. Module unload does no damage when # unnecessary module('unload', 'cray-libsci') if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) implicit_rpaths = pkg.compiler.implicit_rpaths() if implicit_rpaths: build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS', ':'.join(implicit_rpaths)) # Make sure nothing's strange about the Spack environment. validate(build_env, tty.warn) build_env.apply_modifications()
def test_module_function_change_env(tmpdir, working_env): src_file = str(tmpdir.join('src_me')) with open(src_file, 'w') as f: f.write('export TEST_MODULE_ENV_VAR=TEST_SUCCESS\n') os.environ['NOT_AFFECTED'] = "NOT_AFFECTED" module('load', src_file, module_template='. {0} 2>&1'.format(src_file)) assert os.environ['TEST_MODULE_ENV_VAR'] == 'TEST_SUCCESS' assert os.environ['NOT_AFFECTED'] == "NOT_AFFECTED"
def test_module_function_change_env(tmpdir, working_env, monkeypatch): monkeypatch.setattr(spack.util.module_cmd, '_cmd_template', _test_template) src_file = str(tmpdir.join('src_me')) with open(src_file, 'w') as f: f.write('export TEST_MODULE_ENV_VAR=TEST_SUCCESS\n') os.environ['NOT_AFFECTED'] = "NOT_AFFECTED" module('load', src_file) assert os.environ['TEST_MODULE_ENV_VAR'] == 'TEST_SUCCESS' assert os.environ['NOT_AFFECTED'] == "NOT_AFFECTED"
def compiler_search_paths(self): """Calls the default function but unloads Cray's programming environments first. This prevents from detecting Cray compiler wrappers and avoids possible false detections. """ import spack.compilers with unload_programming_environment(): search_paths = get_path('PATH') extract_path_re = re.compile(r'prepend-path[\s]*PATH[\s]*([/\w\.:-]*)') for compiler_cls in spack.compilers.all_compiler_types(): # Check if the compiler class is supported on Cray prg_env = getattr(compiler_cls, 'PrgEnv', None) compiler_module = getattr(compiler_cls, 'PrgEnv_compiler', None) if not (prg_env and compiler_module): continue # It is supported, check which versions are available output = module('avail', compiler_cls.PrgEnv_compiler) version_regex = r'({0})/([\d\.]+[\d]-?[\w]*)'.format( compiler_cls.PrgEnv_compiler ) matches = re.findall(version_regex, output) versions = tuple(version for _, version in matches if 'classic' not in version) # Now inspect the modules and add to paths msg = "[CRAY FE] Detected FE compiler [name={0}, versions={1}]" tty.debug(msg.format(compiler_module, versions)) for v in versions: try: current_module = compiler_module + '/' + v out = module('show', current_module) match = extract_path_re.search(out) search_paths += match.group(1).split(':') except Exception as e: msg = ("[CRAY FE] An unexpected error occurred while " "detecting FE compiler [compiler={0}, " " version={1}, error={2}]") tty.debug(msg.format(compiler_cls.name, v, str(e))) search_paths = list(llnl.util.lang.dedupe(search_paths)) return fs.search_paths_for_executables(*search_paths)
def _avail_targets(self): '''Return a list of available CrayPE CPU targets.''' if getattr(self, '_craype_targets', None) is None: output = module('avail', '-t', 'craype-') craype_modules = _get_modules_in_modulecmd_output(output) self._craype_targets = targets = [] _fill_craype_targets_from_modules(targets, craype_modules) return self._craype_targets
def test_module_function_no_change(tmpdir, module_function_test_mode): src_file = str(tmpdir.join('src_me')) with open(src_file, 'w') as f: f.write('echo TEST_MODULE_FUNCTION_PRINT') old_env = os.environ.copy() text = module('show', src_file) assert text == 'TEST_MODULE_FUNCTION_PRINT\n' assert os.environ == old_env
def test_module_function_no_change(tmpdir, monkeypatch): monkeypatch.setattr(spack.util.module_cmd, '_cmd_template', _test_template) src_file = str(tmpdir.join('src_me')) with open(src_file, 'w') as f: f.write('echo TEST_MODULE_FUNCTION_PRINT') old_env = os.environ.copy() text = module('show', src_file) assert text == 'TEST_MODULE_FUNCTION_PRINT\n' assert os.environ == old_env
def setup_platform_environment(cls, pkg, env): """ Change the linker to default dynamic to be more similar to linux/standard linker behavior """ # Unload these modules to prevent any silent linking or unnecessary # I/O profiling in the case of darshan. modules_to_unload = ["cray-mpich", "darshan", "cray-libsci", "altd"] for mod in modules_to_unload: module('unload', mod) env.set('CRAYPE_LINK_TYPE', 'dynamic') cray_wrapper_names = os.path.join(build_env_path, 'cray') if os.path.isdir(cray_wrapper_names): env.prepend_path('PATH', cray_wrapper_names) env.prepend_path('SPACK_ENV_PATH', cray_wrapper_names) # Makes spack installed pkg-config work on Crays env.append_path("PKG_CONFIG_PATH", "/usr/lib64/pkgconfig") env.append_path("PKG_CONFIG_PATH", "/usr/local/lib64/pkgconfig")
def external_prefix(self): mpich_module = module("show", self.modname).splitlines() for line in mpich_module: if "CRAY_MPICH_DIR" in line: return get_path_args_from_module_line(line)[0] # Fixes an issue on Archer2 cray-mpich/8.0.16 where there is # no CRAY_MPICH_DIR variable in the module file. for line in mpich_module: if "CRAY_LD_LIBRARY_PATH" in line: libdir = get_path_args_from_module_line(line)[0] return os.path.dirname(os.path.normpath(libdir))
def _avail_targets(self): '''Return a list of available CrayPE CPU targets.''' def modules_in_output(output): """Returns a list of valid modules parsed from modulecmd output""" return [i for i in re.split(r'\s\s+|\n', output)] def target_names_from_modules(modules): # Craype- module prefixes that are not valid CPU targets. targets = [] for mod in modules: if 'craype-' in mod: name = mod[7:] name = name.split()[0] _n = name.replace('-', '_') # test for mic-knl/mic_knl is_target_name = (name in archspec.cpu.TARGETS or _n in archspec.cpu.TARGETS) is_cray_target_name = name in _craype_name_to_target_name if is_target_name or is_cray_target_name: targets.append(name) return targets def modules_from_listdir(): craype_default_path = '/opt/cray/pe/craype/default/modulefiles' if os.path.isdir(craype_default_path): return os.listdir(craype_default_path) return [] if getattr(self, '_craype_targets', None) is None: strategies = [ lambda: modules_in_output(module('avail', '-t', 'craype-')), modules_from_listdir ] for available_craype_modules in strategies: craype_modules = available_craype_modules() craype_targets = target_names_from_modules(craype_modules) if craype_targets: self._craype_targets = craype_targets break else: # If nothing is found add platform.machine() # to avoid Spack erroring out self._craype_targets = [platform.machine()] return self._craype_targets
def external_prefix(self): libsci_module = module("show", self.modname).splitlines() for line in libsci_module: if "CRAY_LIBSCI_PREFIX_DIR" in line: return get_path_args_from_module_line(line)[0]
def external_prefix(self): mpich_module = module("show", self.modname).splitlines() for line in mpich_module: if "CRAY_MPICH_DIR" in line: return get_path_args_from_module_line(line)[0]