def modpath_extensions_for(self, mod_names):
"""
Determine dictionary with $MODULEPATH extensions for specified modules.
Modules with an empty list of $MODULEPATH extensions are included.
"""
self.log.debug("Determining $MODULEPATH extensions for modules %s" % mod_names)
# copy environment so we can restore it
env = os.environ.copy()
modpath_exts = {}
for mod_name in mod_names:
modtxt = self.read_module_file(mod_name)
useregex = re.compile(r'^\s*module\s+use\s+"?([^"\s]+)"?', re.M)
exts = useregex.findall(modtxt)
self.log.debug("Found $MODULEPATH extensions for %s: %s" % (mod_name, exts))
modpath_exts.update({mod_name: exts})
if exts:
# load this module, since it may extend $MODULEPATH to make other modules available
# this is required to obtain the list of $MODULEPATH extensions they make (via 'module show')
self.load([mod_name])
# restore environment (modules may have been loaded above)
restore_env(env)
return modpath_exts
def modpath_extensions_for(self, mod_names):
"""
Determine dictionary with $MODULEPATH extensions for specified modules.
Modules with an empty list of $MODULEPATH extensions are included.
"""
self.log.debug("Determining $MODULEPATH extensions for modules %s" %
mod_names)
# copy environment so we can restore it
orig_env = os.environ.copy()
modpath_exts = {}
for mod_name in mod_names:
modtxt = self.read_module_file(mod_name)
useregex = re.compile(r"^\s*module\s+use\s+(\S+)", re.M)
exts = useregex.findall(modtxt)
self.log.debug("Found $MODULEPATH extensions for %s: %s" %
(mod_name, exts))
modpath_exts.update({mod_name: exts})
if exts:
# load this module, since it may extend $MODULEPATH to make other modules available
# this is required to obtain the list of $MODULEPATH extensions they make (via 'module show')
self.load([mod_name])
# restore original environment (modules may have been loaded above)
restore_env(orig_env)
return modpath_exts
def load(self, modules, mod_paths=None, purge=False, init_env=None):
"""
Load all requested modules.
@param modules: list of modules to load
@param mod_paths: list of module paths to activate before loading
@param purge: whether or not a 'module purge' should be run before loading
@param init_env: original environment to restore after running 'module purge'
"""
if mod_paths is None:
mod_paths = []
# purge all loaded modules if desired
if purge:
self.purge()
# restore initial environment if provided
if init_env is not None:
restore_env(init_env)
# make sure $MODULEPATH is set correctly after purging
self.check_module_path()
# extend $MODULEPATH if needed
for mod_path in mod_paths:
full_mod_path = os.path.join(install_path("mod"), build_option("suffix_modules_path"), mod_path)
self.prepend_module_path(full_mod_path)
for mod in modules:
self.run_module("load", mod)
def load(self, modules, mod_paths=None, purge=False, orig_env=None):
"""
Load all requested modules.
@param modules: list of modules to load
@param mod_paths: list of module paths to activate before loading
@param purge: whether or not a 'module purge' should be run before loading
@param orig_env: original environment to restore after running 'module purge'
"""
if mod_paths is None:
mod_paths = []
# purge all loaded modules if desired
if purge:
self.purge()
# restore original environment if provided
if orig_env is not None:
restore_env(orig_env)
# make sure $MODULEPATH is set correctly after purging
self.check_module_path()
# extend $MODULEPATH if needed
for mod_path in mod_paths:
full_mod_path = os.path.join(install_path('mod'),
build_option('suffix_modules_path'),
mod_path)
self.prepend_module_path(full_mod_path)
for mod in modules:
self.run_module('load', mod)
def load(self, modules, mod_paths=None, purge=False, init_env=None):
"""
Load all requested modules.
@param modules: list of modules to load
@param mod_paths: list of module paths to activate before loading
@param purge: whether or not a 'module purge' should be run before loading
@param init_env: original environment to restore after running 'module purge'
"""
if mod_paths is None:
mod_paths = []
# purge all loaded modules if desired by restoring initial environment
# actually running 'module purge' is futile (and wrong/broken on some systems, e.g. Cray)
if purge:
# restore initial environment if provided
if init_env is None:
raise EasyBuildError("Initial environment required when purging before loading, but not available")
else:
restore_env(init_env)
# make sure $MODULEPATH is set correctly after purging
self.check_module_path()
# extend $MODULEPATH if needed
for mod_path in mod_paths:
full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path)
self.prepend_module_path(full_mod_path)
for mod in modules:
self.run_module('load', mod)
def load(self, modules, mod_paths=None, purge=False, init_env=None):
"""
Load all requested modules.
@param modules: list of modules to load
@param mod_paths: list of module paths to activate before loading
@param purge: whether or not a 'module purge' should be run before loading
@param init_env: original environment to restore after running 'module purge'
"""
if mod_paths is None:
mod_paths = []
# purge all loaded modules if desired by restoring initial environment
# actually running 'module purge' is futile (and wrong/broken on some systems, e.g. Cray)
if purge:
# restore initial environment if provided
if init_env is None:
raise EasyBuildError(
"Initial environment required when purging before loading, but not available"
)
else:
restore_env(init_env)
# make sure $MODULEPATH is set correctly after purging
self.check_module_path()
# extend $MODULEPATH if needed
for mod_path in mod_paths:
full_mod_path = os.path.join(install_path('mod'),
build_option('suffix_modules_path'),
mod_path)
self.prepend_module_path(full_mod_path)
for mod in modules:
self.run_module('load', mod)
def dump_env_script(easyconfigs):
"""
Dump source scripts that set up build environment for specified easyconfigs.
:param easyconfigs: list of easyconfigs to generate scripts for
"""
ecs_and_script_paths = []
for easyconfig in easyconfigs:
script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0]
ecs_and_script_paths.append((easyconfig['ec'], script_path))
# don't just overwrite existing scripts
existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)]
if existing_scripts:
if build_option('force'):
_log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts))
else:
raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s",
' '.join(existing_scripts))
orig_env = copy.deepcopy(os.environ)
for ec, script_path in ecs_and_script_paths:
# obtain EasyBlock instance
app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
app = app_class(ec)
# mimic dry run, and keep quiet
app.dry_run = app.silent = app.toolchain.dry_run = True
# prepare build environment (in dry run mode)
app.check_readiness_step()
app.prepare_step(start_dir=False)
# compose script
ecfile = os.path.basename(ec.path)
script_lines = [
"#!/bin/bash",
"# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile),
"# usage: source %s" % os.path.basename(script_path),
]
script_lines.extend(['', "# toolchain & dependency modules"])
if app.toolchain.modules:
script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules])
else:
script_lines.append("# (no modules loaded)")
script_lines.extend(['', "# build environment"])
if app.toolchain.vars:
env_vars = sorted(app.toolchain.vars.items())
script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars])
else:
script_lines.append("# (no build environment defined)")
write_file(script_path, '\n'.join(script_lines))
print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False)
restore_env(orig_env)
def dump_env_script(easyconfigs):
"""
Dump source scripts that set up build environment for specified easyconfigs.
:param easyconfigs: list of easyconfigs to generate scripts for
"""
ecs_and_script_paths = []
for easyconfig in easyconfigs:
script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0]
ecs_and_script_paths.append((easyconfig['ec'], script_path))
# don't just overwrite existing scripts
existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)]
if existing_scripts:
if build_option('force'):
_log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts))
else:
raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s",
' '.join(existing_scripts))
orig_env = copy.deepcopy(os.environ)
for ec, script_path in ecs_and_script_paths:
# obtain EasyBlock instance
app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
app = app_class(ec)
# mimic dry run, and keep quiet
app.dry_run = app.silent = app.toolchain.dry_run = True
# prepare build environment (in dry run mode)
app.check_readiness_step()
app.prepare_step(start_dir=False)
# compose script
ecfile = os.path.basename(ec.path)
script_lines = [
"#!/bin/bash",
"# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile),
"# usage: source %s" % os.path.basename(script_path),
]
script_lines.extend(['', "# toolchain & dependency modules"])
if app.toolchain.modules:
script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules])
else:
script_lines.append("# (no modules loaded)")
script_lines.extend(['', "# build environment"])
if app.toolchain.vars:
env_vars = sorted(app.toolchain.vars.items())
script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars])
else:
script_lines.append("# (no build environment defined)")
write_file(script_path, '\n'.join(script_lines))
print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False)
restore_env(orig_env)
def dump_env_easyblock(app,
orig_env=None,
ec_path=None,
script_path=None,
silent=False):
if orig_env is None:
orig_env = copy.deepcopy(os.environ)
if ec_path is None:
raise EasyBuildError(
"The path to the easyconfig relevant to this environment dump is required"
)
if script_path is None:
# Assume we are placing it alongside the easyconfig path
script_path = '%s.env' % os.path.splitext(ec_path)[0]
# Compose script
ecfile = os.path.basename(ec_path)
script_lines = [
"#!/bin/bash",
"# script to set up build environment as defined by EasyBuild v%s for %s"
% (EASYBUILD_VERSION, ecfile),
"# usage: source %s" % os.path.basename(script_path),
]
script_lines.extend(['', "# toolchain & dependency modules"])
if app.toolchain.modules:
script_lines.extend(
["module load %s" % mod for mod in app.toolchain.modules])
else:
script_lines.append("# (no modules loaded)")
script_lines.extend(['', "# build environment"])
if app.toolchain.vars:
env_vars = sorted(app.toolchain.vars.items())
script_lines.extend([
"export %s='%s'" % (var, val.replace("'", "\\'"))
for (var, val) in env_vars
])
else:
script_lines.append("# (no build environment defined)")
write_file(script_path, '\n'.join(script_lines))
msg = "Script to set up build environment for %s dumped to %s" % (
ecfile, script_path)
if silent:
_log.info(msg)
else:
print_msg(msg, prefix=False)
restore_env(orig_env)
def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, modpath_exts=None):
"""
Recursively determine path to the top of the module tree,
for given module, module subdir and list of $MODULEPATH extensions per dependency module.
For example, when to determine the path to the top of the module tree for the HPL/2.1 module being
installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS):
* starting point:
top_paths = ['<prefix>', '<prefix>/Core']
mod_name = 'HPL/2.1'
full_mod_subdir = '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5'
deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...']
* 1st iteration: find module that extends $MODULEPATH with '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5',
=> OpenMPI/1.6.5 (in '<prefix>/Compiler/GCC/4.8.2' subdir);
recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '<prefix>/Compiler/GCC/4.8.2'
* 2nd iteration: find module that extends $MODULEPATH with '<prefix>/Compiler/GCC/4.8.2'
=> GCC/4.8.2 (in '<prefix>/Core' subdir);
recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '<prefix>/Core'
* 3rd iteration: try to find module that extends $MODULEPATH with '<prefix>/Core'
=> '<prefix>/Core' is in top_paths, so stop recursion
@param top_paths: list of potentation 'top of module tree' (absolute) paths
@param mod_name: (short) module name for starting point (only used in log messages)
@param full_mod_subdir: absolute path to module subdirectory for starting point
@param deps: list of dependency modules for module at starting point
@param modpath_exts: list of module path extensions for each of the dependency modules
"""
# copy environment so we can restore it
env = os.environ.copy()
if path_matches(full_mod_subdir, top_paths):
self.log.debug("Top of module tree reached with %s (module subdir: %s)" % (mod_name, full_mod_subdir))
return []
self.log.debug("Checking for dependency that extends $MODULEPATH with %s" % full_mod_subdir)
if modpath_exts is None:
# only retain dependencies that have a non-empty lists of $MODULEPATH extensions
modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v])
self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts)
mods_to_top = []
full_mod_subdirs = []
for dep in modpath_exts:
# if a $MODULEPATH extension is identical to where this module will be installed, we have a hit
# use os.path.samefile when comparing paths to avoid issues with resolved symlinks
full_modpath_exts = modpath_exts[dep]
if path_matches(full_mod_subdir, full_modpath_exts):
# full path to module subdir of dependency is simply path to module file without (short) module name
dep_full_mod_subdir = self.modulefile_path(dep)[: -len(dep) - 1]
full_mod_subdirs.append(dep_full_mod_subdir)
mods_to_top.append(dep)
self.log.debug(
"Found module to top of module tree: %s (subdir: %s, modpath extensions %s)",
dep,
dep_full_mod_subdir,
full_modpath_exts,
)
if full_modpath_exts:
# load module for this dependency, since it may extend $MODULEPATH to make dependencies available
# this is required to obtain the corresponding module file paths (via 'module show')
self.load([dep])
# restore original environment (modules may have been loaded above)
restore_env(env)
path = mods_to_top[:]
if mods_to_top:
# remove retained dependencies from the list, since we're climbing up the module tree
remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top])
self.log.debug(
"Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, mods_to_top)
)
for mod_name, full_mod_subdir in zip(mods_to_top, full_mod_subdirs):
path.extend(
self.path_to_top_of_module_tree(
top_paths, mod_name, full_mod_subdir, None, modpath_exts=remaining_modpath_exts
)
)
else:
self.log.debug("Path not extended, we must have reached the top of the module tree")
self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path))
return path
def path_to_top_of_module_tree(self,
top_paths,
mod_name,
full_mod_subdir,
deps,
modpath_exts=None):
"""
Recursively determine path to the top of the module tree,
for given module, module subdir and list of $MODULEPATH extensions per dependency module.
For example, when to determine the path to the top of the module tree for the HPL/2.1 module being
installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS):
* starting point:
top_paths = ['<prefix>', '<prefix>/Core']
mod_name = 'HPL/2.1'
full_mod_subdir = '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5'
deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...']
* 1st iteration: find module that extends $MODULEPATH with '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5',
=> OpenMPI/1.6.5 (in '<prefix>/Compiler/GCC/4.8.2' subdir);
recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '<prefix>/Compiler/GCC/4.8.2'
* 2nd iteration: find module that extends $MODULEPATH with '<prefix>/Compiler/GCC/4.8.2'
=> GCC/4.8.2 (in '<prefix>/Core' subdir);
recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '<prefix>/Core'
* 3rd iteration: try to find module that extends $MODULEPATH with '<prefix>/Core'
=> '<prefix>/Core' is in top_paths, so stop recursion
@param top_paths: list of potentation 'top of module tree' (absolute) paths
@param mod_name: (short) module name for starting point (only used in log messages)
@param full_mod_subdir: absolute path to module subdirectory for starting point
@param deps: list of dependency modules for module at starting point
@param modpath_exts: list of module path extensions for each of the dependency modules
"""
# copy environment so we can restore it
orig_env = os.environ.copy()
if path_matches(full_mod_subdir, top_paths):
self.log.debug(
"Top of module tree reached with %s (module subdir: %s)" %
(mod_name, full_mod_subdir))
return []
self.log.debug(
"Checking for dependency that extends $MODULEPATH with %s" %
full_mod_subdir)
if modpath_exts is None:
# only retain dependencies that have a non-empty lists of $MODULEPATH extensions
modpath_exts = dict([
(k, v) for k, v in self.modpath_extensions_for(deps).items()
if v
])
self.log.debug(
"Non-empty lists of module path extensions for dependencies: %s"
% modpath_exts)
mods_to_top = []
full_mod_subdirs = []
for dep in modpath_exts:
# if a $MODULEPATH extension is identical to where this module will be installed, we have a hit
# use os.path.samefile when comparing paths to avoid issues with resolved symlinks
full_modpath_exts = modpath_exts[dep]
if path_matches(full_mod_subdir, full_modpath_exts):
# full path to module subdir of dependency is simply path to module file without (short) module name
dep_full_mod_subdir = self.modulefile_path(dep)[:-len(dep) - 1]
full_mod_subdirs.append(dep_full_mod_subdir)
mods_to_top.append(dep)
tup = (dep, dep_full_mod_subdir, full_modpath_exts)
self.log.debug(
"Found module to top of module tree: %s (subdir: %s, modpath extensions %s)"
% tup)
if full_modpath_exts:
# load module for this dependency, since it may extend $MODULEPATH to make dependencies available
# this is required to obtain the corresponding module file paths (via 'module show')
self.load([dep])
# restore original environment (modules may have been loaded above)
restore_env(orig_env)
path = mods_to_top[:]
if mods_to_top:
# remove retained dependencies from the list, since we're climbing up the module tree
remaining_modpath_exts = dict(
[m for m in modpath_exts.items() if not m[0] in mods_to_top])
self.log.debug(
"Path to top from %s extended to %s, so recursing to find way to the top"
% (mod_name, mods_to_top))
for mod_name, full_mod_subdir in zip(mods_to_top,
full_mod_subdirs):
path.extend(
self.path_to_top_of_module_tree(
top_paths,
mod_name,
full_mod_subdir,
None,
modpath_exts=remaining_modpath_exts))
else:
self.log.debug(
"Path not extended, we must have reached the top of the module tree"
)
self.log.debug("Path to top of module tree from %s: %s" %
(mod_name, path))
return path