def modpath_extensions_for(self, mod_names): """ Determine dictionary with $MODULEPATH extensions for specified modules. Modules with an empty list of $MODULEPATH extensions are included. """ self.log.debug("Determining $MODULEPATH extensions for modules %s" % mod_names) # copy environment so we can restore it env = os.environ.copy() modpath_exts = {} for mod_name in mod_names: modtxt = self.read_module_file(mod_name) useregex = re.compile(r'^\s*module\s+use\s+"?([^"\s]+)"?', re.M) exts = useregex.findall(modtxt) self.log.debug("Found $MODULEPATH extensions for %s: %s" % (mod_name, exts)) modpath_exts.update({mod_name: exts}) if exts: # load this module, since it may extend $MODULEPATH to make other modules available # this is required to obtain the list of $MODULEPATH extensions they make (via 'module show') self.load([mod_name]) # restore environment (modules may have been loaded above) restore_env(env) return modpath_exts
def modpath_extensions_for(self, mod_names): """ Determine dictionary with $MODULEPATH extensions for specified modules. Modules with an empty list of $MODULEPATH extensions are included. """ self.log.debug("Determining $MODULEPATH extensions for modules %s" % mod_names) # copy environment so we can restore it orig_env = os.environ.copy() modpath_exts = {} for mod_name in mod_names: modtxt = self.read_module_file(mod_name) useregex = re.compile(r"^\s*module\s+use\s+(\S+)", re.M) exts = useregex.findall(modtxt) self.log.debug("Found $MODULEPATH extensions for %s: %s" % (mod_name, exts)) modpath_exts.update({mod_name: exts}) if exts: # load this module, since it may extend $MODULEPATH to make other modules available # this is required to obtain the list of $MODULEPATH extensions they make (via 'module show') self.load([mod_name]) # restore original environment (modules may have been loaded above) restore_env(orig_env) return modpath_exts
def load(self, modules, mod_paths=None, purge=False, init_env=None): """ Load all requested modules. @param modules: list of modules to load @param mod_paths: list of module paths to activate before loading @param purge: whether or not a 'module purge' should be run before loading @param init_env: original environment to restore after running 'module purge' """ if mod_paths is None: mod_paths = [] # purge all loaded modules if desired if purge: self.purge() # restore initial environment if provided if init_env is not None: restore_env(init_env) # make sure $MODULEPATH is set correctly after purging self.check_module_path() # extend $MODULEPATH if needed for mod_path in mod_paths: full_mod_path = os.path.join(install_path("mod"), build_option("suffix_modules_path"), mod_path) self.prepend_module_path(full_mod_path) for mod in modules: self.run_module("load", mod)
def load(self, modules, mod_paths=None, purge=False, orig_env=None): """ Load all requested modules. @param modules: list of modules to load @param mod_paths: list of module paths to activate before loading @param purge: whether or not a 'module purge' should be run before loading @param orig_env: original environment to restore after running 'module purge' """ if mod_paths is None: mod_paths = [] # purge all loaded modules if desired if purge: self.purge() # restore original environment if provided if orig_env is not None: restore_env(orig_env) # make sure $MODULEPATH is set correctly after purging self.check_module_path() # extend $MODULEPATH if needed for mod_path in mod_paths: full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path) self.prepend_module_path(full_mod_path) for mod in modules: self.run_module('load', mod)
def load(self, modules, mod_paths=None, purge=False, init_env=None): """ Load all requested modules. @param modules: list of modules to load @param mod_paths: list of module paths to activate before loading @param purge: whether or not a 'module purge' should be run before loading @param init_env: original environment to restore after running 'module purge' """ if mod_paths is None: mod_paths = [] # purge all loaded modules if desired by restoring initial environment # actually running 'module purge' is futile (and wrong/broken on some systems, e.g. Cray) if purge: # restore initial environment if provided if init_env is None: raise EasyBuildError("Initial environment required when purging before loading, but not available") else: restore_env(init_env) # make sure $MODULEPATH is set correctly after purging self.check_module_path() # extend $MODULEPATH if needed for mod_path in mod_paths: full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path) self.prepend_module_path(full_mod_path) for mod in modules: self.run_module('load', mod)
def load(self, modules, mod_paths=None, purge=False, init_env=None): """ Load all requested modules. @param modules: list of modules to load @param mod_paths: list of module paths to activate before loading @param purge: whether or not a 'module purge' should be run before loading @param init_env: original environment to restore after running 'module purge' """ if mod_paths is None: mod_paths = [] # purge all loaded modules if desired by restoring initial environment # actually running 'module purge' is futile (and wrong/broken on some systems, e.g. Cray) if purge: # restore initial environment if provided if init_env is None: raise EasyBuildError( "Initial environment required when purging before loading, but not available" ) else: restore_env(init_env) # make sure $MODULEPATH is set correctly after purging self.check_module_path() # extend $MODULEPATH if needed for mod_path in mod_paths: full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path) self.prepend_module_path(full_mod_path) for mod in modules: self.run_module('load', mod)
def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. :param easyconfigs: list of easyconfigs to generate scripts for """ ecs_and_script_paths = [] for easyconfig in easyconfigs: script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0] ecs_and_script_paths.append((easyconfig['ec'], script_path)) # don't just overwrite existing scripts existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)] if existing_scripts: if build_option('force'): _log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts)) else: raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s", ' '.join(existing_scripts)) orig_env = copy.deepcopy(os.environ) for ec, script_path in ecs_and_script_paths: # obtain EasyBlock instance app_class = get_easyblock_class(ec['easyblock'], name=ec['name']) app = app_class(ec) # mimic dry run, and keep quiet app.dry_run = app.silent = app.toolchain.dry_run = True # prepare build environment (in dry run mode) app.check_readiness_step() app.prepare_step(start_dir=False) # compose script ecfile = os.path.basename(ec.path) script_lines = [ "#!/bin/bash", "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), "# usage: source %s" % os.path.basename(script_path), ] script_lines.extend(['', "# toolchain & dependency modules"]) if app.toolchain.modules: script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) else: script_lines.append("# (no modules loaded)") script_lines.extend(['', "# build environment"]) if app.toolchain.vars: env_vars = sorted(app.toolchain.vars.items()) script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) else: script_lines.append("# (no build environment defined)") write_file(script_path, '\n'.join(script_lines)) print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False) restore_env(orig_env)
def dump_env_easyblock(app, orig_env=None, ec_path=None, script_path=None, silent=False): if orig_env is None: orig_env = copy.deepcopy(os.environ) if ec_path is None: raise EasyBuildError( "The path to the easyconfig relevant to this environment dump is required" ) if script_path is None: # Assume we are placing it alongside the easyconfig path script_path = '%s.env' % os.path.splitext(ec_path)[0] # Compose script ecfile = os.path.basename(ec_path) script_lines = [ "#!/bin/bash", "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), "# usage: source %s" % os.path.basename(script_path), ] script_lines.extend(['', "# toolchain & dependency modules"]) if app.toolchain.modules: script_lines.extend( ["module load %s" % mod for mod in app.toolchain.modules]) else: script_lines.append("# (no modules loaded)") script_lines.extend(['', "# build environment"]) if app.toolchain.vars: env_vars = sorted(app.toolchain.vars.items()) script_lines.extend([ "export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars ]) else: script_lines.append("# (no build environment defined)") write_file(script_path, '\n'.join(script_lines)) msg = "Script to set up build environment for %s dumped to %s" % ( ecfile, script_path) if silent: _log.info(msg) else: print_msg(msg, prefix=False) restore_env(orig_env)
def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, modpath_exts=None): """ Recursively determine path to the top of the module tree, for given module, module subdir and list of $MODULEPATH extensions per dependency module. For example, when to determine the path to the top of the module tree for the HPL/2.1 module being installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS): * starting point: top_paths = ['<prefix>', '<prefix>/Core'] mod_name = 'HPL/2.1' full_mod_subdir = '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5' deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...'] * 1st iteration: find module that extends $MODULEPATH with '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5', => OpenMPI/1.6.5 (in '<prefix>/Compiler/GCC/4.8.2' subdir); recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '<prefix>/Compiler/GCC/4.8.2' * 2nd iteration: find module that extends $MODULEPATH with '<prefix>/Compiler/GCC/4.8.2' => GCC/4.8.2 (in '<prefix>/Core' subdir); recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '<prefix>/Core' * 3rd iteration: try to find module that extends $MODULEPATH with '<prefix>/Core' => '<prefix>/Core' is in top_paths, so stop recursion @param top_paths: list of potentation 'top of module tree' (absolute) paths @param mod_name: (short) module name for starting point (only used in log messages) @param full_mod_subdir: absolute path to module subdirectory for starting point @param deps: list of dependency modules for module at starting point @param modpath_exts: list of module path extensions for each of the dependency modules """ # copy environment so we can restore it env = os.environ.copy() if path_matches(full_mod_subdir, top_paths): self.log.debug("Top of module tree reached with %s (module subdir: %s)" % (mod_name, full_mod_subdir)) return [] self.log.debug("Checking for dependency that extends $MODULEPATH with %s" % full_mod_subdir) if modpath_exts is None: # only retain dependencies that have a non-empty lists of $MODULEPATH extensions modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v]) self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts) mods_to_top = [] full_mod_subdirs = [] for dep in modpath_exts: # if a $MODULEPATH extension is identical to where this module will be installed, we have a hit # use os.path.samefile when comparing paths to avoid issues with resolved symlinks full_modpath_exts = modpath_exts[dep] if path_matches(full_mod_subdir, full_modpath_exts): # full path to module subdir of dependency is simply path to module file without (short) module name dep_full_mod_subdir = self.modulefile_path(dep)[: -len(dep) - 1] full_mod_subdirs.append(dep_full_mod_subdir) mods_to_top.append(dep) self.log.debug( "Found module to top of module tree: %s (subdir: %s, modpath extensions %s)", dep, dep_full_mod_subdir, full_modpath_exts, ) if full_modpath_exts: # load module for this dependency, since it may extend $MODULEPATH to make dependencies available # this is required to obtain the corresponding module file paths (via 'module show') self.load([dep]) # restore original environment (modules may have been loaded above) restore_env(env) path = mods_to_top[:] if mods_to_top: # remove retained dependencies from the list, since we're climbing up the module tree remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top]) self.log.debug( "Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, mods_to_top) ) for mod_name, full_mod_subdir in zip(mods_to_top, full_mod_subdirs): path.extend( self.path_to_top_of_module_tree( top_paths, mod_name, full_mod_subdir, None, modpath_exts=remaining_modpath_exts ) ) else: self.log.debug("Path not extended, we must have reached the top of the module tree") self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path)) return path
def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, modpath_exts=None): """ Recursively determine path to the top of the module tree, for given module, module subdir and list of $MODULEPATH extensions per dependency module. For example, when to determine the path to the top of the module tree for the HPL/2.1 module being installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS): * starting point: top_paths = ['<prefix>', '<prefix>/Core'] mod_name = 'HPL/2.1' full_mod_subdir = '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5' deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...'] * 1st iteration: find module that extends $MODULEPATH with '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5', => OpenMPI/1.6.5 (in '<prefix>/Compiler/GCC/4.8.2' subdir); recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '<prefix>/Compiler/GCC/4.8.2' * 2nd iteration: find module that extends $MODULEPATH with '<prefix>/Compiler/GCC/4.8.2' => GCC/4.8.2 (in '<prefix>/Core' subdir); recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '<prefix>/Core' * 3rd iteration: try to find module that extends $MODULEPATH with '<prefix>/Core' => '<prefix>/Core' is in top_paths, so stop recursion @param top_paths: list of potentation 'top of module tree' (absolute) paths @param mod_name: (short) module name for starting point (only used in log messages) @param full_mod_subdir: absolute path to module subdirectory for starting point @param deps: list of dependency modules for module at starting point @param modpath_exts: list of module path extensions for each of the dependency modules """ # copy environment so we can restore it orig_env = os.environ.copy() if path_matches(full_mod_subdir, top_paths): self.log.debug( "Top of module tree reached with %s (module subdir: %s)" % (mod_name, full_mod_subdir)) return [] self.log.debug( "Checking for dependency that extends $MODULEPATH with %s" % full_mod_subdir) if modpath_exts is None: # only retain dependencies that have a non-empty lists of $MODULEPATH extensions modpath_exts = dict([ (k, v) for k, v in self.modpath_extensions_for(deps).items() if v ]) self.log.debug( "Non-empty lists of module path extensions for dependencies: %s" % modpath_exts) mods_to_top = [] full_mod_subdirs = [] for dep in modpath_exts: # if a $MODULEPATH extension is identical to where this module will be installed, we have a hit # use os.path.samefile when comparing paths to avoid issues with resolved symlinks full_modpath_exts = modpath_exts[dep] if path_matches(full_mod_subdir, full_modpath_exts): # full path to module subdir of dependency is simply path to module file without (short) module name dep_full_mod_subdir = self.modulefile_path(dep)[:-len(dep) - 1] full_mod_subdirs.append(dep_full_mod_subdir) mods_to_top.append(dep) tup = (dep, dep_full_mod_subdir, full_modpath_exts) self.log.debug( "Found module to top of module tree: %s (subdir: %s, modpath extensions %s)" % tup) if full_modpath_exts: # load module for this dependency, since it may extend $MODULEPATH to make dependencies available # this is required to obtain the corresponding module file paths (via 'module show') self.load([dep]) # restore original environment (modules may have been loaded above) restore_env(orig_env) path = mods_to_top[:] if mods_to_top: # remove retained dependencies from the list, since we're climbing up the module tree remaining_modpath_exts = dict( [m for m in modpath_exts.items() if not m[0] in mods_to_top]) self.log.debug( "Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, mods_to_top)) for mod_name, full_mod_subdir in zip(mods_to_top, full_mod_subdirs): path.extend( self.path_to_top_of_module_tree( top_paths, mod_name, full_mod_subdir, None, modpath_exts=remaining_modpath_exts)) else: self.log.debug( "Path not extended, we must have reached the top of the module tree" ) self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path)) return path