def test_path_matches(self):
"""Test path_matches function."""
# set up temporary directories
path1 = os.path.join(self.test_prefix, 'path1')
ft.mkdir(path1)
path2 = os.path.join(self.test_prefix, 'path2')
ft.mkdir(path1)
symlink = os.path.join(self.test_prefix, 'symlink')
os.symlink(path1, symlink)
missing = os.path.join(self.test_prefix, 'missing')
self.assertFalse(ft.path_matches(missing, [path1, path2]))
self.assertFalse(ft.path_matches(path1, [missing]))
self.assertFalse(ft.path_matches(path1, [missing, path2]))
self.assertFalse(ft.path_matches(path2, [missing, symlink]))
self.assertTrue(ft.path_matches(path1, [missing, symlink]))
def test_path_matches(self):
"""Test path_matches function."""
# set up temporary directories
path1 = os.path.join(self.test_prefix, 'path1')
ft.mkdir(path1)
path2 = os.path.join(self.test_prefix, 'path2')
ft.mkdir(path1)
symlink = os.path.join(self.test_prefix, 'symlink')
os.symlink(path1, symlink)
missing = os.path.join(self.test_prefix, 'missing')
self.assertFalse(ft.path_matches(missing, [path1, path2]))
self.assertFalse(ft.path_matches(path1, [missing]))
self.assertFalse(ft.path_matches(path1, [missing, path2]))
self.assertFalse(ft.path_matches(path2, [missing, symlink]))
self.assertTrue(ft.path_matches(path1, [missing, symlink]))
def test_path_matches(self):
# set up temporary directories
tmpdir = tempfile.mkdtemp()
path1 = os.path.join(tmpdir, 'path1')
ft.mkdir(path1)
path2 = os.path.join(tmpdir, 'path2')
ft.mkdir(path1)
symlink = os.path.join(tmpdir, 'symlink')
os.symlink(path1, symlink)
missing = os.path.join(tmpdir, 'missing')
self.assertFalse(ft.path_matches(missing, [path1, path2]))
self.assertFalse(ft.path_matches(path1, [missing]))
self.assertFalse(ft.path_matches(path1, [missing, path2]))
self.assertFalse(ft.path_matches(path2, [missing, symlink]))
self.assertTrue(ft.path_matches(path1, [missing, symlink]))
# cleanup
shutil.rmtree(tmpdir)
def test_path_matches(self):
# set up temporary directories
tmpdir = tempfile.mkdtemp()
path1 = os.path.join(tmpdir, 'path1')
ft.mkdir(path1)
path2 = os.path.join(tmpdir, 'path2')
ft.mkdir(path1)
symlink = os.path.join(tmpdir, 'symlink')
os.symlink(path1, symlink)
missing = os.path.join(tmpdir, 'missing')
self.assertFalse(ft.path_matches(missing, [path1, path2]))
self.assertFalse(ft.path_matches(path1, [missing]))
self.assertFalse(ft.path_matches(path1, [missing, path2]))
self.assertFalse(ft.path_matches(path2, [missing, symlink]))
self.assertTrue(ft.path_matches(path1, [missing, symlink]))
# cleanup
shutil.rmtree(tmpdir)
def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, modpath_exts=None):
"""
Recursively determine path to the top of the module tree,
for given module, module subdir and list of $MODULEPATH extensions per dependency module.
For example, when to determine the path to the top of the module tree for the HPL/2.1 module being
installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS):
* starting point:
top_paths = ['<prefix>', '<prefix>/Core']
mod_name = 'HPL/2.1'
full_mod_subdir = '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5'
deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...']
* 1st iteration: find module that extends $MODULEPATH with '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5',
=> OpenMPI/1.6.5 (in '<prefix>/Compiler/GCC/4.8.2' subdir);
recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '<prefix>/Compiler/GCC/4.8.2'
* 2nd iteration: find module that extends $MODULEPATH with '<prefix>/Compiler/GCC/4.8.2'
=> GCC/4.8.2 (in '<prefix>/Core' subdir);
recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '<prefix>/Core'
* 3rd iteration: try to find module that extends $MODULEPATH with '<prefix>/Core'
=> '<prefix>/Core' is in top_paths, so stop recursion
@param top_paths: list of potentation 'top of module tree' (absolute) paths
@param mod_name: (short) module name for starting point (only used in log messages)
@param full_mod_subdir: absolute path to module subdirectory for starting point
@param deps: list of dependency modules for module at starting point
@param modpath_exts: list of module path extensions for each of the dependency modules
"""
# copy environment so we can restore it
env = os.environ.copy()
if path_matches(full_mod_subdir, top_paths):
self.log.debug("Top of module tree reached with %s (module subdir: %s)" % (mod_name, full_mod_subdir))
return []
self.log.debug("Checking for dependency that extends $MODULEPATH with %s" % full_mod_subdir)
if modpath_exts is None:
# only retain dependencies that have a non-empty lists of $MODULEPATH extensions
modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v])
self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts)
mods_to_top = []
full_mod_subdirs = []
for dep in modpath_exts:
# if a $MODULEPATH extension is identical to where this module will be installed, we have a hit
# use os.path.samefile when comparing paths to avoid issues with resolved symlinks
full_modpath_exts = modpath_exts[dep]
if path_matches(full_mod_subdir, full_modpath_exts):
# full path to module subdir of dependency is simply path to module file without (short) module name
dep_full_mod_subdir = self.modulefile_path(dep)[: -len(dep) - 1]
full_mod_subdirs.append(dep_full_mod_subdir)
mods_to_top.append(dep)
self.log.debug(
"Found module to top of module tree: %s (subdir: %s, modpath extensions %s)",
dep,
dep_full_mod_subdir,
full_modpath_exts,
)
if full_modpath_exts:
# load module for this dependency, since it may extend $MODULEPATH to make dependencies available
# this is required to obtain the corresponding module file paths (via 'module show')
self.load([dep])
# restore original environment (modules may have been loaded above)
restore_env(env)
path = mods_to_top[:]
if mods_to_top:
# remove retained dependencies from the list, since we're climbing up the module tree
remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top])
self.log.debug(
"Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, mods_to_top)
)
for mod_name, full_mod_subdir in zip(mods_to_top, full_mod_subdirs):
path.extend(
self.path_to_top_of_module_tree(
top_paths, mod_name, full_mod_subdir, None, modpath_exts=remaining_modpath_exts
)
)
else:
self.log.debug("Path not extended, we must have reached the top of the module tree")
self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path))
return path
def path_to_top_of_module_tree(self,
top_paths,
mod_name,
full_mod_subdir,
deps,
modpath_exts=None):
"""
Recursively determine path to the top of the module tree,
for given module, module subdir and list of $MODULEPATH extensions per dependency module.
For example, when to determine the path to the top of the module tree for the HPL/2.1 module being
installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS):
* starting point:
top_paths = ['<prefix>', '<prefix>/Core']
mod_name = 'HPL/2.1'
full_mod_subdir = '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5'
deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...']
* 1st iteration: find module that extends $MODULEPATH with '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5',
=> OpenMPI/1.6.5 (in '<prefix>/Compiler/GCC/4.8.2' subdir);
recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '<prefix>/Compiler/GCC/4.8.2'
* 2nd iteration: find module that extends $MODULEPATH with '<prefix>/Compiler/GCC/4.8.2'
=> GCC/4.8.2 (in '<prefix>/Core' subdir);
recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '<prefix>/Core'
* 3rd iteration: try to find module that extends $MODULEPATH with '<prefix>/Core'
=> '<prefix>/Core' is in top_paths, so stop recursion
@param top_paths: list of potentation 'top of module tree' (absolute) paths
@param mod_name: (short) module name for starting point (only used in log messages)
@param full_mod_subdir: absolute path to module subdirectory for starting point
@param deps: list of dependency modules for module at starting point
@param modpath_exts: list of module path extensions for each of the dependency modules
"""
# copy environment so we can restore it
orig_env = os.environ.copy()
if path_matches(full_mod_subdir, top_paths):
self.log.debug(
"Top of module tree reached with %s (module subdir: %s)" %
(mod_name, full_mod_subdir))
return []
self.log.debug(
"Checking for dependency that extends $MODULEPATH with %s" %
full_mod_subdir)
if modpath_exts is None:
# only retain dependencies that have a non-empty lists of $MODULEPATH extensions
modpath_exts = dict([
(k, v) for k, v in self.modpath_extensions_for(deps).items()
if v
])
self.log.debug(
"Non-empty lists of module path extensions for dependencies: %s"
% modpath_exts)
mods_to_top = []
full_mod_subdirs = []
for dep in modpath_exts:
# if a $MODULEPATH extension is identical to where this module will be installed, we have a hit
# use os.path.samefile when comparing paths to avoid issues with resolved symlinks
full_modpath_exts = modpath_exts[dep]
if path_matches(full_mod_subdir, full_modpath_exts):
# full path to module subdir of dependency is simply path to module file without (short) module name
dep_full_mod_subdir = self.modulefile_path(dep)[:-len(dep) - 1]
full_mod_subdirs.append(dep_full_mod_subdir)
mods_to_top.append(dep)
tup = (dep, dep_full_mod_subdir, full_modpath_exts)
self.log.debug(
"Found module to top of module tree: %s (subdir: %s, modpath extensions %s)"
% tup)
if full_modpath_exts:
# load module for this dependency, since it may extend $MODULEPATH to make dependencies available
# this is required to obtain the corresponding module file paths (via 'module show')
self.load([dep])
# restore original environment (modules may have been loaded above)
restore_env(orig_env)
path = mods_to_top[:]
if mods_to_top:
# remove retained dependencies from the list, since we're climbing up the module tree
remaining_modpath_exts = dict(
[m for m in modpath_exts.items() if not m[0] in mods_to_top])
self.log.debug(
"Path to top from %s extended to %s, so recursing to find way to the top"
% (mod_name, mods_to_top))
for mod_name, full_mod_subdir in zip(mods_to_top,
full_mod_subdirs):
path.extend(
self.path_to_top_of_module_tree(
top_paths,
mod_name,
full_mod_subdir,
None,
modpath_exts=remaining_modpath_exts))
else:
self.log.debug(
"Path not extended, we must have reached the top of the module tree"
)
self.log.debug("Path to top of module tree from %s: %s" %
(mod_name, path))
return path
