def test_path_matches(self): """Test path_matches function.""" # set up temporary directories path1 = os.path.join(self.test_prefix, 'path1') ft.mkdir(path1) path2 = os.path.join(self.test_prefix, 'path2') ft.mkdir(path1) symlink = os.path.join(self.test_prefix, 'symlink') os.symlink(path1, symlink) missing = os.path.join(self.test_prefix, 'missing') self.assertFalse(ft.path_matches(missing, [path1, path2])) self.assertFalse(ft.path_matches(path1, [missing])) self.assertFalse(ft.path_matches(path1, [missing, path2])) self.assertFalse(ft.path_matches(path2, [missing, symlink])) self.assertTrue(ft.path_matches(path1, [missing, symlink]))
def test_path_matches(self): # set up temporary directories tmpdir = tempfile.mkdtemp() path1 = os.path.join(tmpdir, 'path1') ft.mkdir(path1) path2 = os.path.join(tmpdir, 'path2') ft.mkdir(path1) symlink = os.path.join(tmpdir, 'symlink') os.symlink(path1, symlink) missing = os.path.join(tmpdir, 'missing') self.assertFalse(ft.path_matches(missing, [path1, path2])) self.assertFalse(ft.path_matches(path1, [missing])) self.assertFalse(ft.path_matches(path1, [missing, path2])) self.assertFalse(ft.path_matches(path2, [missing, symlink])) self.assertTrue(ft.path_matches(path1, [missing, symlink])) # cleanup shutil.rmtree(tmpdir)
def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, modpath_exts=None): """ Recursively determine path to the top of the module tree, for given module, module subdir and list of $MODULEPATH extensions per dependency module. For example, when to determine the path to the top of the module tree for the HPL/2.1 module being installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS): * starting point: top_paths = ['<prefix>', '<prefix>/Core'] mod_name = 'HPL/2.1' full_mod_subdir = '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5' deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...'] * 1st iteration: find module that extends $MODULEPATH with '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5', => OpenMPI/1.6.5 (in '<prefix>/Compiler/GCC/4.8.2' subdir); recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '<prefix>/Compiler/GCC/4.8.2' * 2nd iteration: find module that extends $MODULEPATH with '<prefix>/Compiler/GCC/4.8.2' => GCC/4.8.2 (in '<prefix>/Core' subdir); recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '<prefix>/Core' * 3rd iteration: try to find module that extends $MODULEPATH with '<prefix>/Core' => '<prefix>/Core' is in top_paths, so stop recursion @param top_paths: list of potentation 'top of module tree' (absolute) paths @param mod_name: (short) module name for starting point (only used in log messages) @param full_mod_subdir: absolute path to module subdirectory for starting point @param deps: list of dependency modules for module at starting point @param modpath_exts: list of module path extensions for each of the dependency modules """ # copy environment so we can restore it env = os.environ.copy() if path_matches(full_mod_subdir, top_paths): self.log.debug("Top of module tree reached with %s (module subdir: %s)" % (mod_name, full_mod_subdir)) return [] self.log.debug("Checking for dependency that extends $MODULEPATH with %s" % full_mod_subdir) if modpath_exts is None: # only retain dependencies that have a non-empty lists of $MODULEPATH extensions modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v]) self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts) mods_to_top = [] full_mod_subdirs = [] for dep in modpath_exts: # if a $MODULEPATH extension is identical to where this module will be installed, we have a hit # use os.path.samefile when comparing paths to avoid issues with resolved symlinks full_modpath_exts = modpath_exts[dep] if path_matches(full_mod_subdir, full_modpath_exts): # full path to module subdir of dependency is simply path to module file without (short) module name dep_full_mod_subdir = self.modulefile_path(dep)[: -len(dep) - 1] full_mod_subdirs.append(dep_full_mod_subdir) mods_to_top.append(dep) self.log.debug( "Found module to top of module tree: %s (subdir: %s, modpath extensions %s)", dep, dep_full_mod_subdir, full_modpath_exts, ) if full_modpath_exts: # load module for this dependency, since it may extend $MODULEPATH to make dependencies available # this is required to obtain the corresponding module file paths (via 'module show') self.load([dep]) # restore original environment (modules may have been loaded above) restore_env(env) path = mods_to_top[:] if mods_to_top: # remove retained dependencies from the list, since we're climbing up the module tree remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top]) self.log.debug( "Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, mods_to_top) ) for mod_name, full_mod_subdir in zip(mods_to_top, full_mod_subdirs): path.extend( self.path_to_top_of_module_tree( top_paths, mod_name, full_mod_subdir, None, modpath_exts=remaining_modpath_exts ) ) else: self.log.debug("Path not extended, we must have reached the top of the module tree") self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path)) return path
def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, modpath_exts=None): """ Recursively determine path to the top of the module tree, for given module, module subdir and list of $MODULEPATH extensions per dependency module. For example, when to determine the path to the top of the module tree for the HPL/2.1 module being installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS): * starting point: top_paths = ['<prefix>', '<prefix>/Core'] mod_name = 'HPL/2.1' full_mod_subdir = '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5' deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...'] * 1st iteration: find module that extends $MODULEPATH with '<prefix>/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5', => OpenMPI/1.6.5 (in '<prefix>/Compiler/GCC/4.8.2' subdir); recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '<prefix>/Compiler/GCC/4.8.2' * 2nd iteration: find module that extends $MODULEPATH with '<prefix>/Compiler/GCC/4.8.2' => GCC/4.8.2 (in '<prefix>/Core' subdir); recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '<prefix>/Core' * 3rd iteration: try to find module that extends $MODULEPATH with '<prefix>/Core' => '<prefix>/Core' is in top_paths, so stop recursion @param top_paths: list of potentation 'top of module tree' (absolute) paths @param mod_name: (short) module name for starting point (only used in log messages) @param full_mod_subdir: absolute path to module subdirectory for starting point @param deps: list of dependency modules for module at starting point @param modpath_exts: list of module path extensions for each of the dependency modules """ # copy environment so we can restore it orig_env = os.environ.copy() if path_matches(full_mod_subdir, top_paths): self.log.debug( "Top of module tree reached with %s (module subdir: %s)" % (mod_name, full_mod_subdir)) return [] self.log.debug( "Checking for dependency that extends $MODULEPATH with %s" % full_mod_subdir) if modpath_exts is None: # only retain dependencies that have a non-empty lists of $MODULEPATH extensions modpath_exts = dict([ (k, v) for k, v in self.modpath_extensions_for(deps).items() if v ]) self.log.debug( "Non-empty lists of module path extensions for dependencies: %s" % modpath_exts) mods_to_top = [] full_mod_subdirs = [] for dep in modpath_exts: # if a $MODULEPATH extension is identical to where this module will be installed, we have a hit # use os.path.samefile when comparing paths to avoid issues with resolved symlinks full_modpath_exts = modpath_exts[dep] if path_matches(full_mod_subdir, full_modpath_exts): # full path to module subdir of dependency is simply path to module file without (short) module name dep_full_mod_subdir = self.modulefile_path(dep)[:-len(dep) - 1] full_mod_subdirs.append(dep_full_mod_subdir) mods_to_top.append(dep) tup = (dep, dep_full_mod_subdir, full_modpath_exts) self.log.debug( "Found module to top of module tree: %s (subdir: %s, modpath extensions %s)" % tup) if full_modpath_exts: # load module for this dependency, since it may extend $MODULEPATH to make dependencies available # this is required to obtain the corresponding module file paths (via 'module show') self.load([dep]) # restore original environment (modules may have been loaded above) restore_env(orig_env) path = mods_to_top[:] if mods_to_top: # remove retained dependencies from the list, since we're climbing up the module tree remaining_modpath_exts = dict( [m for m in modpath_exts.items() if not m[0] in mods_to_top]) self.log.debug( "Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, mods_to_top)) for mod_name, full_mod_subdir in zip(mods_to_top, full_mod_subdirs): path.extend( self.path_to_top_of_module_tree( top_paths, mod_name, full_mod_subdir, None, modpath_exts=remaining_modpath_exts)) else: self.log.debug( "Path not extended, we must have reached the top of the module tree" ) self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path)) return path