def module(command, *arguments): """ `module` wrapper for the LMOD based modules function. Args: command (str): The `module` command that we're executing. i.e., `load`, `unload`, `swap`, etc. *arguments : Variable length argument list. Returns: int: status indicating success or failure. 0 = success, nonzero for failure For now, because the LMOD `module()` command returns nothing (`NoneType`) and provides no way to determine success or failure, we will only return 0. Raises: Any exception thrown by env_modules_python.module() will get caught and passed along. """ output = 0 try: status = env_modules_python.module(command, *arguments) except BaseException as error: print("") print("An ERROR occurred during execution of module command") print("") raise error return output
def loadModules(self, modules, testDef): # Logging of results from the environment modules usage is the responsibility of the plugin that is making use of this utility. if self.env_module_wrapper is None: # cannot perform this operation return (1, None, "Module capability was not found") # Load the lmod python module() definition sys.path.insert(0, os.path.dirname(self.env_module_link)) try: from env_modules_python import module except: return (1, None, "No module named env_modules_python found") # We have to run this in the same python context for it to take effect and be propagated to future children # Redirect the sys.stdout and sys.stderr for the module loads and unloads. saved_stdout = sys.stdout saved_stderr = sys.stderr load_stdout = sys.stdout = StringIO() load_stderr = sys.stderr = StringIO() modules = modules.split() for mod in modules: mod = mod.strip() try: module("load", mod) except: # If a module name is not found the module python flow will trigger this exception. return (1, None, "Attempt to load environment module " + mod + " failed") # Restore sys.stdout and sys.stderr sys.stdout = saved_stdout sys.stderr = saved_stderr status = load_stderr.seek(0, os.SEEK_END) stdout = load_stdout.getvalue() stderr = load_stderr.getvalue() load_stdout.close() load_stderr.close() return (status, stdout, stderr)
def unloadModules(self, modules, testDef): if self.env_module_wrapper is None: # cannot perform this operation return (1, None, "Module capability was not found") # Load the lmod python module() definition sys.path.insert(0, os.path.dirname(self.env_module_link)) try: from env_modules_python import module except: return (1, None, "No module named env_modules_python found") # We have to run this in the same python context for it to take effect and be propagated to future children # Redirect the sys.stdout and sys.stderr for the module loads and unloads. saved_stdout = sys.stdout saved_stderr = sys.stderr unload_stdout = sys.stdout = StringIO() unload_stderr = sys.stderr = StringIO() modules = modules.split() for mod in modules: mod = mod.strip() try: module("unload", mod) except: # Unlike the load, the module python flow will not cause an exception if the module can not be found. # Add this to catch any other unexpected exceptions. return (1, None, "Attempt to unload environment module " + mod + " failed") # Restore sys.stdout and sys.stderr sys.stdout = saved_stdout sys.stderr = saved_stderr status = unload_stderr.seek(0, os.SEEK_END) stdout = unload_stdout.getvalue() stderr = unload_stderr.getvalue() unload_stdout.close() unload_stderr.close() return (status, stdout, stderr)
def loadModules(self, modules, testDef): # Logging of results from the environment modules usage is the responsibility # of the plugin that is making use of this utility. if self.env_module_wrapper is None: # cannot perform this operation return (1, None, "Module capability was not found") # Load the lmod python module() definition sys.path.insert(0, os.path.dirname(self.env_module_link)) try: from env_modules_python import module except: return (1, None, "No module named env_modules_python found") # We have to run this in the same python context for it to take effect and be propagated to future children # Redirect the sys.stdout and sys.stderr for the module loads and unloads. saved_stdout = sys.stdout saved_stderr = sys.stderr load_stdout = sys.stdout = StringIO() load_stderr = sys.stderr = StringIO() modules = modules.split() for mod in modules: mod = mod.strip() try: module("load", mod) except: # If a module name is not found the module python flow will trigger this exception. return (1, None, "Attempt to load environment module " + mod + " failed") # Restore sys.stdout and sys.stderr sys.stdout = saved_stdout sys.stderr = saved_stderr status = load_stderr.seek(0, os.SEEK_END) stdout = load_stdout.getvalue() stderr = load_stderr.getvalue() load_stdout.close() load_stderr.close() return (status, stdout, stderr)
def setBuildEnviron(arguments): moduleMap = { 'Trilinos_pullrequest_gcc_4.8.4': [ 'sems-env', 'sems-git/2.10.1', 'sems-gcc/4.8.4', 'sems-openmpi/1.10.1', 'sems-python/2.7.9', 'sems-boost/1.63.0/base', 'sems-zlib/1.2.8/base', 'sems-hdf5/1.8.12/parallel', 'sems-netcdf/4.4.1/exo_parallel', 'sems-parmetis/4.0.3/parallel', 'sems-scotch/6.0.3/nopthread_64bit_parallel', 'sems-superlu/4.3/base', 'sems-cmake/3.10.3', 'atdm-env', 'atdm-ninja_fortran/1.7.2' ], 'Trilinos_pullrequest_gcc_4.9.3_SERIAL': [ 'sems-env', 'sems-git/2.10.1', 'sems-gcc/4.9.3', 'sems-python/2.7.9', 'sems-boost/1.63.0/base', 'sems-zlib/1.2.8/base', 'sems-hdf5/1.8.12/base', 'sems-netcdf/4.4.1/exo', 'sems-metis/5.1.0/base', 'sems-superlu/4.3/base', 'sems-cmake/3.10.3', 'atdm-env', 'atdm-ninja_fortran/1.7.2' ], 'Trilinos_pullrequest_python_2': [ 'sems-git/2.10.1', 'sems-gcc/7.2.0', ('', 'sems-python/2.7.9'), 'sems-cmake/3.10.3', 'atdm-env', 'atdm-ninja_fortran/1.7.2' ], 'Trilinos_pullrequest_python_3': [ 'sems-git/2.10.1', 'sems-gcc/7.2.0', ('', 'sems-python/2.7.9'), 'sems-cmake/3.10.3', 'atdm-env', 'atdm-ninja_fortran/1.7.2' ], 'Trilinos_pullrequest_gcc_7.2.0': [ 'sems-env', 'sems-git/2.10.1', 'sems-gcc/7.2.0', 'sems-openmpi/1.10.1', 'sems-python/2.7.9', 'sems-boost/1.63.0/base', 'sems-zlib/1.2.8/base', 'sems-hdf5/1.8.12/parallel', 'sems-netcdf/4.4.1/exo_parallel', 'sems-parmetis/4.0.3/parallel', 'sems-scotch/6.0.3/nopthread_64bit_parallel', 'sems-superlu/4.3/base', 'sems-cmake/3.10.3', 'atdm-env', 'atdm-ninja_fortran/1.7.2' ], 'Trilinos_pullrequest_gcc_8.3.0': [ 'sems-env', 'sems-git/2.10.1', 'sems-gcc/8.3.0', 'sems-openmpi/1.10.1', 'sems-python/2.7.9', 'sems-boost/1.66.0/base', 'sems-zlib/1.2.8/base', 'sems-hdf5/1.8.12/parallel', 'sems-netcdf/4.4.1/exo_parallel', 'sems-parmetis/4.0.3/parallel', 'sems-scotch/6.0.3/nopthread_64bit_parallel', 'sems-superlu/4.3/base', 'sems-cmake/3.10.3', 'atdm-env', 'atdm-ninja_fortran/1.7.2' ], 'Trilinos_pullrequest_intel_17.0.1': [ 'sems-env', 'sems-git/2.10.1', 'sems-gcc/4.9.3', 'sems-intel/17.0.1', 'sems-mpich/3.2', 'sems-python/2.7.9', 'sems-boost/1.63.0/base', 'sems-zlib/1.2.8/base', 'sems-hdf5/1.8.12/parallel', 'sems-netcdf/4.4.1/exo_parallel', 'sems-parmetis/4.0.3/parallel', 'sems-scotch/6.0.3/nopthread_64bit_parallel', 'sems-superlu/4.3/base', 'sems-cmake/3.10.3', 'atdm-env', 'atdm-ninja_fortran/1.7.2' ], 'Trilinos_pullrequest_cuda_9.2': [ 'git/2.10.1', 'devpack/20180521/openmpi/2.1.2/gcc/7.2.0/cuda/9.2.88', ('openblas/0.2.20/gcc/7.2.0', 'netlib/3.8.0/gcc/7.2.0') ] } environMap = { 'Trilinos_pullrequest_gcc_4.8.4': { 'OMP_NUM_THREADS': '2' }, 'Trilinos_pullrequest_gcc_4.9.3_SERIAL': { 'OMP_NUM_THREADS': '2' }, 'Trilinos_pullrequest_python_2': { 'PYTHONPATH': os.path.join(os.path.sep, 'projects', 'sierra', 'linux_rh7', 'install', 'Python', 'extras', 'lib', 'python2.7', 'site-packages'), 'MANPATH': os.path.join(os.path.sep, 'projects', 'sierra', 'linux_rh7', 'install', 'Python', '2.7.15', 'share', 'man'), 'PATH': os.path.join(os.path.sep, 'projects', 'sierra', 'linux_rh7', 'install', 'Python', '2.7.15', 'bin') + os.pathsep + os.path.join(os.path.sep, 'projects', 'sierra', 'linux_rh7', 'install', 'Python', 'extras' 'bin') }, 'Trilinos_pullrequest_python_3': { 'PYTHONPATH': os.path.join(os.path.sep, 'projects', 'sierra', 'linux_rh7', 'install', 'Python', 'extras', 'lib', 'python3.6', 'site-packages'), 'MANPATH': os.path.join(os.path.sep, 'projects', 'sierra', 'linux_rh7', 'install', 'Python', '3.6.3', 'share', 'man'), 'PATH': os.path.join(os.path.sep, 'projects', 'sierra', 'linux_rh7', 'install', 'Python', '3.6.3', 'bin') + os.pathsep + os.path.join(os.path.sep, 'projects', 'sierra', 'linux_rh7', 'install', 'Python', 'extras' 'bin') }, 'Trilinos_pullrequest_gcc_7.2.0': { 'SEMS_FORCE_LOCAL_COMPILER_VERSION': '4.9.3', 'OMP_NUM_THREADS': '2' }, 'Trilinos_pullrequest_gcc_8.3.0': { 'SEMS_FORCE_LOCAL_COMPILER_VERSION': '4.9.3', 'OMP_NUM_THREADS': '2' }, 'Trilinos_pullrequest_intel_17.0.1': { 'SEMS_FORCE_LOCAL_COMPILER_VERSION': '4.9.3', 'OMP_NUM_THREADS': '2' }, 'Trilinos_pullrequest_cuda_9.2': { 'OMPI_CXX': os.path.join(os.environ['WORKSPACE'], 'Trilinos', 'packages', 'kokkos', 'bin', 'nvcc_wrapper'), 'OMPI_CC': os.environ.get('CC', ''), 'OMPI_FC': os.environ.get('FC', ''), 'CUDA_LAUNCH_BLOCKING': '1', 'CUDA_MANAGED_FORCE_DEVICE_ALLOC': '1', 'PATH': os.path.join(os.path.sep, 'ascldap', 'users', 'rabartl', 'install', 'white-ride', 'cmake-3.11.2', 'bin') + os.pathsep + os.path.join(os.path.sep, 'ascldap', 'users', 'rabartl', 'install', 'white-ride', 'ninja-1.8.2', 'bin') } } try: moduleList = moduleMap[arguments.job_base_name] l_environMap = environMap[arguments.job_base_name] except KeyError: sys.exit( """ERROR: Unable to find matching environment for job: Trilinos_pullrequest_UNKOWN Error code was: 42""") if 'sems-env' == moduleList[0]: module('use', '/projects/sems/modulefiles/projects') for mod in moduleList: if isinstance(mod, str): module('load', mod) else: unl_mod, load_mod = mod if '' == unl_mod: module('unload', load_mod) else: module('swap', unl_mod, load_mod) if 'OMPI_CC' in l_environMap: l_environMap['OMPI_CC'] = os.environ.get('CC', '') if 'OMPI_FC' in l_environMap: l_environMap['OMPI_FC'] = os.environ.get('FC', '') for key, value in l_environMap.items(): if key in os.environ: # we are assuming these are paths to be prepended os.environ[key] = value + os.pathsep + os.environ[key] else: os.environ[key] = value confirmGitVersion() print("Environment:\n", file=sys.stdout) print(" pwd = {cwd}".format(cwd=os.getcwd()), file=sys.stdout) print("", file=sys.stdout) for key in os.environ: print(key + ' = ' + os.environ[key], file=sys.stdout) print( "\n==========================================================================================", file=sys.stdout) print(module('list'))
def load_module(mach, compiler, module_name): logger = logging.getLogger(__name__) logger.info("Building with %s on %s" % (compiler, mach)) logger.info("Loading module %s..." % module_name) if mach == 'yellowstone': sys.path.insert( 0, os.path.join(os.sep, 'glade', 'apps', 'opt', 'lmod', 'lmod', 'init')) from env_modules_python import module module('purge') module('load', module_name) module('load', 'ncarcompilers') module('load', 'ncarbinlibs') if mach == 'cheyenne': sys.path.insert( 0, os.path.join(os.sep, 'glade', 'u', 'apps', 'ch', 'opt', 'lmod', '7.2.1', 'lmod', 'lmod', 'init')) from env_modules_python import module module('purge') module('load', module_name) module('load', 'ncarcompilers') module('load', 'mpt/2.15') if mach == 'hobart': sys.path.insert( 0, os.path.join(os.sep, 'usr', 'share', 'Modules', 'init')) from python import module module('purge') module(['load', module_name]) if mach == 'edison': sys.path.insert( 0, os.path.join(os.sep, 'opt', 'modules', 'default', 'init')) from python import module module('purge') module(['load', 'craype/2.5.12']) module(['load', module_name])
def setBuildEnviron(arguments): """ TODO: Needs explanation. - How is environMap structured? - How is moduleMap structured? TODO: wcm: Could it be possible to offload environMap and moduleMap into a JSON or Yaml file? (Yaml allows comments which is more helpful than JSON IMO.) """ moduleMap = {"Trilinos_pullrequest_gcc_4.8.4": ["sems-env", "sems-git/2.10.1", "sems-gcc/4.8.4", "sems-openmpi/1.10.1", "sems-python/2.7.9", "sems-boost/1.63.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/parallel", "sems-netcdf/4.7.3/parallel", "sems-parmetis/4.0.3/parallel", "sems-scotch/6.0.3/nopthread_64bit_parallel", "sems-superlu/4.3/base", "sems-cmake/3.10.3", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_gcc_4.9.3_SERIAL": ["sems-env", "sems-git/2.10.1", "sems-gcc/4.9.3", "sems-python/2.7.9", "sems-boost/1.63.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/base", "sems-netcdf/4.7.3/base", "sems-metis/5.1.0/base", "sems-superlu/4.3/base", "sems-cmake/3.10.3", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_python_2": ["sems-git/2.10.1", "sems-gcc/7.2.0", ("", "sems-python/2.7.9"), "sems-cmake/3.10.3", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_python_3": ["sems-git/2.10.1", "sems-gcc/7.2.0", ("", "sems-python/2.7.9"), "sems-cmake/3.10.3", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_gcc_7.2.0": ["sems-env", "sems-git/2.10.1", "sems-gcc/7.2.0", "sems-openmpi/1.10.1", "sems-python/2.7.9", "sems-boost/1.63.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/parallel", "sems-netcdf/4.7.3/parallel", "sems-parmetis/4.0.3/parallel", "sems-scotch/6.0.3/nopthread_64bit_parallel", "sems-superlu/4.3/base", "sems-cmake/3.10.3", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_gcc_7.2.0_debug": ["sems-env", "sems-git/2.10.1", "sems-gcc/7.2.0", "sems-openmpi/1.10.1", "sems-python/2.7.9", "sems-boost/1.63.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/parallel", "sems-netcdf/4.7.3/parallel", "sems-parmetis/4.0.3/parallel", "sems-scotch/6.0.3/nopthread_64bit_parallel", "sems-superlu/4.3/base", "sems-cmake/3.10.3", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_gcc_8.3.0": ["sems-env", "sems-git/2.10.1", "sems-gcc/8.3.0", "sems-openmpi/1.10.1", "sems-python/2.7.9", "sems-boost/1.66.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/parallel", "sems-netcdf/4.7.3/parallel", "sems-parmetis/4.0.3/parallel", "sems-scotch/6.0.3/nopthread_64bit_parallel", "sems-superlu/4.3/base", "sems-cmake/3.17.1", "sems-ninja_fortran/1.10.0", "atdm-env" ], "Trilinos_pullrequest_intel_17.0.1": ["sems-env", "sems-git/2.10.1", "sems-gcc/4.9.3", "sems-intel/17.0.1", "sems-mpich/3.2", "sems-python/2.7.9", "sems-boost/1.63.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/parallel", "sems-netcdf/4.7.3/parallel", "sems-parmetis/4.0.3/parallel", "sems-scotch/6.0.3/nopthread_64bit_parallel", "sems-superlu/4.3/base", "sems-cmake/3.10.3", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_clang_7.0.1": ["sems-env", "sems-git/2.10.1", "sems-gcc/5.3.0", "sems-clang/7.0.1", "sems-openmpi/1.10.1", "sems-python/2.7.9", "sems-boost/1.63.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/parallel", "sems-netcdf/4.7.3/parallel", "sems-parmetis/4.0.3/parallel", "sems-scotch/6.0.3/nopthread_64bit_parallel", "sems-superlu/4.3/base", "sems-cmake/3.12.2", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_clang_9.0.0": ["sems-env", "sems-git/2.10.1", "sems-gcc/5.3.0", "sems-clang/9.0.0", "sems-openmpi/1.10.1", "sems-python/2.7.9", "sems-boost/1.63.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/parallel", "sems-netcdf/4.7.3/parallel", "sems-parmetis/4.0.3/parallel", "sems-scotch/6.0.3/nopthread_64bit_parallel", "sems-superlu/4.3/base", "sems-cmake/3.12.2", "atdm-env", "atdm-ninja_fortran/1.7.2"], "Trilinos_pullrequest_clang_10.0.0": ["sems-env", "sems-git/2.10.1", "sems-gcc/5.3.0", "sems-clang/10.0.0", "sems-openmpi/1.10.1", "sems-python/2.7.9", "sems-boost/1.69.0/base", "sems-zlib/1.2.8/base", "sems-hdf5/1.10.6/parallel", "sems-netcdf/4.7.3/parallel", "sems-parmetis/4.0.3/parallel", "sems-scotch/6.0.3/nopthread_64bit_parallel", "sems-superlu/4.3/base", "sems-cmake/3.17.1", "sems-ninja_fortran/1.10.0"], "Trilinos_pullrequest_cuda_9.2": ["git/2.10.1", "devpack/20180521/openmpi/2.1.2/gcc/7.2.0/cuda/9.2.88", ("openblas/0.2.20/gcc/7.2.0", "netlib/3.8.0/gcc/7.2.0")]} environMap = { "Trilinos_pullrequest_gcc_4.8.4": {"OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_gcc_4.9.3_SERIAL": {"OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_python_2": {"PYTHONPATH": os.path.join(os.path.sep, "projects", "sierra", "linux_rh7", "install", "Python", "extras", "lib", "python2.7", "site-packages"), "MANPATH": os.path.join(os.path.sep, "projects", "sierra", "linux_rh7", "install", "Python", "2.7.15", "share", "man"), "PATH": os.path.join(os.path.sep, "projects", "sierra", "linux_rh7", "install", "Python", "2.7.15", "bin") + os.pathsep + os.path.join(os.path.sep, "projects", "sierra", "linux_rh7", "install", "Python", "extras" "bin")}, "Trilinos_pullrequest_python_3": {"PYTHONPATH": os.path.join(os.path.sep, "projects", "sierra", "linux_rh7", "install", "Python", "extras", "lib", "python3.6", "site-packages"), "MANPATH": os.path.join(os.path.sep, "projects", "sierra", "linux_rh7", "install", "Python", "3.6.3", "share", "man"), "PATH": os.path.join(os.path.sep, "projects", "sierra", "linux_rh7", "install", "Python", "3.6.3", "bin") + os.pathsep + os.path.join(os.path.sep, "projects", "sierra", "linux_rh7", "install", "Python", "extras" "bin")}, "Trilinos_pullrequest_gcc_7.2.0": {"SEMS_FORCE_LOCAL_COMPILER_VERSION": "4.9.3", "OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_gcc_7.2.0_debug": {"SEMS_FORCE_LOCAL_COMPILER_VERSION": "4.9.3", "OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_gcc_8.3.0": {"SEMS_FORCE_LOCAL_COMPILER_VERSION": "4.9.3", "OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_intel_17.0.1": {"SEMS_FORCE_LOCAL_COMPILER_VERSION": "4.9.3", "OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_clang_7.0.1": {"SEMS_FORCE_LOCAL_COMPILER_VERSION": "5.3.0", "OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_clang_9.0.0": {"SEMS_FORCE_LOCAL_COMPILER_VERSION": "5.3.0", "OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_clang_10.0.0": {"SEMS_FORCE_LOCAL_COMPILER_VERSION": "5.3.0", "OMP_NUM_THREADS": "2"}, "Trilinos_pullrequest_cuda_9.2": {"OMPI_CXX": os.path.join(os.environ["WORKSPACE"], "Trilinos", "packages", "kokkos", "bin", "nvcc_wrapper"), "OMPI_CC": os.environ.get("CC", ""), "OMPI_FC": os.environ.get("FC", ""), "CUDA_LAUNCH_BLOCKING": "1", "CUDA_MANAGED_FORCE_DEVICE_ALLOC": "1", "PATH": os.path.join(os.path.sep, "ascldap", "users", "rabartl", "install", "white-ride", "cmake-3.11.2", "bin") + os.pathsep + os.path.join(os.path.sep, "ascldap", "users", "rabartl", "install", "white-ride", "ninja-1.8.2", "bin")} } try: moduleList = moduleMap[arguments.job_base_name] l_environMap = environMap[arguments.job_base_name] except KeyError: sys.exit(dedent("""\ ERROR: Unable to find matching environment for job: Trilinos_pullrequest_UNKOWN Error code was: 42""")) if 'sems-env' == moduleList[0]: module('use', '/projects/sems/modulefiles/projects') for mod in moduleList: if isinstance(mod, str): module('load', mod) else: unl_mod, load_mod = mod if '' == unl_mod: module('unload', load_mod) else: module('swap', unl_mod, load_mod) if 'OMPI_CC' in l_environMap: l_environMap['OMPI_CC'] = os.environ.get('CC', '') if 'OMPI_FC' in l_environMap: l_environMap['OMPI_FC'] = os.environ.get('FC', '') for key, value in l_environMap.items(): if key in os.environ: # we are assuming these are paths to be prepended os.environ[key] = str(value) + os.pathsep + os.environ[key] else: os.environ[key] = str(value) confirmGitVersion() print ("Environment:\n") print(" pwd = {cwd}".format(cwd=os.getcwd())) print("") for key in os.environ: print(key + ' = ' + os.environ[key], file=sys.stdout) print("\n"+"="*90) print(module('list')) return None
################################################################################ ### Helper functions to generate LD info using PLINK and the 1000G ### data provided with LocusZoom 1.4 import os import sys ################################################################################ ### MAGIC ENVIRONMENT MODULE STUFF ### Automatically load plink environment module when this module is loaded if "LMOD_PACKAGE_PATH" in os.environ: sys.path.insert(0, os.path.join(os.environ["LMOD_PACKAGE_PATH"], "lmod/init")) from env_modules_python import module module('load', 'plink') ################################################################################ # LOCUSZOOM_1000G_DIR = "locuszoom-1.4/locuszoom/data/1000G/genotypes/2014-10-14" # EUR ################################################################################ def invoke_system(command_parameters): cmd = ' '.join(command_parameters) errcode = os.system(cmd) if 0 != errcode: raise Exception("ERROR: failed (returns {errcode}):".format( errcode=errcode) + cmd + '\n')
def source_g5_modules(g5_modules, fout=None): """ #--------------------------------------------------------------------------- # def source_g5_modules(g5_modules, fout): # # source_g5_modules is a wrapper for the csh script g5_modules. It # queries the csh script for basedir, modules and modinit, adds basedir # to os.environ and loads library modules # # Input: # g5_modules: full path of g5_modules # fout: handle of (open) log file, if None - set to sys.stdout #--------------------------------------------------------------------------- """ if not fout: fout = sys.stdout # check if g5_modules exists # -------------------------- if not os.path.isfile(g5_modules): raise Exception('g5_modules does not exist') # part of the command to run # -------------------------- cmd = ['/bin/csh', g5_modules] # query for basedir # ----------------- run = sp.Popen(cmd + ['basedir'], stdout=sp.PIPE, stderr=sp.PIPE) output = run.communicate() rtrnCode = run.wait() if rtrnCode != 0: print('0:') print(output[0]) print('1:') print(output[1]) raise Exception('cant query g5_modules for basedir') #BASEDIR = output[0].strip() BASEDIR = output[0].split('\n')[0].strip() # query for modules to load # ------------------------- run = sp.Popen(cmd + ['modules'], stdout=sp.PIPE, stderr=sp.PIPE) output = run.communicate() rtrnCode = run.wait() if rtrnCode != 0: print('0:') print(output[0]) print('1:') print(output[1]) raise Exception('cant query g5_modules for modules') #MODULES = output[0].strip().split() MODULES = output[0].split('\n')[0].strip().split() #print("MATMAT MODULES: ", MODULES) # query for modinit # ----------------- run = sp.Popen(cmd + ['modinit'], stdout=sp.PIPE, stderr=sp.PIPE) output = run.communicate() rtrnCode = run.wait() if rtrnCode != 0: print('0:') print(output[0]) print('1:') print(output[1]) raise Exception('cant query g5_modules for modinit') # MODINIT = output[0].strip().replace('csh', 'python') # For Matt, modinit query results in '/usr/share/modules/init/csh\n/usr/..' tmpdir = output[0].split('\n')[0].strip() newdir = tmpdir.split('/') HOST = get_hostname() # MAT On anvil, at least, the modules has python.py if HOST == 'PLEIADES' or HOST == 'DESKTOP': newdir[-1] = 'python.py' else: newdir[-1] = 'python' MODINIT = '/'.join(newdir) # set BASEDIR # ----------- ARCH = os.uname()[0] writemsg(' %s: Setting BASEDIR' % os.path.basename(g5_modules), fout) os.environ['BASEDIR'] = BASEDIR # this only modifies the local environment BASELIB = '%s/%s/lib' % (BASEDIR, ARCH) if 'LD_LIBRARY_PATH' in os.environ: os.environ['LD_LIBRARY_PATH'] += os.pathsep + BASELIB else: os.environ['LD_LIBRARY_PATH'] = BASELIB # load library modules # -------------------- if (os.path.isfile(MODINIT)): writemsg(' and modules.\n', fout) exec(open(MODINIT).read()) module('purge') for mod in MODULES: module('load', mod) # At NAS something weird is happening with python # if you force it to load this at the end, things work #if HOST=='PLEIADES': #module('load','python/2.7.15') #module('list') elif os.environ.get('LMOD_PKG') is not None: writemsg(' and modules.\n', fout) sys.path.insert(0, os.path.join(os.environ['LMOD_PKG'], "init")) from env_modules_python import module module('purge') for mod in MODULES: module('load', mod) else: raise Exception('could not load required modules') # set ESMA_FC to gfortran, if needed # ---------------------------------- if BASEDIR.split(os.sep)[-1].split('_')[0] == 'gfortran': writemsg(' Setting ESMA_FC to gfortran\n', fout) os.environ['ESMA_FC'] = 'gfortran' # set ESMA_FC to pgfortran, if needed # ----------------------------------- if BASEDIR.split(os.sep)[-1].split('_')[0] == 'pgfortran': writemsg(' Setting ESMA_FC to pgfortran\n', fout) os.environ['ESMA_FC'] = 'pgfortran' os.environ[ 'PGI_LOCALRC'] = '/discover/swdev/mathomp4/PGILocalRC/linux86-64/17.10/bin/localrc.60300' writemsg(' Setting PGI_LOCALRC to %s\n' % os.environ['PGI_LOCALRC'], fout)