示例#1
0
    def test_toy_tweaked(self):
        """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters."""
        test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "easyconfigs")
        ec_file = os.path.join(self.test_buildpath, "toy-0.0-tweaked.eb")
        shutil.copy2(os.path.join(test_ecs_dir, "toy-0.0.eb"), ec_file)

        # tweak easyconfig by appending to it
        ec_extra = "\n".join(
            [
                "versionsuffix = '-tweaked'",
                "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}",
                "modextravars = {'FOO': 'bar'}",
                "modloadmsg =  'THANKS FOR LOADING ME, I AM %(name)s v%(version)s'",
                "modtclfooter = 'puts stderr \"oh hai!\"'",  # ignored when module syntax is Lua
                "modluafooter = 'io.stderr:write(\"oh hai!\")'",  # ignored when module syntax is Tcl
            ]
        )
        write_file(ec_file, ec_extra, append=True)

        args = [
            ec_file,
            "--sourcepath=%s" % self.test_sourcepath,
            "--buildpath=%s" % self.test_buildpath,
            "--installpath=%s" % self.test_installpath,
            "--debug",
            "--force",
        ]
        outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
        self.check_toy(self.test_installpath, outtxt, versionsuffix="-tweaked")
        toy_module = os.path.join(self.test_installpath, "modules", "all", "toy", "0.0-tweaked")
        if get_module_syntax() == "Lua":
            toy_module += ".lua"
        toy_module_txt = read_file(toy_module)

        if get_module_syntax() == "Tcl":
            self.assertTrue(re.search(r'^setenv\s*FOO\s*"bar"$', toy_module_txt, re.M))
            self.assertTrue(re.search(r"^prepend-path\s*SOMEPATH\s*\$root/foo/bar$", toy_module_txt, re.M))
            self.assertTrue(re.search(r"^prepend-path\s*SOMEPATH\s*\$root/baz$", toy_module_txt, re.M))
            self.assertTrue(re.search(r"^prepend-path\s*SOMEPATH\s*\$root$", toy_module_txt, re.M))
            self.assertTrue(
                re.search(r'module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0"$', toy_module_txt, re.M)
            )
            self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M))
        elif get_module_syntax() == "Lua":
            self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M))
            self.assertTrue(
                re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$', toy_module_txt, re.M)
            )
            self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M))
            self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M))
            self.assertTrue(
                re.search(
                    r'^if mode\(\) == "load" then\n\s*io.stderr:write\(".*I AM toy v0.0"\)$', toy_module_txt, re.M
                )
            )
            self.assertTrue(re.search(r'^io.stderr:write\("oh hai!"\)$', toy_module_txt, re.M))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
示例#2
0
    def test_exclude_path_to_top_of_module_tree(self):
        """
        Make sure that modules under the HierarchicalMNS are correct,
        w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
        """
        self.orig_module_naming_scheme = config.get_module_naming_scheme()
        test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
        all_stops = [x[0] for x in EasyBlock.get_steps()]
        build_options = {
            'check_osdeps': False,
            'robot_path': [test_ecs_path],
            'valid_stops': all_stops,
            'validate': False,
        }
        os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
        init_config(build_options=build_options)
        self.setup_hierarchical_modules()

        modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all')
        mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True)
        mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True)

        impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049')
        imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144')
        if get_module_syntax() == 'Lua':
            impi_modfile_path += '.lua'
            imkl_modfile_path += '.lua'

        # example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
        # not for the toolchain or any of the toolchain components,
        # since both icc/ifort and impi form the path to the top of the module tree
        tests = [
            ('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']),
            ('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']),
        ]
        for ec_file, modfile_path, excluded_deps in tests:
            ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
            eb = EasyBlock(ec)
            eb.toolchain.prepare()
            modpath = eb.make_module_step()
            modfile_path = os.path.join(modpath, modfile_path)
            modtxt = read_file(modfile_path)

            for dep in excluded_deps:
                tup = (dep, modfile_path, modtxt)
                failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
                if get_module_syntax() == 'Tcl':
                    self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg)
                elif get_module_syntax() == 'Lua':
                    self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg)
                else:
                    self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
        init_config(build_options=build_options)
    def test_make_module_extend_modpath(self):
        """Test for make_module_extend_modpath"""
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            'toolchain = {"name":"dummy", "version": "dummy"}',
            'moduleclass = "compiler"',
        ])
        self.writeEC()
        eb = EasyBlock(EasyConfig(self.eb_file))
        eb.installdir = config.install_path()

        # no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
        self.assertEqual(eb.make_module_extend_modpath(), '')

        usermodsdir = 'my/own/modules'
        modclasses = ['compiler', 'tools']
        os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
        build_options = {
            'subdir_user_modules': usermodsdir,
            'valid_module_classes': modclasses,
        }
        init_config(build_options=build_options)
        eb = EasyBlock(EasyConfig(self.eb_file))
        eb.installdir = config.install_path()

        txt = eb.make_module_extend_modpath()
        if get_module_syntax() == 'Tcl':
            regexs = [r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c for c in modclasses]
            home = r'\$env\(HOME\)'
            regexs.extend([
                # extension for user modules is guarded
                r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir),
                # no per-moduleclass extension for user modules
                r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' % (home, usermodsdir),
            ])
        elif get_module_syntax() == 'Lua':
            regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$' % c for c in modclasses]
            home = r'os.getenv\("HOME"\)'
            regexs.extend([
                # extension for user modules is guarded
                r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' % (home, usermodsdir),
                # no per-moduleclass extension for user modules
                r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)' % (home, usermodsdir),
            ])
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
        for regex in regexs:
            regex = re.compile(regex, re.M)
            self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
    def test_make_module_extra(self):
        """Test for make_module_extra."""
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            "toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
            'dependencies = [',
            "   ('FFTW', '3.3.1'),",
            "   ('LAPACK', '3.4.0'),",
            ']',
        ])
        self.writeEC()
        eb = EasyBlock(EasyConfig(self.eb_file))
        eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')

        if get_module_syntax() == 'Tcl':
            expected_default = re.compile(r'\n'.join([
                r'setenv\s+EBROOTPI\s+\"\$root"',
                r'setenv\s+EBVERSIONPI\s+"3.14"',
                r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
            ]))
            expected_alt = re.compile(r'\n'.join([
                r'setenv\s+EBROOTPI\s+"/opt/software/tau/6.28"',
                r'setenv\s+EBVERSIONPI\s+"6.28"',
                r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
            ]))
        elif get_module_syntax() == 'Lua':
            expected_default = re.compile(r'\n'.join([
                r'setenv\("EBROOTPI", root\)',
                r'setenv\("EBVERSIONPI", "3.14"\)',
                r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
            ]))
            expected_alt = re.compile(r'\n'.join([
                r'setenv\("EBROOTPI", "/opt/software/tau/6.28"\)',
                r'setenv\("EBVERSIONPI", "6.28"\)',
                r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
            ]))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        defaulttxt = eb.make_module_extra().strip()
        self.assertTrue(expected_default.match(defaulttxt),
                        "Pattern %s found in %s" % (expected_default.pattern, defaulttxt))

        alttxt = eb.make_module_extra(altroot='/opt/software/tau/6.28', altversion='6.28').strip()
        self.assertTrue(expected_alt.match(alttxt),
                        "Pattern %s found in %s" % (expected_alt.pattern, alttxt))
示例#5
0
    def test_module_filepath_tweaking(self):
        """Test using --suffix-modules-path."""
        # install test module naming scheme dynamically
        test_mns_parent_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox")
        sys.path.append(test_mns_parent_dir)
        reload(easybuild)
        reload(easybuild.tools)
        reload(easybuild.tools.module_naming_scheme)
        mns_path = "easybuild.tools.module_naming_scheme.test_module_naming_scheme"
        __import__(mns_path, globals(), locals(), [""])

        eb_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "toy-0.0.eb")
        args = [
            eb_file,
            "--sourcepath=%s" % self.test_sourcepath,
            "--buildpath=%s" % self.test_buildpath,
            "--installpath=%s" % self.test_installpath,
            "--force",
            "--debug",
            "--suffix-modules-path=foobarbaz",
            "--module-naming-scheme=TestModuleNamingScheme",
        ]
        self.eb_main(args, do_build=True, verbose=True)
        mod_file_prefix = os.path.join(self.test_installpath, "modules")
        mod_file_suffix = ""
        if get_module_syntax() == "Lua":
            mod_file_suffix += ".lua"

        self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, "foobarbaz", "toy", "0.0" + mod_file_suffix)))
        self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, "TOOLS", "toy", "0.0" + mod_file_suffix)))
        self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, "TOOLS", "toy", "0.0" + mod_file_suffix)))
        self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, "t", "toy", "0.0" + mod_file_suffix)))
        self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, "t", "toy", "0.0" + mod_file_suffix)))
    def test_make_module_req(self):
        """Testcase for make_module_req"""
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            'toolchain = {"name":"dummy", "version": "dummy"}',
        ])
        self.writeEC()
        eb = EasyBlock(EasyConfig(self.eb_file))
        eb.installdir = config.install_path()

        # create fake directories and files that should be guessed
        os.makedirs(eb.installdir)
        open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
        open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
        os.mkdir(os.path.join(eb.installdir, 'bin'))
        os.mkdir(os.path.join(eb.installdir, 'share'))
        os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
        # this is not a path that should be picked up
        os.mkdir(os.path.join(eb.installdir, 'CPATH'))

        guess = eb.make_module_req()

        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
            self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
        elif get_module_syntax() == 'Lua':
            self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M))
            self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # cleanup
        eb.close_log()
        os.remove(eb.logfile)
示例#7
0
 def test_toy_hidden(self):
     """Test installing a hidden module."""
     ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "easyconfigs", "toy-0.0.eb")
     self.test_toy_build(ec_file=ec_file, extra_args=["--hidden"], verify=False)
     # module file is hidden
     toy_module = os.path.join(self.test_installpath, "modules", "all", "toy", ".0.0")
     if get_module_syntax() == "Lua":
         toy_module += ".lua"
     self.assertTrue(os.path.exists(toy_module), "Found hidden module %s" % toy_module)
     # installed software is not hidden
     toybin = os.path.join(self.test_installpath, "software", "toy", "0.0", "bin", "toy")
     self.assertTrue(os.path.exists(toybin))
def module_generator(app, fake=False):
    """
    Return ModuleGenerator instance that matches the selected module file syntax to be used
    """
    module_syntax = get_module_syntax()
    available_mod_gens = avail_module_generators()

    if module_syntax not in available_mod_gens:
        raise EasyBuildError("No module generator available for specified syntax '%s' (available: %s)",
                             module_syntax, available_mod_gens)

    module_generator_class = available_mod_gens[module_syntax]
    return module_generator_class(app, fake=fake)
示例#9
0
    def check_toy(self, installpath, outtxt, version="0.0", versionprefix="", versionsuffix=""):
        """Check whether toy build succeeded."""

        full_version = "".join([versionprefix, version, versionsuffix])

        # check for success
        success = re.compile("COMPLETED: Installation ended successfully")
        self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s" % outtxt)

        # if the module exists, it should be fine
        toy_module = os.path.join(installpath, "modules", "all", "toy", full_version)
        msg = "module for toy build toy/%s found (path %s)" % (full_version, toy_module)
        if get_module_syntax() == "Lua":
            toy_module += ".lua"
        self.assertTrue(os.path.exists(toy_module), msg)

        # module file is symlinked according to moduleclass
        toy_module_symlink = os.path.join(installpath, "modules", "tools", "toy", full_version)
        if get_module_syntax() == "Lua":
            toy_module_symlink += ".lua"
        self.assertTrue(os.path.islink(toy_module_symlink))
        self.assertTrue(os.path.exists(toy_module_symlink))

        # make sure installation log file and easyconfig file are copied to install dir
        software_path = os.path.join(installpath, "software", "toy", full_version)
        install_log_path_pattern = os.path.join(software_path, "easybuild", "easybuild-toy-%s*.log" % version)
        self.assertTrue(len(glob.glob(install_log_path_pattern)) == 1, "Found 1 file at %s" % install_log_path_pattern)

        # make sure test report is available
        test_report_path_pattern = os.path.join(software_path, "easybuild", "easybuild-toy-%s*test_report.md" % version)
        self.assertTrue(len(glob.glob(test_report_path_pattern)) == 1, "Found 1 file at %s" % test_report_path_pattern)

        ec_file_path = os.path.join(software_path, "easybuild", "toy-%s.eb" % full_version)
        self.assertTrue(os.path.exists(ec_file_path))

        devel_module_path = os.path.join(software_path, "easybuild", "toy-%s-easybuild-devel" % full_version)
        self.assertTrue(os.path.exists(devel_module_path))
    def test_make_module_req(self):
        """Testcase for make_module_req"""
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            'toolchain = {"name":"dummy", "version": "dummy"}',
        ])
        self.writeEC()
        eb = EasyBlock(EasyConfig(self.eb_file))
        eb.installdir = config.install_path()

        # create fake directories and files that should be guessed
        os.makedirs(eb.installdir)
        open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
        open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
        os.mkdir(os.path.join(eb.installdir, 'bin'))
        os.mkdir(os.path.join(eb.installdir, 'share'))
        os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
        # this is not a path that should be picked up
        os.mkdir(os.path.join(eb.installdir, 'CPATH'))

        guess = eb.make_module_req()

        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
            self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
        elif get_module_syntax() == 'Lua':
            self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M))
            self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # check for behavior when a string value is used as dict value by make_module_req_guesses
        eb.make_module_req_guess = lambda: {'PATH': 'bin'}
        txt = eb.make_module_req()
        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M))
        elif get_module_syntax() == 'Lua':
            self.assertTrue(re.match(r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$', txt, re.M))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # check for correct behaviour if empty string is specified as one of the values
        # prepend-path statements should be included for both the 'bin' subdir and the install root
        eb.make_module_req_guess = lambda: {'PATH': ['bin', '']}
        txt = eb.make_module_req()
        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root/bin\n", txt, re.M))
            self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root\n", txt, re.M))
        elif get_module_syntax() == 'Lua':
            self.assertTrue(re.search(r'\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n', txt, re.M))
            self.assertTrue(re.search(r'\nprepend_path\("PATH", root\)\n', txt, re.M))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # cleanup
        eb.close_log()
        os.remove(eb.logfile)
示例#11
0
    def test_exclude_path_to_top_of_module_tree(self):
        """
        Make sure that modules under the HierarchicalMNS are correct,
        w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
        """
        self.orig_module_naming_scheme = config.get_module_naming_scheme()
        test_ecs_path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
            'test_ecs')
        all_stops = [x[0] for x in EasyBlock.get_steps()]
        build_options = {
            'check_osdeps': False,
            'robot_path': [test_ecs_path],
            'valid_stops': all_stops,
            'validate': False,
        }
        os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
        init_config(build_options=build_options)
        self.setup_hierarchical_modules()

        intel_ver = '2013.5.192-GCC-4.8.3'
        impi_modfile_path = os.path.join('Compiler', 'intel', intel_ver,
                                         'impi', '4.1.3.049')
        imkl_modfile_path = os.path.join('MPI', 'intel', intel_ver, 'impi',
                                         '4.1.3.049', 'imkl', '11.1.2.144')
        if get_module_syntax() == 'Lua':
            impi_modfile_path += '.lua'
            imkl_modfile_path += '.lua'

        # example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
        # not for the toolchain or any of the toolchain components,
        # since both icc/ifort and impi form the path to the top of the module tree
        iccifort_mods = ['icc', 'ifort', 'iccifort']
        tests = [
            ('i/impi/impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb',
             impi_modfile_path, iccifort_mods),
            ('i/imkl/imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb',
             imkl_modfile_path, iccifort_mods + ['iimpi', 'impi']),
        ]
        for ec_file, modfile_path, excluded_deps in tests:
            ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
            eb = EasyBlock(ec)
            eb.toolchain.prepare()
            modpath = eb.make_module_step()
            modfile_path = os.path.join(modpath, modfile_path)
            modtxt = read_file(modfile_path)

            for dep in excluded_deps:
                tup = (dep, modfile_path, modtxt)
                failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
                if get_module_syntax() == 'Tcl':
                    self.assertFalse(re.search('module load %s' % dep, modtxt),
                                     failmsg)
                elif get_module_syntax() == 'Lua':
                    self.assertFalse(re.search('load("%s")' % dep, modtxt),
                                     failmsg)
                else:
                    self.assertTrue(
                        False,
                        "Unknown module syntax: %s" % get_module_syntax())

        # modpath_extensions_for should spit out correct result, even if modules are loaded
        icc_mod = 'icc/%s' % intel_ver
        impi_mod = 'impi/4.1.3.049'
        self.modtool.load([icc_mod])
        self.assertTrue(impi_modfile_path in self.modtool.show(impi_mod))
        self.modtool.load([impi_mod])
        expected = {
            icc_mod: [os.path.join(modpath, 'Compiler', 'intel', intel_ver)],
            impi_mod: [
                os.path.join(modpath, 'MPI', 'intel', intel_ver, 'impi',
                             '4.1.3.049')
            ],
        }
        self.assertEqual(
            self.modtool.modpath_extensions_for([icc_mod, impi_mod]), expected)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
    """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""

    print('\n')
    info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)

    preinstallopts = ''

    if distribute_egg_dir is not None:
        # inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
        # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
        # this is required for the easy_install from stage 0 to work
        preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir

        # ensure that (latest) setuptools is installed as well alongside EasyBuild,
        # since it is a required runtime dependency for recent vsc-base and EasyBuild versions
        # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
        preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable

    # vsc-install is a runtime dependency for the EasyBuild unit test suite,
    # and is easily picked up from stage1 rather than being actually installed, so force it
    vsc_install = 'vsc-install'
    if sourcepath:
        vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
        if len(vsc_install_tarball_paths) == 1:
            vsc_install = vsc_install_tarball_paths[0]
    preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)

    templates.update({
        'preinstallopts': preinstallopts,
    })

    # determine PyPI URLs for individual packages
    pkg_urls = []
    for pkg in EASYBUILD_PACKAGES:
        # format of pkg entries in templates: "'<pkg_filename>',"
        pkg_filename = templates[pkg][1:-2]

        # the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
        # which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)

        # determine download URL via PyPI's 'simple' API
        pkg_simple = None
        try:
            pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
        except (urllib2.URLError, urllib2.HTTPError) as err:
            # failing to figure out the package download URl may be OK when source tarballs are provided
            if sourcepath:
                info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg)
            else:
                raise err

        if pkg_simple:
            pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename)
            res = pkg_url_part_regex.search(pkg_simple)
            if res:
                pkg_url_part = res.group(1)
            else:
                error("Failed to determine PyPI package URL for %s: %s\n" % (pkg, pkg_simple))

            pkg_url = 'https://pypi.python.org/' + pkg_url_part
            pkg_urls.append(pkg_url)

    templates.update({
        'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]),
        'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
        'pythonpath': distribute_egg_dir,
    })

    # create easyconfig file
    ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
    handle = open(ebfile, 'w')
    ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
    handle.write(ebfile_txt)
    handle.close()
    debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)

    # set command line arguments for eb
    eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
    if print_debug:
        eb_args.extend(['--debug', '--logtostdout'])
    if forced_install:
        info("Performing FORCED installation, as requested...")
        eb_args.append('--force')

    # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
    # and set build and install path explicitely
    if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
        os.environ['EASYBUILD_PREFIX'] = tmpdir
        os.environ['EASYBUILD_BUILDPATH'] = tmpdir
        if install_path is not None:
            os.environ['EASYBUILD_INSTALLPATH'] = install_path
    else:
        # only for v1.3 and up
        eb_args.append('--prefix=%s' % tmpdir)
        eb_args.append('--buildpath=%s' % tmpdir)
        if install_path is not None:
            eb_args.append('--installpath=%s' % install_path)
        if sourcepath is not None:
            eb_args.append('--sourcepath=%s' % sourcepath)

    # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
    if install_path is not None:
        modules_path = det_modules_path(install_path)
        if not os.path.exists(modules_path):
            os.makedirs(modules_path)
        debug("Created path %s" % modules_path)

    debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
    sys.argv = eb_args

    # location to 'eb' command (from stage 1) may be expected to be included in $PATH
    # it usually is there after stage1, unless 'prep' is called again with another location
    # (only when stage 0 is not skipped)
    # cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
    curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
    os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
    debug("$PATH: %s" % os.environ['PATH'])

    # install EasyBuild with EasyBuild
    from easybuild.main import main as easybuild_main
    easybuild_main()

    if print_debug:
        os.environ['EASYBUILD_DEBUG'] = '1'

    # make sure the EasyBuild module was actually installed
    # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
    # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
    from easybuild.tools.config import build_option, install_path, get_module_syntax
    from easybuild.framework.easyconfig.easyconfig import ActiveMNS
    eb_spec = {
        'name': 'EasyBuild',
        'hidden': False,
        'toolchain': {'name': 'dummy', 'version': 'dummy'},
        'version': templates['version'],
        'versionprefix': '',
        'versionsuffix': '',
        'moduleclass': 'tools',
    }

    mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
    debug("EasyBuild module should have been installed to %s" % mod_path)

    eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
    debug("EasyBuild module name: %s" % eb_mod_name)

    eb_mod_path = os.path.join(mod_path, eb_mod_name)
    if get_module_syntax() == 'Lua':
        eb_mod_path += '.lua'

    if os.path.exists(eb_mod_path):
        info("EasyBuild module installed: %s" % eb_mod_path)
    else:
        error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
示例#13
0
    def test_toy_module_fulltxt(self):
        """Strict text comparison of generated module file."""
        self.test_toy_tweaked()

        toy_module = os.path.join(self.test_installpath, "modules", "all", "toy", "0.0-tweaked")
        if get_module_syntax() == "Lua":
            toy_module += ".lua"
        toy_mod_txt = read_file(toy_module)

        if get_module_syntax() == "Lua":
            mod_txt_regex_pattern = "\n".join(
                [
                    r"help\(\[\[Toy C program. - Homepage: http://hpcugent.github.com/easybuild\]\]\)",
                    r"whatis\(\[\[Name: toy\]\]\)",
                    r"whatis\(\[\[Version: 0.0\]\]\)",
                    r"whatis\(\[\[Description: Toy C program. - Homepage: http://hpcugent.github.com/easybuild\]\]\)",
                    r"whatis\(\[\[Homepage: http://hpcugent.github.com/easybuild\]\]\)",
                    r"",
                    r'local root = "%s/software/toy/0.0-tweaked"' % self.test_installpath,
                    r"",
                    r'conflict\("toy"\)',
                    r"",
                    r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib"\)\)',
                    r'prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib"\)\)',
                    r'prepend_path\("PATH", pathJoin\(root, "bin"\)\)',
                    r'setenv\("EBROOTTOY", root\)',
                    r'setenv\("EBVERSIONTOY", "0.0"\)',
                    r'setenv\("EBDEVELTOY", pathJoin\(root, "easybuild/toy-0.0-tweaked-easybuild-devel"\)\)',
                    r"",
                    r'setenv\("FOO", "bar"\)',
                    r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)',
                    r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)',
                    r'prepend_path\("SOMEPATH", root\)',
                    r"",
                    r'if mode\(\) == "load" then',
                    r'    io.stderr:write\("THANKS FOR LOADING ME, I AM toy v0.0"\)',
                    r"end",
                    r'io.stderr:write\("oh hai\!"\)',
                    r"-- Built with EasyBuild version .*$",
                ]
            )
        elif get_module_syntax() == "Tcl":
            mod_txt_regex_pattern = "\n".join(
                [
                    r"^#%Module",
                    r"proc ModulesHelp { } {",
                    r"    puts stderr { Toy C program. - Homepage: http://hpcugent.github.com/easybuild",
                    r"    }",
                    r"}",
                    r"",
                    r"module-whatis {Description: Toy C program. - Homepage: http://hpcugent.github.com/easybuild}",
                    r"",
                    r"set root %s/software/toy/0.0-tweaked" % self.test_installpath,
                    r"",
                    r"conflict toy",
                    r"",
                    r"prepend-path	LD_LIBRARY_PATH		\$root/lib",
                    r"prepend-path	LIBRARY_PATH		\$root/lib",
                    r"prepend-path	PATH		\$root/bin",
                    r'setenv	EBROOTTOY		"\$root"',
                    r'setenv	EBVERSIONTOY		"0.0"',
                    r'setenv	EBDEVELTOY		"\$root/easybuild/toy-0.0-tweaked-easybuild-devel"',
                    r"",
                    r'setenv	FOO		"bar"',
                    r"prepend-path	SOMEPATH		\$root/foo/bar",
                    r"prepend-path	SOMEPATH		\$root/baz",
                    r"prepend-path	SOMEPATH		\$root",
                    r"",
                    r"if { \[ module-info mode load \] } {",
                    r'    puts stderr "THANKS FOR LOADING ME, I AM toy v0.0"',
                    r"}",
                    r'puts stderr "oh hai\!"',
                    r"# Built with EasyBuild version .*$",
                ]
            )
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        mod_txt_regex = re.compile(mod_txt_regex_pattern)
        msg = "Pattern '%s' matches with: %s" % (mod_txt_regex.pattern, toy_mod_txt)
        self.assertTrue(mod_txt_regex.match(toy_mod_txt), msg)
示例#14
0
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
    """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""

    print('\n')
    info(
        "+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n"
        % install_path)

    preinstallopts = ''

    if distribute_egg_dir is not None:
        # inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
        # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
        # this is required for the easy_install from stage 0 to work
        preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir

        # ensure that (latest) setuptools is installed as well alongside EasyBuild,
        # since it is a required runtime dependency for recent vsc-base and EasyBuild versions
        # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
        preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable

    # vsc-install is a runtime dependency for the EasyBuild unit test suite,
    # and is easily picked up from stage1 rather than being actually installed, so force it
    vsc_install = "'%s<0.11.4'" % VSC_INSTALL
    if sourcepath:
        vsc_install_tarball_paths = glob.glob(
            os.path.join(sourcepath, 'vsc-install*.tar.gz'))
        if len(vsc_install_tarball_paths) == 1:
            vsc_install = vsc_install_tarball_paths[0]
    preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (
        sys.executable, vsc_install)

    templates.update({
        'preinstallopts': preinstallopts,
    })

    # determine PyPI URLs for individual packages
    pkg_urls = []
    for pkg in EASYBUILD_PACKAGES:
        # format of pkg entries in templates: "'<pkg_filename>',"
        pkg_filename = templates[pkg][1:-2]

        # the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
        # which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)

        # determine download URL via PyPI's 'simple' API
        pkg_simple = None
        try:
            pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' %
                                         pkg,
                                         timeout=10).read()
        except (urllib2.URLError, urllib2.HTTPError) as err:
            # failing to figure out the package download URl may be OK when source tarballs are provided
            if sourcepath:
                info(
                    "Ignoring failed attempt to determine '%s' download URL since source tarballs are provided"
                    % pkg)
            else:
                raise err

        if pkg_simple:
            pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' %
                                            pkg_filename)
            res = pkg_url_part_regex.search(pkg_simple)
            if res:
                pkg_url_part = res.group(1)
            else:
                error("Failed to determine PyPI package URL for %s: %s\n" %
                      (pkg, pkg_simple))

            pkg_url = 'https://pypi.python.org/' + pkg_url_part
            pkg_urls.append(pkg_url)

    templates.update({
        'source_urls':
        '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]),
        'sources':
        "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s"
        % templates,
        'pythonpath':
        distribute_egg_dir,
    })

    # create easyconfig file
    ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
    handle = open(ebfile, 'w')
    ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
    handle.write(ebfile_txt)
    handle.close()
    debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)

    # set command line arguments for eb
    eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
    if print_debug:
        eb_args.extend(['--debug', '--logtostdout'])
    if forced_install:
        info("Performing FORCED installation, as requested...")
        eb_args.append('--force')

    # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
    # and set build and install path explicitely
    if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
        os.environ['EASYBUILD_PREFIX'] = tmpdir
        os.environ['EASYBUILD_BUILDPATH'] = tmpdir
        if install_path is not None:
            os.environ['EASYBUILD_INSTALLPATH'] = install_path
    else:
        # only for v1.3 and up
        eb_args.append('--prefix=%s' % tmpdir)
        eb_args.append('--buildpath=%s' % tmpdir)
        if install_path is not None:
            eb_args.append('--installpath=%s' % install_path)
        if sourcepath is not None:
            eb_args.append('--sourcepath=%s' % sourcepath)

    # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
    if install_path is not None:
        modules_path = det_modules_path(install_path)
        if not os.path.exists(modules_path):
            os.makedirs(modules_path)
        debug("Created path %s" % modules_path)

    debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
    sys.argv = eb_args

    # location to 'eb' command (from stage 1) may be expected to be included in $PATH
    # it usually is there after stage1, unless 'prep' is called again with another location
    # (only when stage 0 is not skipped)
    # cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
    curr_path = [
        x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0
    ]
    os.environ['PATH'] = os.pathsep.join(
        [os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
    debug("$PATH: %s" % os.environ['PATH'])

    # install EasyBuild with EasyBuild
    from easybuild.main import main as easybuild_main
    easybuild_main()

    if print_debug:
        os.environ['EASYBUILD_DEBUG'] = '1'

    # make sure the EasyBuild module was actually installed
    # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
    # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
    from easybuild.tools.config import build_option, install_path, get_module_syntax
    from easybuild.framework.easyconfig.easyconfig import ActiveMNS
    eb_spec = {
        'name': 'EasyBuild',
        'hidden': False,
        'toolchain': {
            'name': 'dummy',
            'version': 'dummy'
        },
        'version': templates['version'],
        'versionprefix': '',
        'versionsuffix': '',
        'moduleclass': 'tools',
    }

    mod_path = os.path.join(install_path('mod'),
                            build_option('suffix_modules_path'))
    debug("EasyBuild module should have been installed to %s" % mod_path)

    eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
    debug("EasyBuild module name: %s" % eb_mod_name)

    eb_mod_path = os.path.join(mod_path, eb_mod_name)
    if get_module_syntax() == 'Lua':
        eb_mod_path += '.lua'

    if os.path.exists(eb_mod_path):
        info("EasyBuild module installed: %s" % eb_mod_path)
    else:
        error(
            "EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug"
            % eb_mod_path)
示例#15
0
    def test_make_module_req(self):
        """Testcase for make_module_req"""
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            'toolchain = {"name":"dummy", "version": "dummy"}',
        ])
        self.writeEC()
        eb = EasyBlock(EasyConfig(self.eb_file))
        eb.installdir = config.install_path()

        # create fake directories and files that should be guessed
        os.makedirs(eb.installdir)
        open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
        open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
        os.mkdir(os.path.join(eb.installdir, 'bin'))
        os.mkdir(os.path.join(eb.installdir, 'share'))
        os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
        # this is not a path that should be picked up
        os.mkdir(os.path.join(eb.installdir, 'CPATH'))

        guess = eb.make_module_req()

        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
            self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
            self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
        elif get_module_syntax() == 'Lua':
            self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M))
            self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M))
            self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # check for behavior when a string value is used as dict value by make_module_req_guesses
        eb.make_module_req_guess = lambda: {'PATH': 'bin'}
        txt = eb.make_module_req()
        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M))
        elif get_module_syntax() == 'Lua':
            self.assertTrue(re.match(r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$', txt, re.M))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # check for correct behaviour if empty string is specified as one of the values
        # prepend-path statements should be included for both the 'bin' subdir and the install root
        eb.make_module_req_guess = lambda: {'PATH': ['bin', '']}
        txt = eb.make_module_req()
        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root/bin\n", txt, re.M))
            self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root\n", txt, re.M))
        elif get_module_syntax() == 'Lua':
            self.assertTrue(re.search(r'\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n', txt, re.M))
            self.assertTrue(re.search(r'\nprepend_path\("PATH", root\)\n', txt, re.M))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # cleanup
        eb.close_log()
        os.remove(eb.logfile)
示例#16
0
    def test_make_module_step(self):
        """Test the make_module_step"""
        name = "pi"
        version = "3.14"
        deps = [('GCC', '4.6.4')]
        modextravars = {'PI': '3.1415', 'FOO': 'bar'}
        modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "%s"' % name,
            'version = "%s"' % version,
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            "toolchain = {'name': 'dummy', 'version': 'dummy'}",
            "dependencies = [('GCC', '4.6.4'), ('toy', '0.0-deps')]",
            "builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
            # hidden deps must be included in list of (build)deps
            "hiddendependencies = [('toy', '0.0-deps'), ('OpenMPI', '1.6.4-GCC-4.6.4')]",
            "modextravars = %s" % str(modextravars),
            "modextrapaths = %s" % str(modextrapaths),
        ])

        test_dir = os.path.dirname(os.path.abspath(__file__))
        os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')

        # test if module is generated correctly
        self.writeEC()
        ec = EasyConfig(self.eb_file)
        eb = EasyBlock(ec)
        eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
        eb.check_readiness_step()

        modpath = os.path.join(eb.make_module_step(), name, version)
        if get_module_syntax() == 'Lua':
            modpath += '.lua'
        self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)

        # verify contents of module
        txt = read_file(modpath)
        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0]))
            self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M))

            self.assertTrue(
                re.search(r"^set\s+root\s+%s$" % eb.installdir, txt, re.M))
            ebroot_regex = re.compile(
                r'^setenv\s+EBROOT%s\s+"\$root"\s*$' % name.upper(), re.M)
            self.assertTrue(ebroot_regex.search(txt),
                            "%s in %s" % (ebroot_regex.pattern, txt))
            self.assertTrue(
                re.search(
                    r'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version),
                    txt, re.M))

        elif get_module_syntax() == 'Lua':
            ebroot_regex = re.compile(
                r'^setenv\("EBROOT%s", root\)$' % name.upper(), re.M)
            self.assertTrue(ebroot_regex.search(txt),
                            "%s in %s" % (ebroot_regex.pattern, txt))
            self.assertTrue(
                re.search(
                    r'^setenv\("EBVERSION%s", "%s"\)$' %
                    (name.upper(), version), txt, re.M))

        else:
            self.assertTrue(False,
                            "Unknown module syntax: %s" % get_module_syntax())

        for (key, val) in modextravars.items():
            if get_module_syntax() == 'Tcl':
                regex = re.compile(r'^setenv\s+%s\s+"%s"$' % (key, val), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(r'^setenv\("%s", "%s"\)$' % (key, val),
                                   re.M)
            else:
                self.assertTrue(
                    False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertTrue(regex.search(txt),
                            "Pattern %s found in %s" % (regex.pattern, txt))

        for (key, val) in modextrapaths.items():
            if get_module_syntax() == 'Tcl':
                regex = re.compile(
                    r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(
                    r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' %
                    (key, val), re.M)
            else:
                self.assertTrue(
                    False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertTrue(regex.search(txt),
                            "Pattern %s found in %s" % (regex.pattern, txt))

        for (name, ver) in [('GCC', '4.6.4')]:
            if get_module_syntax() == 'Tcl':
                regex = re.compile(
                    r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(
                    r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M)
            else:
                self.assertTrue(
                    False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertTrue(regex.search(txt),
                            "Pattern %s found in %s" % (regex.pattern, txt))

        for (name, ver) in [('toy', '0.0-deps')]:
            if get_module_syntax() == 'Tcl':
                regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver),
                                   re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver),
                                   re.M)
            else:
                self.assertTrue(
                    False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertTrue(regex.search(txt),
                            "Pattern %s found in %s" % (regex.pattern, txt))

        for (name, ver) in [('OpenMPI', '1.6.4-GCC-4.6.4')]:
            if get_module_syntax() == 'Tcl':
                regex = re.compile(
                    r'^\s*module load %s/.?%s\s*$' % (name, ver), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(r'^\s*load\("%s/.?%s"\)$' % (name, ver),
                                   re.M)
            else:
                self.assertTrue(
                    False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertFalse(
                regex.search(txt),
                "Pattern '%s' *not* found in %s" % (regex.pattern, txt))
    def test_make_module_step(self):
        """Test the make_module_step"""
        name = "pi"
        version = "3.14"
        deps = [('GCC', '4.6.4')]
        modextravars = {'PI': '3.1415', 'FOO': 'bar'}
        modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "%s"' % name,
            'version = "%s"' % version,
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            "toolchain = {'name': 'dummy', 'version': 'dummy'}",
            "dependencies = [('GCC', '4.6.4'), ('toy', '0.0-deps')]",
            "builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
            # hidden deps must be included in list of (build)deps
            "hiddendependencies = [('toy', '0.0-deps'), ('OpenMPI', '1.6.4-GCC-4.6.4')]",
            "modextravars = %s" % str(modextravars),
            "modextrapaths = %s" % str(modextrapaths),
        ])

        test_dir = os.path.dirname(os.path.abspath(__file__))
        os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')

        # test if module is generated correctly
        self.writeEC()
        ec = EasyConfig(self.eb_file)
        eb = EasyBlock(ec)
        eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
        eb.check_readiness_step()

        modpath = os.path.join(eb.make_module_step(), name, version)
        if get_module_syntax() == 'Lua':
            modpath += '.lua'
        self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)

        # verify contents of module
        txt = read_file(modpath)
        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0]))
            self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M))

            self.assertTrue(re.search(r"^set\s+root\s+%s$" % eb.installdir, txt, re.M))
            ebroot_regex = re.compile(r'^setenv\s+EBROOT%s\s+"\$root"\s*$' % name.upper(), re.M)
            self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt))
            self.assertTrue(re.search(r'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))

        elif get_module_syntax() == 'Lua':
            ebroot_regex = re.compile(r'^setenv\("EBROOT%s", root\)$' % name.upper(), re.M)
            self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt))
            self.assertTrue(re.search(r'^setenv\("EBVERSION%s", "%s"\)$' % (name.upper(), version), txt, re.M))

        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        for (key, val) in modextravars.items():
            if get_module_syntax() == 'Tcl':
                regex = re.compile(r'^setenv\s+%s\s+"%s"$' % (key, val), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(r'^setenv\("%s", "%s"\)$' % (key, val), re.M)
            else:
                self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))

        for (key, val) in modextrapaths.items():
            if get_module_syntax() == 'Tcl':
                regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
            else:
                self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))

        for (name, ver) in [('GCC', '4.6.4')]:
            if get_module_syntax() == 'Tcl':
                regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M)
            else:
                self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))

        for (name, ver) in [('toy', '0.0-deps')]:
            if get_module_syntax() == 'Tcl':
                regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver), re.M)
            else:
                self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))

        for (name, ver) in [('OpenMPI', '1.6.4-GCC-4.6.4')]:
            if get_module_syntax() == 'Tcl':
                regex = re.compile(r'^\s*module load %s/.?%s\s*$' % (name, ver), re.M)
            elif get_module_syntax() == 'Lua':
                regex = re.compile(r'^\s*load\("%s/.?%s"\)$' % (name, ver), re.M)
            else:
                self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
            self.assertFalse(regex.search(txt), "Pattern '%s' *not* found in %s" % (regex.pattern, txt))
示例#18
0
    def test_make_module_dep(self):
        """Test for make_module_dep"""
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            "toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
            'dependencies = [',
            "   ('FFTW', '3.3.1'),",
            "   ('LAPACK', '3.4.0'),",
            ']',
        ])
        self.writeEC()
        eb = EasyBlock(EasyConfig(self.eb_file))

        eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
        eb.check_readiness_step()

        if get_module_syntax() == 'Tcl':
            tc_load = '\n'.join([
                "if { ![ is-loaded gompi/1.1.0-no-OFED ] } {",
                "    module load gompi/1.1.0-no-OFED",
                "}",
            ])
            fftw_load = '\n'.join([
                "if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
                "    module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
                "}",
            ])
            lapack_load = '\n'.join([
                "if { ![ is-loaded LAPACK/3.4.0-gompi-1.1.0-no-OFED ] } {",
                "    module load LAPACK/3.4.0-gompi-1.1.0-no-OFED",
                "}",
            ])
        elif get_module_syntax() == 'Lua':
            tc_load = '\n'.join([
                'if not isloaded("gompi/1.1.0-no-OFED") then',
                '    load("gompi/1.1.0-no-OFED")',
                'end',
            ])
            fftw_load = '\n'.join([
                'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
                '    load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
                'end',
            ])
            lapack_load = '\n'.join([
                'if not isloaded("LAPACK/3.4.0-gompi-1.1.0-no-OFED") then',
                '    load("LAPACK/3.4.0-gompi-1.1.0-no-OFED")',
                'end',
            ])
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
        self.assertEqual(eb.make_module_dep().strip(), expected)

        # provide swap info for FFTW to trigger an extra 'unload FFTW'
        unload_info = {
            'FFTW/3.3.1-gompi-1.1.0-no-OFED': 'FFTW',
        }

        if get_module_syntax() == 'Tcl':
            fftw_load = '\n'.join([
                "if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
                "    module unload FFTW",
                "    module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
                "}",
            ])
        elif get_module_syntax() == 'Lua':
            fftw_load = '\n'.join([
                'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
                '    unload("FFTW")',
                '    load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
                'end',
            ])
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
        expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
        self.assertEqual(eb.make_module_dep(unload_info=unload_info).strip(), expected)
示例#19
0
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
    """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""

    print('\n')
    info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)

    preinstallopts = ''

    if distribute_egg_dir is not None:
        # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
        # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
        # this is required for the easy_install from stage 1 to work
        preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir

        # ensure that (latest) setuptools is installed as well alongside EasyBuild,
        # since it is a required runtime dependency for recent vsc-base and EasyBuild versions
        # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
        preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable

    # vsc-install is a runtime dependency for the EasyBuild unit test suite,
    # and is easily picked up from stage1 rather than being actually installed, so force it
    vsc_install = 'vsc-install'
    if sourcepath:
        vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
        if len(vsc_install_tarball_paths) == 1:
            vsc_install = vsc_install_tarball_paths[0]
    preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)

    templates.update({
        'preinstallopts': preinstallopts,
    })

    # create easyconfig file
    ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
    handle = open(ebfile, 'w')
    templates.update({
        'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]),
        'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
        'pythonpath': distribute_egg_dir,
    })
    handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
    handle.close()

    # set command line arguments for eb
    eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
    if print_debug:
        eb_args.extend(['--debug', '--logtostdout'])
    if forced_install:
        info("Performing FORCED installation, as requested...")
        eb_args.append('--force')

    # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
    # and set build and install path explicitely
    if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
        os.environ['EASYBUILD_PREFIX'] = tmpdir
        os.environ['EASYBUILD_BUILDPATH'] = tmpdir
        if install_path is not None:
            os.environ['EASYBUILD_INSTALLPATH'] = install_path
    else:
        # only for v1.3 and up
        eb_args.append('--prefix=%s' % tmpdir)
        eb_args.append('--buildpath=%s' % tmpdir)
        if install_path is not None:
            eb_args.append('--installpath=%s' % install_path)
        if sourcepath is not None:
            eb_args.append('--sourcepath=%s' % sourcepath)

    # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
    if install_path is not None:
        modules_path = det_modules_path(install_path)
        if not os.path.exists(modules_path):
            os.makedirs(modules_path)
        debug("Created path %s" % modules_path)

    debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
    sys.argv = eb_args

    # location to 'eb' command (from stage 1) may be expected to be included in $PATH
    # it usually is there after stage1, unless 'prep' is called again with another location
    # (only when stage 0 is not skipped)
    # cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
    curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
    os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
    debug("$PATH: %s" % os.environ['PATH'])

    # install EasyBuild with EasyBuild
    from easybuild.main import main as easybuild_main
    easybuild_main()

    # make sure the EasyBuild module was actually installed
    # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
    # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
    from easybuild.tools.config import build_option, install_path, get_module_syntax
    from easybuild.framework.easyconfig.easyconfig import ActiveMNS
    eb_spec = {
        'name': 'EasyBuild',
        'hidden': False,
        'toolchain': {'name': 'dummy', 'version': 'dummy'},
        'version': templates['version'],
        'versionprefix': '',
        'versionsuffix': '',
        'moduleclass': 'tools',
    }

    mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
    debug("EasyBuild module should have been installed to %s" % mod_path)

    eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
    debug("EasyBuild module name: %s" % eb_mod_name)

    eb_mod_path = os.path.join(mod_path, eb_mod_name)
    if get_module_syntax() == 'Lua':
        eb_mod_path += '.lua'

    if os.path.exists(eb_mod_path):
        info("EasyBuild module installed: %s" % eb_mod_path)
    else:
        error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
    def test_make_module_dep(self):
        """Test for make_module_dep"""
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            "toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
            'dependencies = [',
            "   ('FFTW', '3.3.1'),",
            "   ('LAPACK', '3.4.0'),",
            ']',
        ])
        self.writeEC()
        eb = EasyBlock(EasyConfig(self.eb_file))

        eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
        eb.check_readiness_step()

        if get_module_syntax() == 'Tcl':
            tc_load = '\n'.join([
                "if { ![ is-loaded gompi/1.1.0-no-OFED ] } {",
                "    module load gompi/1.1.0-no-OFED",
                "}",
            ])
            fftw_load = '\n'.join([
                "if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
                "    module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
                "}",
            ])
            lapack_load = '\n'.join([
                "if { ![ is-loaded LAPACK/3.4.0-gompi-1.1.0-no-OFED ] } {",
                "    module load LAPACK/3.4.0-gompi-1.1.0-no-OFED",
                "}",
            ])
        elif get_module_syntax() == 'Lua':
            tc_load = '\n'.join([
                'if not isloaded("gompi/1.1.0-no-OFED") then',
                '    load("gompi/1.1.0-no-OFED")',
                'end',
            ])
            fftw_load = '\n'.join([
                'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
                '    load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
                'end',
            ])
            lapack_load = '\n'.join([
                'if not isloaded("LAPACK/3.4.0-gompi-1.1.0-no-OFED") then',
                '    load("LAPACK/3.4.0-gompi-1.1.0-no-OFED")',
                'end',
            ])
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
        self.assertEqual(eb.make_module_dep().strip(), expected)

        # provide swap info for FFTW to trigger an extra 'unload FFTW'
        unload_info = {
            'FFTW/3.3.1-gompi-1.1.0-no-OFED': 'FFTW',
        }

        if get_module_syntax() == 'Tcl':
            fftw_load = '\n'.join([
                "if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
                "    module unload FFTW",
                "    module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
                "}",
            ])
        elif get_module_syntax() == 'Lua':
            fftw_load = '\n'.join([
                'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
                '    unload("FFTW")',
                '    load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
                'end',
            ])
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
        expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
        self.assertEqual(eb.make_module_dep(unload_info=unload_info).strip(), expected)
示例#21
0
    def test_toy_permissions(self):
        """Test toy build with custom umask settings."""
        toy_ec_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "toy-0.0.eb")
        args = [
            "--sourcepath=%s" % self.test_sourcepath,
            "--buildpath=%s" % self.test_buildpath,
            "--installpath=%s" % self.test_installpath,
            "--debug",
            "--unittest-file=%s" % self.logfile,
            "--force",
        ]

        # set umask hard to verify default reliably
        orig_umask = os.umask(0022)

        # test specifying a non-existing group
        allargs = [toy_ec_file] + args + ["--group=thisgroupdoesnotexist"]
        outtxt, err = self.eb_main(allargs, logfile=self.dummylogfn, do_build=True, return_error=True)
        err_regex = re.compile("Failed to get group ID .* group does not exist")
        self.assertTrue(err_regex.search(outtxt), "Pattern '%s' found in '%s'" % (err_regex.pattern, outtxt))

        # determine current group name (at least we can use that)
        gid = os.getgid()
        curr_grp = grp.getgrgid(gid).gr_name

        for umask, cfg_group, ec_group, dir_perms, fil_perms, bin_perms in [
            (None, None, None, 0755, 0644, 0755),  # default: inherit session umask
            (None, None, curr_grp, 0750, 0640, 0750),  # default umask, but with specified group in ec
            (None, curr_grp, None, 0750, 0640, 0750),  # default umask, but with specified group in cfg
            (None, "notagrp", curr_grp, 0750, 0640, 0750),  # default umask, but with specified group in both cfg and ec
            ("000", None, None, 0777, 0666, 0777),  # stupid empty umask
            ("032", None, None, 0745, 0644, 0745),  # no write/execute for group, no write for other
            ("030", None, curr_grp, 0740, 0640, 0740),  # no write for group, with specified group
            ("077", None, None, 0700, 0600, 0700),  # no access for other/group
        ]:
            # empty the install directory, to ensure any created directories adher to the permissions
            shutil.rmtree(self.test_installpath)

            if cfg_group is None and ec_group is None:
                allargs = [toy_ec_file]
            elif ec_group is not None:
                shutil.copy2(toy_ec_file, self.test_buildpath)
                tmp_ec_file = os.path.join(self.test_buildpath, os.path.basename(toy_ec_file))
                write_file(tmp_ec_file, "\ngroup = '%s'" % ec_group, append=True)
                allargs = [tmp_ec_file]
            allargs.extend(args)
            if umask is not None:
                allargs.append("--umask=%s" % umask)
            if cfg_group is not None:
                allargs.append("--group=%s" % cfg_group)
            outtxt = self.eb_main(allargs, logfile=self.dummylogfn, do_build=True, verbose=True)

            # verify that installation was correct
            self.check_toy(self.test_installpath, outtxt)

            # group specified in easyconfig overrules configured group
            group = cfg_group
            if ec_group is not None:
                group = ec_group

            # verify permissions
            paths_perms = [
                # no write permissions for group/other, regardless of umask
                (("software", "toy", "0.0"), dir_perms & ~0022),
                (("software", "toy", "0.0", "bin"), dir_perms & ~0022),
                (("software", "toy", "0.0", "bin", "toy"), bin_perms & ~0022),
            ]
            # only software subdirs are chmod'ed for 'protected' installs, so don't check those if a group is specified
            if group is None:
                paths_perms.extend(
                    [
                        (("software",), dir_perms),
                        (("software", "toy"), dir_perms),
                        (("software", "toy", "0.0", "easybuild", "*.log"), fil_perms),
                        (("modules",), dir_perms),
                        (("modules", "all"), dir_perms),
                        (("modules", "all", "toy"), dir_perms),
                    ]
                )
                if get_module_syntax() == "Tcl":
                    paths_perms.append((("modules", "all", "toy", "0.0"), fil_perms))
                elif get_module_syntax() == "Lua":
                    paths_perms.append((("modules", "all", "toy", "0.0.lua"), fil_perms))

            for path, correct_perms in paths_perms:
                fullpath = glob.glob(os.path.join(self.test_installpath, *path))[0]
                perms = os.stat(fullpath).st_mode & 0777
                msg = "Path %s has %s permissions: %s" % (fullpath, oct(correct_perms), oct(perms))
                self.assertEqual(perms, correct_perms, msg)
                if group is not None:
                    path_gid = os.stat(fullpath).st_gid
                    self.assertEqual(path_gid, grp.getgrnam(group).gr_gid)
示例#22
0
    def test_toy_hierarchical(self):
        """Test toy build under example hierarchical module naming scheme."""

        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), "easyconfigs")
        self.setup_hierarchical_modules()
        mod_prefix = os.path.join(self.test_installpath, "modules", "all")

        args = [
            os.path.join(test_easyconfigs, "toy-0.0.eb"),
            "--sourcepath=%s" % self.test_sourcepath,
            "--buildpath=%s" % self.test_buildpath,
            "--installpath=%s" % self.test_installpath,
            "--debug",
            "--unittest-file=%s" % self.logfile,
            "--force",
            "--robot=%s" % test_easyconfigs,
            "--module-naming-scheme=HierarchicalMNS",
        ]

        # test module paths/contents with gompi build
        extra_args = ["--try-toolchain=goolf,1.4.10"]
        self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)

        # make sure module file is installed in correct path
        toy_module_path = os.path.join(mod_prefix, "MPI", "GCC", "4.7.2", "OpenMPI", "1.6.4", "toy", "0.0")
        if get_module_syntax() == "Lua":
            toy_module_path += ".lua"
        self.assertTrue(os.path.exists(toy_module_path))

        # check that toolchain load is expanded to loads for toolchain dependencies,
        # except for the ones that extend $MODULEPATH to make the toy module available
        if get_module_syntax() == "Tcl":
            load_regex_template = "load %s"
        elif get_module_syntax() == "Lua":
            load_regex_template = r'load\("%s/.*"\)'
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        modtxt = read_file(toy_module_path)
        for dep in ["goolf", "GCC", "OpenMPI"]:
            load_regex = re.compile(load_regex_template % dep)
            self.assertFalse(load_regex.search(modtxt), "Pattern '%s' not found in %s" % (load_regex.pattern, modtxt))
        for dep in ["OpenBLAS", "FFTW", "ScaLAPACK"]:
            load_regex = re.compile(load_regex_template % dep)
            self.assertTrue(load_regex.search(modtxt), "Pattern '%s' found in %s" % (load_regex.pattern, modtxt))

        os.remove(toy_module_path)

        # test module path with GCC/4.7.2 build
        extra_args = ["--try-toolchain=GCC,4.7.2"]
        self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)

        # make sure module file is installed in correct path
        toy_module_path = os.path.join(mod_prefix, "Compiler", "GCC", "4.7.2", "toy", "0.0")
        if get_module_syntax() == "Lua":
            toy_module_path += ".lua"
        self.assertTrue(os.path.exists(toy_module_path))

        # no dependencies or toolchain => no module load statements in module file
        modtxt = read_file(toy_module_path)
        self.assertFalse(re.search("module load", modtxt))
        os.remove(toy_module_path)

        # test module path with GCC/4.7.2 build, pretend to be an MPI lib by setting moduleclass
        extra_args = ["--try-toolchain=GCC,4.7.2", "--try-amend=moduleclass=mpi"]
        self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)

        # make sure module file is installed in correct path
        toy_module_path = os.path.join(mod_prefix, "Compiler", "GCC", "4.7.2", "toy", "0.0")
        if get_module_syntax() == "Lua":
            toy_module_path += ".lua"
        self.assertTrue(os.path.exists(toy_module_path))

        # 'module use' statements to extend $MODULEPATH are present
        modtxt = read_file(toy_module_path)
        modpath_extension = os.path.join(mod_prefix, "MPI", "GCC", "4.7.2", "toy", "0.0")
        if get_module_syntax() == "Tcl":
            self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M))
        elif get_module_syntax() == "Lua":
            fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension)
            regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M)
            self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
        os.remove(toy_module_path)

        # ... unless they shouldn't be
        extra_args.append("--try-amend=include_modpath_extensions=")  # pass empty string as equivalent to False
        self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
        modtxt = read_file(toy_module_path)
        modpath_extension = os.path.join(mod_prefix, "MPI", "GCC", "4.7.2", "toy", "0.0")
        if get_module_syntax() == "Tcl":
            self.assertFalse(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M))
        elif get_module_syntax() == "Lua":
            fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension)
            regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M)
            self.assertFalse(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
        os.remove(toy_module_path)

        # test module path with dummy/dummy build
        extra_args = ["--try-toolchain=dummy,dummy"]
        self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)

        # make sure module file is installed in correct path
        toy_module_path = os.path.join(mod_prefix, "Core", "toy", "0.0")
        if get_module_syntax() == "Lua":
            toy_module_path += ".lua"
        self.assertTrue(os.path.exists(toy_module_path))

        # no dependencies or toolchain => no module load statements in module file
        modtxt = read_file(toy_module_path)
        self.assertFalse(re.search("module load", modtxt))
        os.remove(toy_module_path)

        # test module path with dummy/dummy build, pretend to be a compiler by setting moduleclass
        extra_args = ["--try-toolchain=dummy,dummy", "--try-amend=moduleclass=compiler"]
        self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)

        # make sure module file is installed in correct path
        toy_module_path = os.path.join(mod_prefix, "Core", "toy", "0.0")
        if get_module_syntax() == "Lua":
            toy_module_path += ".lua"
        self.assertTrue(os.path.exists(toy_module_path))

        # no dependencies or toolchain => no module load statements in module file
        modtxt = read_file(toy_module_path)
        modpath_extension = os.path.join(mod_prefix, "Compiler", "toy", "0.0")
        if get_module_syntax() == "Tcl":
            self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M))
        elif get_module_syntax() == "Lua":
            fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension)
            regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M)
            self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt))
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
        os.remove(toy_module_path)

        # building a toolchain module should also work
        args[0] = os.path.join(test_easyconfigs, "gompi-1.4.10.eb")
        modules_tool().purge()
        self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=False)
        gompi_module_path = os.path.join(mod_prefix, "Core", "gompi", "1.4.10")
        if get_module_syntax() == "Lua":
            gompi_module_path += ".lua"
        self.assertTrue(os.path.exists(gompi_module_path))
示例#23
0
    def test_make_module_extend_modpath(self):
        """Test for make_module_extend_modpath"""
        self.contents = '\n'.join([
            'easyblock = "ConfigureMake"',
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            'toolchain = {"name":"dummy", "version": "dummy"}',
            'moduleclass = "compiler"',
        ])
        self.writeEC()
        eb = EasyBlock(EasyConfig(self.eb_file))
        eb.installdir = config.install_path()

        # no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
        self.assertEqual(eb.make_module_extend_modpath(), '')

        usermodsdir = 'my/own/modules'
        modclasses = ['compiler', 'tools']
        os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
        build_options = {
            'subdir_user_modules': usermodsdir,
            'valid_module_classes': modclasses,
        }
        init_config(build_options=build_options)
        eb = EasyBlock(EasyConfig(self.eb_file))
        eb.installdir = config.install_path()

        txt = eb.make_module_extend_modpath()
        if get_module_syntax() == 'Tcl':
            regexs = [
                r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c
                for c in modclasses
            ]
            home = r'\$env\(HOME\)'
            regexs.extend([
                # extension for user modules is guarded
                r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$'
                % (home, usermodsdir),
                # no per-moduleclass extension for user modules
                r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' %
                (home, usermodsdir),
            ])
        elif get_module_syntax() == 'Lua':
            regexs = [
                r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$'
                % c for c in modclasses
            ]
            home = r'os.getenv\("HOME"\)'
            regexs.extend([
                # extension for user modules is guarded
                r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' %
                (home, usermodsdir),
                # no per-moduleclass extension for user modules
                r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)'
                % (home, usermodsdir),
            ])
        else:
            self.assertTrue(False,
                            "Unknown module syntax: %s" % get_module_syntax())
        for regex in regexs:
            regex = re.compile(regex, re.M)
            self.assertTrue(regex.search(txt),
                            "Pattern '%s' found in: %s" % (regex.pattern, txt))