def test_toy_tweaked(self):
"""Test toy build with tweaked easyconfig, for testing extra easyconfig parameters."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "easyconfigs")
ec_file = os.path.join(self.test_buildpath, "toy-0.0-tweaked.eb")
shutil.copy2(os.path.join(test_ecs_dir, "toy-0.0.eb"), ec_file)
# tweak easyconfig by appending to it
ec_extra = "\n".join(
[
"versionsuffix = '-tweaked'",
"modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}",
"modextravars = {'FOO': 'bar'}",
"modloadmsg = 'THANKS FOR LOADING ME, I AM %(name)s v%(version)s'",
"modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua
"modluafooter = 'io.stderr:write(\"oh hai!\")'", # ignored when module syntax is Tcl
]
)
write_file(ec_file, ec_extra, append=True)
args = [
ec_file,
"--sourcepath=%s" % self.test_sourcepath,
"--buildpath=%s" % self.test_buildpath,
"--installpath=%s" % self.test_installpath,
"--debug",
"--force",
]
outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.check_toy(self.test_installpath, outtxt, versionsuffix="-tweaked")
toy_module = os.path.join(self.test_installpath, "modules", "all", "toy", "0.0-tweaked")
if get_module_syntax() == "Lua":
toy_module += ".lua"
toy_module_txt = read_file(toy_module)
if get_module_syntax() == "Tcl":
self.assertTrue(re.search(r'^setenv\s*FOO\s*"bar"$', toy_module_txt, re.M))
self.assertTrue(re.search(r"^prepend-path\s*SOMEPATH\s*\$root/foo/bar$", toy_module_txt, re.M))
self.assertTrue(re.search(r"^prepend-path\s*SOMEPATH\s*\$root/baz$", toy_module_txt, re.M))
self.assertTrue(re.search(r"^prepend-path\s*SOMEPATH\s*\$root$", toy_module_txt, re.M))
self.assertTrue(
re.search(r'module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0"$', toy_module_txt, re.M)
)
self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M))
elif get_module_syntax() == "Lua":
self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M))
self.assertTrue(
re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$', toy_module_txt, re.M)
)
self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M))
self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M))
self.assertTrue(
re.search(
r'^if mode\(\) == "load" then\n\s*io.stderr:write\(".*I AM toy v0.0"\)$', toy_module_txt, re.M
)
)
self.assertTrue(re.search(r'^io.stderr:write\("oh hai!"\)$', toy_module_txt, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
def test_exclude_path_to_top_of_module_tree(self):
"""
Make sure that modules under the HierarchicalMNS are correct,
w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
"""
self.orig_module_naming_scheme = config.get_module_naming_scheme()
test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all')
mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True)
mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True)
impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049')
imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144')
if get_module_syntax() == 'Lua':
impi_modfile_path += '.lua'
imkl_modfile_path += '.lua'
# example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
# not for the toolchain or any of the toolchain components,
# since both icc/ifort and impi form the path to the top of the module tree
tests = [
('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']),
('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']),
]
for ec_file, modfile_path, excluded_deps in tests:
ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
eb = EasyBlock(ec)
eb.toolchain.prepare()
modpath = eb.make_module_step()
modfile_path = os.path.join(modpath, modfile_path)
modtxt = read_file(modfile_path)
for dep in excluded_deps:
tup = (dep, modfile_path, modtxt)
failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
if get_module_syntax() == 'Tcl':
self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg)
elif get_module_syntax() == 'Lua':
self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
def test_make_module_extend_modpath(self):
"""Test for make_module_extend_modpath"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'moduleclass = "compiler"',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
self.assertEqual(eb.make_module_extend_modpath(), '')
usermodsdir = 'my/own/modules'
modclasses = ['compiler', 'tools']
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
build_options = {
'subdir_user_modules': usermodsdir,
'valid_module_classes': modclasses,
}
init_config(build_options=build_options)
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
txt = eb.make_module_extend_modpath()
if get_module_syntax() == 'Tcl':
regexs = [r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c for c in modclasses]
home = r'\$env\(HOME\)'
regexs.extend([
# extension for user modules is guarded
r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' % (home, usermodsdir),
])
elif get_module_syntax() == 'Lua':
regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$' % c for c in modclasses]
home = r'os.getenv\("HOME"\)'
regexs.extend([
# extension for user modules is guarded
r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)' % (home, usermodsdir),
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
for regex in regexs:
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def test_make_module_extra(self):
"""Test for make_module_extra."""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
if get_module_syntax() == 'Tcl':
expected_default = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+\"\$root"',
r'setenv\s+EBVERSIONPI\s+"3.14"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+"/opt/software/tau/6.28"',
r'setenv\s+EBVERSIONPI\s+"6.28"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
elif get_module_syntax() == 'Lua':
expected_default = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", root\)',
r'setenv\("EBVERSIONPI", "3.14"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", "/opt/software/tau/6.28"\)',
r'setenv\("EBVERSIONPI", "6.28"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
defaulttxt = eb.make_module_extra().strip()
self.assertTrue(expected_default.match(defaulttxt),
"Pattern %s found in %s" % (expected_default.pattern, defaulttxt))
alttxt = eb.make_module_extra(altroot='/opt/software/tau/6.28', altversion='6.28').strip()
self.assertTrue(expected_alt.match(alttxt),
"Pattern %s found in %s" % (expected_alt.pattern, alttxt))
def test_module_filepath_tweaking(self):
"""Test using --suffix-modules-path."""
# install test module naming scheme dynamically
test_mns_parent_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox")
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
mns_path = "easybuild.tools.module_naming_scheme.test_module_naming_scheme"
__import__(mns_path, globals(), locals(), [""])
eb_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "toy-0.0.eb")
args = [
eb_file,
"--sourcepath=%s" % self.test_sourcepath,
"--buildpath=%s" % self.test_buildpath,
"--installpath=%s" % self.test_installpath,
"--force",
"--debug",
"--suffix-modules-path=foobarbaz",
"--module-naming-scheme=TestModuleNamingScheme",
]
self.eb_main(args, do_build=True, verbose=True)
mod_file_prefix = os.path.join(self.test_installpath, "modules")
mod_file_suffix = ""
if get_module_syntax() == "Lua":
mod_file_suffix += ".lua"
self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, "foobarbaz", "toy", "0.0" + mod_file_suffix)))
self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, "TOOLS", "toy", "0.0" + mod_file_suffix)))
self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, "TOOLS", "toy", "0.0" + mod_file_suffix)))
self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, "t", "toy", "0.0" + mod_file_suffix)))
self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, "t", "toy", "0.0" + mod_file_suffix)))
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M))
self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_toy_hidden(self):
"""Test installing a hidden module."""
ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "easyconfigs", "toy-0.0.eb")
self.test_toy_build(ec_file=ec_file, extra_args=["--hidden"], verify=False)
# module file is hidden
toy_module = os.path.join(self.test_installpath, "modules", "all", "toy", ".0.0")
if get_module_syntax() == "Lua":
toy_module += ".lua"
self.assertTrue(os.path.exists(toy_module), "Found hidden module %s" % toy_module)
# installed software is not hidden
toybin = os.path.join(self.test_installpath, "software", "toy", "0.0", "bin", "toy")
self.assertTrue(os.path.exists(toybin))
def module_generator(app, fake=False):
"""
Return ModuleGenerator instance that matches the selected module file syntax to be used
"""
module_syntax = get_module_syntax()
available_mod_gens = avail_module_generators()
if module_syntax not in available_mod_gens:
raise EasyBuildError("No module generator available for specified syntax '%s' (available: %s)",
module_syntax, available_mod_gens)
module_generator_class = available_mod_gens[module_syntax]
return module_generator_class(app, fake=fake)
def check_toy(self, installpath, outtxt, version="0.0", versionprefix="", versionsuffix=""):
"""Check whether toy build succeeded."""
full_version = "".join([versionprefix, version, versionsuffix])
# check for success
success = re.compile("COMPLETED: Installation ended successfully")
self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s" % outtxt)
# if the module exists, it should be fine
toy_module = os.path.join(installpath, "modules", "all", "toy", full_version)
msg = "module for toy build toy/%s found (path %s)" % (full_version, toy_module)
if get_module_syntax() == "Lua":
toy_module += ".lua"
self.assertTrue(os.path.exists(toy_module), msg)
# module file is symlinked according to moduleclass
toy_module_symlink = os.path.join(installpath, "modules", "tools", "toy", full_version)
if get_module_syntax() == "Lua":
toy_module_symlink += ".lua"
self.assertTrue(os.path.islink(toy_module_symlink))
self.assertTrue(os.path.exists(toy_module_symlink))
# make sure installation log file and easyconfig file are copied to install dir
software_path = os.path.join(installpath, "software", "toy", full_version)
install_log_path_pattern = os.path.join(software_path, "easybuild", "easybuild-toy-%s*.log" % version)
self.assertTrue(len(glob.glob(install_log_path_pattern)) == 1, "Found 1 file at %s" % install_log_path_pattern)
# make sure test report is available
test_report_path_pattern = os.path.join(software_path, "easybuild", "easybuild-toy-%s*test_report.md" % version)
self.assertTrue(len(glob.glob(test_report_path_pattern)) == 1, "Found 1 file at %s" % test_report_path_pattern)
ec_file_path = os.path.join(software_path, "easybuild", "toy-%s.eb" % full_version)
self.assertTrue(os.path.exists(ec_file_path))
devel_module_path = os.path.join(software_path, "easybuild", "toy-%s-easybuild-devel" % full_version)
self.assertTrue(os.path.exists(devel_module_path))
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M))
self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# check for behavior when a string value is used as dict value by make_module_req_guesses
eb.make_module_req_guess = lambda: {'PATH': 'bin'}
txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.match(r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$', txt, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# check for correct behaviour if empty string is specified as one of the values
# prepend-path statements should be included for both the 'bin' subdir and the install root
eb.make_module_req_guess = lambda: {'PATH': ['bin', '']}
txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root/bin\n", txt, re.M))
self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root\n", txt, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.search(r'\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n', txt, re.M))
self.assertTrue(re.search(r'\nprepend_path\("PATH", root\)\n', txt, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_exclude_path_to_top_of_module_tree(self):
"""
Make sure that modules under the HierarchicalMNS are correct,
w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
"""
self.orig_module_naming_scheme = config.get_module_naming_scheme()
test_ecs_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
intel_ver = '2013.5.192-GCC-4.8.3'
impi_modfile_path = os.path.join('Compiler', 'intel', intel_ver,
'impi', '4.1.3.049')
imkl_modfile_path = os.path.join('MPI', 'intel', intel_ver, 'impi',
'4.1.3.049', 'imkl', '11.1.2.144')
if get_module_syntax() == 'Lua':
impi_modfile_path += '.lua'
imkl_modfile_path += '.lua'
# example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
# not for the toolchain or any of the toolchain components,
# since both icc/ifort and impi form the path to the top of the module tree
iccifort_mods = ['icc', 'ifort', 'iccifort']
tests = [
('i/impi/impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb',
impi_modfile_path, iccifort_mods),
('i/imkl/imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb',
imkl_modfile_path, iccifort_mods + ['iimpi', 'impi']),
]
for ec_file, modfile_path, excluded_deps in tests:
ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
eb = EasyBlock(ec)
eb.toolchain.prepare()
modpath = eb.make_module_step()
modfile_path = os.path.join(modpath, modfile_path)
modtxt = read_file(modfile_path)
for dep in excluded_deps:
tup = (dep, modfile_path, modtxt)
failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
if get_module_syntax() == 'Tcl':
self.assertFalse(re.search('module load %s' % dep, modtxt),
failmsg)
elif get_module_syntax() == 'Lua':
self.assertFalse(re.search('load("%s")' % dep, modtxt),
failmsg)
else:
self.assertTrue(
False,
"Unknown module syntax: %s" % get_module_syntax())
# modpath_extensions_for should spit out correct result, even if modules are loaded
icc_mod = 'icc/%s' % intel_ver
impi_mod = 'impi/4.1.3.049'
self.modtool.load([icc_mod])
self.assertTrue(impi_modfile_path in self.modtool.show(impi_mod))
self.modtool.load([impi_mod])
expected = {
icc_mod: [os.path.join(modpath, 'Compiler', 'intel', intel_ver)],
impi_mod: [
os.path.join(modpath, 'MPI', 'intel', intel_ver, 'impi',
'4.1.3.049')
],
}
self.assertEqual(
self.modtool.modpath_extensions_for([icc_mod, impi_mod]), expected)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 0 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = 'vsc-install'
if sourcepath:
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# determine PyPI URLs for individual packages
pkg_urls = []
for pkg in EASYBUILD_PACKAGES:
# format of pkg entries in templates: "'<pkg_filename>',"
pkg_filename = templates[pkg][1:-2]
# the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
# which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)
# determine download URL via PyPI's 'simple' API
pkg_simple = None
try:
pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
except (urllib2.URLError, urllib2.HTTPError) as err:
# failing to figure out the package download URl may be OK when source tarballs are provided
if sourcepath:
info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg)
else:
raise err
if pkg_simple:
pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename)
res = pkg_url_part_regex.search(pkg_simple)
if res:
pkg_url_part = res.group(1)
else:
error("Failed to determine PyPI package URL for %s: %s\n" % (pkg, pkg_simple))
pkg_url = 'https://pypi.python.org/' + pkg_url_part
pkg_urls.append(pkg_url)
templates.update({
'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
handle.write(ebfile_txt)
handle.close()
debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
if print_debug:
os.environ['EASYBUILD_DEBUG'] = '1'
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def test_toy_module_fulltxt(self):
"""Strict text comparison of generated module file."""
self.test_toy_tweaked()
toy_module = os.path.join(self.test_installpath, "modules", "all", "toy", "0.0-tweaked")
if get_module_syntax() == "Lua":
toy_module += ".lua"
toy_mod_txt = read_file(toy_module)
if get_module_syntax() == "Lua":
mod_txt_regex_pattern = "\n".join(
[
r"help\(\[\[Toy C program. - Homepage: http://hpcugent.github.com/easybuild\]\]\)",
r"whatis\(\[\[Name: toy\]\]\)",
r"whatis\(\[\[Version: 0.0\]\]\)",
r"whatis\(\[\[Description: Toy C program. - Homepage: http://hpcugent.github.com/easybuild\]\]\)",
r"whatis\(\[\[Homepage: http://hpcugent.github.com/easybuild\]\]\)",
r"",
r'local root = "%s/software/toy/0.0-tweaked"' % self.test_installpath,
r"",
r'conflict\("toy"\)',
r"",
r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib"\)\)',
r'prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib"\)\)',
r'prepend_path\("PATH", pathJoin\(root, "bin"\)\)',
r'setenv\("EBROOTTOY", root\)',
r'setenv\("EBVERSIONTOY", "0.0"\)',
r'setenv\("EBDEVELTOY", pathJoin\(root, "easybuild/toy-0.0-tweaked-easybuild-devel"\)\)',
r"",
r'setenv\("FOO", "bar"\)',
r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)',
r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)',
r'prepend_path\("SOMEPATH", root\)',
r"",
r'if mode\(\) == "load" then',
r' io.stderr:write\("THANKS FOR LOADING ME, I AM toy v0.0"\)',
r"end",
r'io.stderr:write\("oh hai\!"\)',
r"-- Built with EasyBuild version .*$",
]
)
elif get_module_syntax() == "Tcl":
mod_txt_regex_pattern = "\n".join(
[
r"^#%Module",
r"proc ModulesHelp { } {",
r" puts stderr { Toy C program. - Homepage: http://hpcugent.github.com/easybuild",
r" }",
r"}",
r"",
r"module-whatis {Description: Toy C program. - Homepage: http://hpcugent.github.com/easybuild}",
r"",
r"set root %s/software/toy/0.0-tweaked" % self.test_installpath,
r"",
r"conflict toy",
r"",
r"prepend-path LD_LIBRARY_PATH \$root/lib",
r"prepend-path LIBRARY_PATH \$root/lib",
r"prepend-path PATH \$root/bin",
r'setenv EBROOTTOY "\$root"',
r'setenv EBVERSIONTOY "0.0"',
r'setenv EBDEVELTOY "\$root/easybuild/toy-0.0-tweaked-easybuild-devel"',
r"",
r'setenv FOO "bar"',
r"prepend-path SOMEPATH \$root/foo/bar",
r"prepend-path SOMEPATH \$root/baz",
r"prepend-path SOMEPATH \$root",
r"",
r"if { \[ module-info mode load \] } {",
r' puts stderr "THANKS FOR LOADING ME, I AM toy v0.0"',
r"}",
r'puts stderr "oh hai\!"',
r"# Built with EasyBuild version .*$",
]
)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
mod_txt_regex = re.compile(mod_txt_regex_pattern)
msg = "Pattern '%s' matches with: %s" % (mod_txt_regex.pattern, toy_mod_txt)
self.assertTrue(mod_txt_regex.match(toy_mod_txt), msg)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info(
"+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n"
% install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 0 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = "'%s<0.11.4'" % VSC_INSTALL
if sourcepath:
vsc_install_tarball_paths = glob.glob(
os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (
sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# determine PyPI URLs for individual packages
pkg_urls = []
for pkg in EASYBUILD_PACKAGES:
# format of pkg entries in templates: "'<pkg_filename>',"
pkg_filename = templates[pkg][1:-2]
# the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
# which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)
# determine download URL via PyPI's 'simple' API
pkg_simple = None
try:
pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' %
pkg,
timeout=10).read()
except (urllib2.URLError, urllib2.HTTPError) as err:
# failing to figure out the package download URl may be OK when source tarballs are provided
if sourcepath:
info(
"Ignoring failed attempt to determine '%s' download URL since source tarballs are provided"
% pkg)
else:
raise err
if pkg_simple:
pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' %
pkg_filename)
res = pkg_url_part_regex.search(pkg_simple)
if res:
pkg_url_part = res.group(1)
else:
error("Failed to determine PyPI package URL for %s: %s\n" %
(pkg, pkg_simple))
pkg_url = 'https://pypi.python.org/' + pkg_url_part
pkg_urls.append(pkg_url)
templates.update({
'source_urls':
'\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]),
'sources':
"%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s"
% templates,
'pythonpath':
distribute_egg_dir,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
handle.write(ebfile_txt)
handle.close()
debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [
x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0
]
os.environ['PATH'] = os.pathsep.join(
[os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
if print_debug:
os.environ['EASYBUILD_DEBUG'] = '1'
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'),
build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error(
"EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug"
% eb_mod_path)
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M))
self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# check for behavior when a string value is used as dict value by make_module_req_guesses
eb.make_module_req_guess = lambda: {'PATH': 'bin'}
txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.match(r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$', txt, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# check for correct behaviour if empty string is specified as one of the values
# prepend-path statements should be included for both the 'bin' subdir and the install root
eb.make_module_req_guess = lambda: {'PATH': ['bin', '']}
txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root/bin\n", txt, re.M))
self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root\n", txt, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.search(r'\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n', txt, re.M))
self.assertTrue(re.search(r'\nprepend_path\("PATH", root\)\n', txt, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
deps = [('GCC', '4.6.4')]
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = [('GCC', '4.6.4'), ('toy', '0.0-deps')]",
"builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
# hidden deps must be included in list of (build)deps
"hiddendependencies = [('toy', '0.0-deps'), ('OpenMPI', '1.6.4-GCC-4.6.4')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
test_dir = os.path.dirname(os.path.abspath(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
# test if module is generated correctly
self.writeEC()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
modpath = os.path.join(eb.make_module_step(), name, version)
if get_module_syntax() == 'Lua':
modpath += '.lua'
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
txt = read_file(modpath)
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(
re.search(r"^set\s+root\s+%s$" % eb.installdir, txt, re.M))
ebroot_regex = re.compile(
r'^setenv\s+EBROOT%s\s+"\$root"\s*$' % name.upper(), re.M)
self.assertTrue(ebroot_regex.search(txt),
"%s in %s" % (ebroot_regex.pattern, txt))
self.assertTrue(
re.search(
r'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version),
txt, re.M))
elif get_module_syntax() == 'Lua':
ebroot_regex = re.compile(
r'^setenv\("EBROOT%s", root\)$' % name.upper(), re.M)
self.assertTrue(ebroot_regex.search(txt),
"%s in %s" % (ebroot_regex.pattern, txt))
self.assertTrue(
re.search(
r'^setenv\("EBVERSION%s", "%s"\)$' %
(name.upper(), version), txt, re.M))
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
for (key, val) in modextravars.items():
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^setenv\s+%s\s+"%s"$' % (key, val), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^setenv\("%s", "%s"\)$' % (key, val),
re.M)
else:
self.assertTrue(
False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt),
"Pattern %s found in %s" % (regex.pattern, txt))
for (key, val) in modextrapaths.items():
if get_module_syntax() == 'Tcl':
regex = re.compile(
r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(
r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' %
(key, val), re.M)
else:
self.assertTrue(
False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt),
"Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('GCC', '4.6.4')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(
r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(
r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M)
else:
self.assertTrue(
False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt),
"Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('toy', '0.0-deps')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver),
re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver),
re.M)
else:
self.assertTrue(
False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt),
"Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('OpenMPI', '1.6.4-GCC-4.6.4')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(
r'^\s*module load %s/.?%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^\s*load\("%s/.?%s"\)$' % (name, ver),
re.M)
else:
self.assertTrue(
False, "Unknown module syntax: %s" % get_module_syntax())
self.assertFalse(
regex.search(txt),
"Pattern '%s' *not* found in %s" % (regex.pattern, txt))
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
deps = [('GCC', '4.6.4')]
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = [('GCC', '4.6.4'), ('toy', '0.0-deps')]",
"builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
# hidden deps must be included in list of (build)deps
"hiddendependencies = [('toy', '0.0-deps'), ('OpenMPI', '1.6.4-GCC-4.6.4')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
test_dir = os.path.dirname(os.path.abspath(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
# test if module is generated correctly
self.writeEC()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
modpath = os.path.join(eb.make_module_step(), name, version)
if get_module_syntax() == 'Lua':
modpath += '.lua'
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
txt = read_file(modpath)
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(re.search(r"^set\s+root\s+%s$" % eb.installdir, txt, re.M))
ebroot_regex = re.compile(r'^setenv\s+EBROOT%s\s+"\$root"\s*$' % name.upper(), re.M)
self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt))
self.assertTrue(re.search(r'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))
elif get_module_syntax() == 'Lua':
ebroot_regex = re.compile(r'^setenv\("EBROOT%s", root\)$' % name.upper(), re.M)
self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt))
self.assertTrue(re.search(r'^setenv\("EBVERSION%s", "%s"\)$' % (name.upper(), version), txt, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
for (key, val) in modextravars.items():
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^setenv\s+%s\s+"%s"$' % (key, val), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^setenv\("%s", "%s"\)$' % (key, val), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, val) in modextrapaths.items():
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('GCC', '4.6.4')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('toy', '0.0-deps')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('OpenMPI', '1.6.4-GCC-4.6.4')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^\s*module load %s/.?%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^\s*load\("%s/.?%s"\)$' % (name, ver), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertFalse(regex.search(txt), "Pattern '%s' *not* found in %s" % (regex.pattern, txt))
def test_make_module_dep(self):
"""Test for make_module_dep"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
if get_module_syntax() == 'Tcl':
tc_load = '\n'.join([
"if { ![ is-loaded gompi/1.1.0-no-OFED ] } {",
" module load gompi/1.1.0-no-OFED",
"}",
])
fftw_load = '\n'.join([
"if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
" module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
"}",
])
lapack_load = '\n'.join([
"if { ![ is-loaded LAPACK/3.4.0-gompi-1.1.0-no-OFED ] } {",
" module load LAPACK/3.4.0-gompi-1.1.0-no-OFED",
"}",
])
elif get_module_syntax() == 'Lua':
tc_load = '\n'.join([
'if not isloaded("gompi/1.1.0-no-OFED") then',
' load("gompi/1.1.0-no-OFED")',
'end',
])
fftw_load = '\n'.join([
'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
' load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
'end',
])
lapack_load = '\n'.join([
'if not isloaded("LAPACK/3.4.0-gompi-1.1.0-no-OFED") then',
' load("LAPACK/3.4.0-gompi-1.1.0-no-OFED")',
'end',
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
self.assertEqual(eb.make_module_dep().strip(), expected)
# provide swap info for FFTW to trigger an extra 'unload FFTW'
unload_info = {
'FFTW/3.3.1-gompi-1.1.0-no-OFED': 'FFTW',
}
if get_module_syntax() == 'Tcl':
fftw_load = '\n'.join([
"if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
" module unload FFTW",
" module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
"}",
])
elif get_module_syntax() == 'Lua':
fftw_load = '\n'.join([
'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
' unload("FFTW")',
' load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
'end',
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
self.assertEqual(eb.make_module_dep(unload_info=unload_info).strip(), expected)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 1 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = 'vsc-install'
if sourcepath:
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
templates.update({
'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
handle.close()
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def test_make_module_dep(self):
"""Test for make_module_dep"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
if get_module_syntax() == 'Tcl':
tc_load = '\n'.join([
"if { ![ is-loaded gompi/1.1.0-no-OFED ] } {",
" module load gompi/1.1.0-no-OFED",
"}",
])
fftw_load = '\n'.join([
"if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
" module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
"}",
])
lapack_load = '\n'.join([
"if { ![ is-loaded LAPACK/3.4.0-gompi-1.1.0-no-OFED ] } {",
" module load LAPACK/3.4.0-gompi-1.1.0-no-OFED",
"}",
])
elif get_module_syntax() == 'Lua':
tc_load = '\n'.join([
'if not isloaded("gompi/1.1.0-no-OFED") then',
' load("gompi/1.1.0-no-OFED")',
'end',
])
fftw_load = '\n'.join([
'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
' load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
'end',
])
lapack_load = '\n'.join([
'if not isloaded("LAPACK/3.4.0-gompi-1.1.0-no-OFED") then',
' load("LAPACK/3.4.0-gompi-1.1.0-no-OFED")',
'end',
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
self.assertEqual(eb.make_module_dep().strip(), expected)
# provide swap info for FFTW to trigger an extra 'unload FFTW'
unload_info = {
'FFTW/3.3.1-gompi-1.1.0-no-OFED': 'FFTW',
}
if get_module_syntax() == 'Tcl':
fftw_load = '\n'.join([
"if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
" module unload FFTW",
" module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
"}",
])
elif get_module_syntax() == 'Lua':
fftw_load = '\n'.join([
'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
' unload("FFTW")',
' load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
'end',
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
self.assertEqual(eb.make_module_dep(unload_info=unload_info).strip(), expected)
def test_toy_permissions(self):
"""Test toy build with custom umask settings."""
toy_ec_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "toy-0.0.eb")
args = [
"--sourcepath=%s" % self.test_sourcepath,
"--buildpath=%s" % self.test_buildpath,
"--installpath=%s" % self.test_installpath,
"--debug",
"--unittest-file=%s" % self.logfile,
"--force",
]
# set umask hard to verify default reliably
orig_umask = os.umask(0022)
# test specifying a non-existing group
allargs = [toy_ec_file] + args + ["--group=thisgroupdoesnotexist"]
outtxt, err = self.eb_main(allargs, logfile=self.dummylogfn, do_build=True, return_error=True)
err_regex = re.compile("Failed to get group ID .* group does not exist")
self.assertTrue(err_regex.search(outtxt), "Pattern '%s' found in '%s'" % (err_regex.pattern, outtxt))
# determine current group name (at least we can use that)
gid = os.getgid()
curr_grp = grp.getgrgid(gid).gr_name
for umask, cfg_group, ec_group, dir_perms, fil_perms, bin_perms in [
(None, None, None, 0755, 0644, 0755), # default: inherit session umask
(None, None, curr_grp, 0750, 0640, 0750), # default umask, but with specified group in ec
(None, curr_grp, None, 0750, 0640, 0750), # default umask, but with specified group in cfg
(None, "notagrp", curr_grp, 0750, 0640, 0750), # default umask, but with specified group in both cfg and ec
("000", None, None, 0777, 0666, 0777), # stupid empty umask
("032", None, None, 0745, 0644, 0745), # no write/execute for group, no write for other
("030", None, curr_grp, 0740, 0640, 0740), # no write for group, with specified group
("077", None, None, 0700, 0600, 0700), # no access for other/group
]:
# empty the install directory, to ensure any created directories adher to the permissions
shutil.rmtree(self.test_installpath)
if cfg_group is None and ec_group is None:
allargs = [toy_ec_file]
elif ec_group is not None:
shutil.copy2(toy_ec_file, self.test_buildpath)
tmp_ec_file = os.path.join(self.test_buildpath, os.path.basename(toy_ec_file))
write_file(tmp_ec_file, "\ngroup = '%s'" % ec_group, append=True)
allargs = [tmp_ec_file]
allargs.extend(args)
if umask is not None:
allargs.append("--umask=%s" % umask)
if cfg_group is not None:
allargs.append("--group=%s" % cfg_group)
outtxt = self.eb_main(allargs, logfile=self.dummylogfn, do_build=True, verbose=True)
# verify that installation was correct
self.check_toy(self.test_installpath, outtxt)
# group specified in easyconfig overrules configured group
group = cfg_group
if ec_group is not None:
group = ec_group
# verify permissions
paths_perms = [
# no write permissions for group/other, regardless of umask
(("software", "toy", "0.0"), dir_perms & ~0022),
(("software", "toy", "0.0", "bin"), dir_perms & ~0022),
(("software", "toy", "0.0", "bin", "toy"), bin_perms & ~0022),
]
# only software subdirs are chmod'ed for 'protected' installs, so don't check those if a group is specified
if group is None:
paths_perms.extend(
[
(("software",), dir_perms),
(("software", "toy"), dir_perms),
(("software", "toy", "0.0", "easybuild", "*.log"), fil_perms),
(("modules",), dir_perms),
(("modules", "all"), dir_perms),
(("modules", "all", "toy"), dir_perms),
]
)
if get_module_syntax() == "Tcl":
paths_perms.append((("modules", "all", "toy", "0.0"), fil_perms))
elif get_module_syntax() == "Lua":
paths_perms.append((("modules", "all", "toy", "0.0.lua"), fil_perms))
for path, correct_perms in paths_perms:
fullpath = glob.glob(os.path.join(self.test_installpath, *path))[0]
perms = os.stat(fullpath).st_mode & 0777
msg = "Path %s has %s permissions: %s" % (fullpath, oct(correct_perms), oct(perms))
self.assertEqual(perms, correct_perms, msg)
if group is not None:
path_gid = os.stat(fullpath).st_gid
self.assertEqual(path_gid, grp.getgrnam(group).gr_gid)
def test_toy_hierarchical(self):
"""Test toy build under example hierarchical module naming scheme."""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), "easyconfigs")
self.setup_hierarchical_modules()
mod_prefix = os.path.join(self.test_installpath, "modules", "all")
args = [
os.path.join(test_easyconfigs, "toy-0.0.eb"),
"--sourcepath=%s" % self.test_sourcepath,
"--buildpath=%s" % self.test_buildpath,
"--installpath=%s" % self.test_installpath,
"--debug",
"--unittest-file=%s" % self.logfile,
"--force",
"--robot=%s" % test_easyconfigs,
"--module-naming-scheme=HierarchicalMNS",
]
# test module paths/contents with gompi build
extra_args = ["--try-toolchain=goolf,1.4.10"]
self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, "MPI", "GCC", "4.7.2", "OpenMPI", "1.6.4", "toy", "0.0")
if get_module_syntax() == "Lua":
toy_module_path += ".lua"
self.assertTrue(os.path.exists(toy_module_path))
# check that toolchain load is expanded to loads for toolchain dependencies,
# except for the ones that extend $MODULEPATH to make the toy module available
if get_module_syntax() == "Tcl":
load_regex_template = "load %s"
elif get_module_syntax() == "Lua":
load_regex_template = r'load\("%s/.*"\)'
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
modtxt = read_file(toy_module_path)
for dep in ["goolf", "GCC", "OpenMPI"]:
load_regex = re.compile(load_regex_template % dep)
self.assertFalse(load_regex.search(modtxt), "Pattern '%s' not found in %s" % (load_regex.pattern, modtxt))
for dep in ["OpenBLAS", "FFTW", "ScaLAPACK"]:
load_regex = re.compile(load_regex_template % dep)
self.assertTrue(load_regex.search(modtxt), "Pattern '%s' found in %s" % (load_regex.pattern, modtxt))
os.remove(toy_module_path)
# test module path with GCC/4.7.2 build
extra_args = ["--try-toolchain=GCC,4.7.2"]
self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, "Compiler", "GCC", "4.7.2", "toy", "0.0")
if get_module_syntax() == "Lua":
toy_module_path += ".lua"
self.assertTrue(os.path.exists(toy_module_path))
# no dependencies or toolchain => no module load statements in module file
modtxt = read_file(toy_module_path)
self.assertFalse(re.search("module load", modtxt))
os.remove(toy_module_path)
# test module path with GCC/4.7.2 build, pretend to be an MPI lib by setting moduleclass
extra_args = ["--try-toolchain=GCC,4.7.2", "--try-amend=moduleclass=mpi"]
self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, "Compiler", "GCC", "4.7.2", "toy", "0.0")
if get_module_syntax() == "Lua":
toy_module_path += ".lua"
self.assertTrue(os.path.exists(toy_module_path))
# 'module use' statements to extend $MODULEPATH are present
modtxt = read_file(toy_module_path)
modpath_extension = os.path.join(mod_prefix, "MPI", "GCC", "4.7.2", "toy", "0.0")
if get_module_syntax() == "Tcl":
self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M))
elif get_module_syntax() == "Lua":
fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension)
regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M)
self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
os.remove(toy_module_path)
# ... unless they shouldn't be
extra_args.append("--try-amend=include_modpath_extensions=") # pass empty string as equivalent to False
self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
modtxt = read_file(toy_module_path)
modpath_extension = os.path.join(mod_prefix, "MPI", "GCC", "4.7.2", "toy", "0.0")
if get_module_syntax() == "Tcl":
self.assertFalse(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M))
elif get_module_syntax() == "Lua":
fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension)
regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M)
self.assertFalse(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
os.remove(toy_module_path)
# test module path with dummy/dummy build
extra_args = ["--try-toolchain=dummy,dummy"]
self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, "Core", "toy", "0.0")
if get_module_syntax() == "Lua":
toy_module_path += ".lua"
self.assertTrue(os.path.exists(toy_module_path))
# no dependencies or toolchain => no module load statements in module file
modtxt = read_file(toy_module_path)
self.assertFalse(re.search("module load", modtxt))
os.remove(toy_module_path)
# test module path with dummy/dummy build, pretend to be a compiler by setting moduleclass
extra_args = ["--try-toolchain=dummy,dummy", "--try-amend=moduleclass=compiler"]
self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True)
# make sure module file is installed in correct path
toy_module_path = os.path.join(mod_prefix, "Core", "toy", "0.0")
if get_module_syntax() == "Lua":
toy_module_path += ".lua"
self.assertTrue(os.path.exists(toy_module_path))
# no dependencies or toolchain => no module load statements in module file
modtxt = read_file(toy_module_path)
modpath_extension = os.path.join(mod_prefix, "Compiler", "toy", "0.0")
if get_module_syntax() == "Tcl":
self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M))
elif get_module_syntax() == "Lua":
fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension)
regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M)
self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
os.remove(toy_module_path)
# building a toolchain module should also work
args[0] = os.path.join(test_easyconfigs, "gompi-1.4.10.eb")
modules_tool().purge()
self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=False)
gompi_module_path = os.path.join(mod_prefix, "Core", "gompi", "1.4.10")
if get_module_syntax() == "Lua":
gompi_module_path += ".lua"
self.assertTrue(os.path.exists(gompi_module_path))
def test_make_module_extend_modpath(self):
"""Test for make_module_extend_modpath"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'moduleclass = "compiler"',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
self.assertEqual(eb.make_module_extend_modpath(), '')
usermodsdir = 'my/own/modules'
modclasses = ['compiler', 'tools']
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
build_options = {
'subdir_user_modules': usermodsdir,
'valid_module_classes': modclasses,
}
init_config(build_options=build_options)
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
txt = eb.make_module_extend_modpath()
if get_module_syntax() == 'Tcl':
regexs = [
r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c
for c in modclasses
]
home = r'\$env\(HOME\)'
regexs.extend([
# extension for user modules is guarded
r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$'
% (home, usermodsdir),
# no per-moduleclass extension for user modules
r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' %
(home, usermodsdir),
])
elif get_module_syntax() == 'Lua':
regexs = [
r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$'
% c for c in modclasses
]
home = r'os.getenv\("HOME"\)'
regexs.extend([
# extension for user modules is guarded
r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' %
(home, usermodsdir),
# no per-moduleclass extension for user modules
r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)'
% (home, usermodsdir),
])
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
for regex in regexs:
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt),
"Pattern '%s' found in: %s" % (regex.pattern, txt))
