class ModuleGeneratorTest(TestCase):
""" testcase for ModuleGenerator """
def setUp(self):
""" initialize ModuleGenerator with test Application """
self.modgen = ModuleGenerator(Application('easybuild/test/easyconfigs/gzip-1.4.eb'))
self.modgen.app.installdir = "/tmp"
def runTest(self):
""" since we set the installdir above, we can predict the output """
expected = """#%Module
proc ModulesHelp { } {
puts stderr { gzip (GNU zip) is a popular data compression program as a replacement for compress - Homepage: http://www.gzip.org/
}
}
module-whatis {gzip (GNU zip) is a popular data compression program as a replacement for compress - Homepage: http://www.gzip.org/}
set root /tmp
conflict gzip
"""
desc = self.modgen.getDescription()
self.assertEqual(desc, expected)
# test loadModule
expected = """
if { ![is-loaded name/version] } {
module load name/version
}
"""
self.assertEqual(expected, self.modgen.loadModule("name", "version"))
# test unloadModule
expected = """
if { ![is-loaded name/version] } {
if { [is-loaded name] } {
module unload name
}
}
"""
self.assertEqual(expected, self.modgen.unloadModule("name", "version"))
# test prependPaths
expected = """prepend-path key $root/path1
prepend-path key $root/path2
"""
self.assertEqual(expected, self.modgen.prependPaths("key", ["path1", "path2"]))
# test setEnvironment
self.assertEqual("setenv\tkey\t\tvalue\n", self.modgen.setEnvironment("key", "value"))
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(
os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(
prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
class ModuleGeneratorTest(EnhancedTestCase):
""" testcase for ModuleGenerator """
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def tearDown(self):
"""cleanup"""
super(ModuleGeneratorTest, self).tearDown()
os.remove(self.eb.logfile)
shutil.rmtree(self.modgen.app.installdir)
def test_descr(self):
"""Test generation of module description (which includes '#%Module' header)."""
gzip_txt = "gzip (GNU zip) is a popular data compression program as a replacement for compress "
gzip_txt += "- Homepage: http://www.gzip.org/"
expected = '\n'.join([
"#%Module",
"",
"proc ModulesHelp { } {",
" puts stderr { %s" % gzip_txt,
" }",
"}",
"",
"module-whatis {Description: %s}" % gzip_txt,
"",
"set root %s" % self.modgen.app.installdir,
"",
"conflict gzip",
"",
])
desc = self.modgen.get_description()
self.assertEqual(desc, expected)
def test_load(self):
"""Test load part in generated module file."""
expected = [
"",
"if { ![is-loaded mod_name] } {",
" module load mod_name",
"}",
"",
]
self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name"))
# with recursive unloading: no if is-loaded guard
expected = [
"",
"module load mod_name",
"",
]
self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name", recursive_unload=True))
def test_unload(self):
"""Test unload part in generated module file."""
expected = '\n'.join([
"",
"if { [is-loaded mod_name] } {",
" module unload mod_name",
"}",
"",
])
self.assertEqual(expected, self.modgen.unload_module("mod_name"))
def test_prepend_paths(self):
"""Test generating prepend-paths statements."""
# test prepend_paths
expected = ''.join([
"prepend-path\tkey\t\t$root/path1\n",
"prepend-path\tkey\t\t$root/path2\n",
])
self.assertEqual(expected, self.modgen.prepend_paths("key", ["path1", "path2"]))
expected = "prepend-path\tbar\t\t$root/foo\n"
self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo"))
self.assertEqual("prepend-path\tkey\t\t/abs/path\n", self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True))
self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to prepend_paths " \
"which only expects relative paths." % self.modgen.app.installdir,
self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
def test_use(self):
"""Test generating module use statements."""
expected = '\n'.join([
"module use /some/path",
"module use /foo/bar/baz",
])
self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected)
def test_env(self):
"""Test setting of environment variables."""
# test set_environment
self.assertEqual('setenv\tkey\t\t"value"\n', self.modgen.set_environment("key", "value"))
self.assertEqual("setenv\tkey\t\t'va\"lue'\n", self.modgen.set_environment("key", 'va"lue'))
self.assertEqual('setenv\tkey\t\t"va\'lue"\n', self.modgen.set_environment("key", "va'lue"))
self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_environment("key", """va"l'ue"""))
def test_alias(self):
"""Test setting of alias in modulefiles."""
# test set_alias
self.assertEqual('set-alias\tkey\t\t"value"\n', self.modgen.set_alias("key", "value"))
self.assertEqual("set-alias\tkey\t\t'va\"lue'\n", self.modgen.set_alias("key", 'va"lue'))
self.assertEqual('set-alias\tkey\t\t"va\'lue"\n', self.modgen.set_alias("key", "va'lue"))
self.assertEqual('set-alias\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_alias("key", """va"l'ue"""))
def test_load_msg(self):
"""Test including a load message in the module file."""
tcl_load_msg = '\nif [ module-info mode load ] {\n puts stderr "test"\n}\n'
self.assertEqual(tcl_load_msg, self.modgen.msg_on_load('test'))
def test_tcl_footer(self):
"""Test including a Tcl footer."""
tcltxt = 'puts stderr "foo"'
self.assertEqual(tcltxt, self.modgen.add_tcl_footer(tcltxt))
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files]
ec_files = [fil for fil in ec_files if not "v2.0" in fil] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(ec2mod_map[ec_fn], ActiveMNS().det_full_module_name(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar', ActiveMNS().det_full_module_name(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
mns_mods = ['broken_module_naming_scheme', 'test_module_naming_scheme', 'test_module_naming_scheme_more']
for test_mns_mod in mns_mods:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
__import__(mns_path, globals(), locals(), [''])
init_config(build_options=build_options)
# verify that key errors in module naming scheme are reported properly
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'BrokenModuleNamingScheme'
init_config(build_options=build_options)
err_pattern = 'nosucheasyconfigparameteravailable'
self.assertErrorRegex(KeyError, err_pattern, EasyConfig, os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(ec.toolchain.det_short_module_name(), 'goolf/1.4.10')
# test module naming scheme using all available easyconfig parameters
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeMore'
init_config(build_options=build_options)
# note: these checksums will change if another easyconfig parameter is added
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/9e9ab5a1e978f0843b5aedb63ac4f14c51efb859',
'gzip-1.4.eb': 'gzip/8805ec3152d2a4a08b6c06d740c23abe1a4d059f',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/863557cc81811f8c3f4426a4b45aa269fa54130b',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/b63c2b8cc518905473ccda023100b2d3cff52d55',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/3d49f0e112708a95f79ed38b91b506366c0299ab',
'toy-0.0.eb': 'toy/44a206d9e8c14130cc9f79e061468303c6e91b53',
'toy-0.0-multiple.eb': 'toy/44a206d9e8c14130cc9f79e061468303c6e91b53',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
]:
# determine full module name
self.assertEqual(ActiveMNS().det_full_module_name(dep_spec), ec2mod_map[dep_ec])
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(ec.toolchain.det_short_module_name(), 'goolf/b7515d0efd346970f55e7aa8522e239a70007021')
# restore default module naming scheme, and retest
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def test_mod_name_validation(self):
"""Test module naming validation."""
# module name must be a string
self.assertTrue(not is_valid_module_name(('foo', 'bar')))
self.assertTrue(not is_valid_module_name(['foo', 'bar']))
self.assertTrue(not is_valid_module_name(123))
# module name must be relative
self.assertTrue(not is_valid_module_name('/foo/bar'))
# module name must only contain valid characters
self.assertTrue(not is_valid_module_name('foo\x0bbar'))
self.assertTrue(not is_valid_module_name('foo\x0cbar'))
self.assertTrue(not is_valid_module_name('foo\rbar'))
self.assertTrue(not is_valid_module_name('foo\0bar'))
# valid module name must be accepted
self.assertTrue(is_valid_module_name('gzip/goolf-1.4.10-suffix'))
self.assertTrue(is_valid_module_name('GCC/4.7.2'))
self.assertTrue(is_valid_module_name('foo-bar/1.2.3'))
self.assertTrue(is_valid_module_name('ictce'))
def test_hierarchical_mns(self):
"""Test hierarchical module naming scheme."""
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
ec = EasyConfig(os.path.join(ecs_dir, 'GCC-4.7.2.eb'))
self.assertEqual(ActiveMNS().det_full_module_name(ec), 'Core/GCC/4.7.2')
self.assertEqual(ActiveMNS().det_short_module_name(ec), 'GCC/4.7.2')
self.assertEqual(ActiveMNS().det_module_subdir(ec), 'Core')
self.assertEqual(ActiveMNS().det_modpath_extensions(ec), ['Compiler/GCC/4.7.2'])
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core'])
ec = EasyConfig(os.path.join(ecs_dir, 'OpenMPI-1.6.4-GCC-4.7.2.eb'))
self.assertEqual(ActiveMNS().det_full_module_name(ec), 'Compiler/GCC/4.7.2/OpenMPI/1.6.4')
self.assertEqual(ActiveMNS().det_short_module_name(ec), 'OpenMPI/1.6.4')
self.assertEqual(ActiveMNS().det_module_subdir(ec), 'Compiler/GCC/4.7.2')
self.assertEqual(ActiveMNS().det_modpath_extensions(ec), ['MPI/GCC/4.7.2/OpenMPI/1.6.4'])
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core'])
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(ActiveMNS().det_full_module_name(ec), 'MPI/GCC/4.7.2/OpenMPI/1.6.4/gzip/1.5')
self.assertEqual(ActiveMNS().det_short_module_name(ec), 'gzip/1.5')
self.assertEqual(ActiveMNS().det_module_subdir(ec), 'MPI/GCC/4.7.2/OpenMPI/1.6.4')
self.assertEqual(ActiveMNS().det_modpath_extensions(ec), [])
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core'])
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
class ModuleGeneratorTest(EnhancedTestCase):
""" testcase for ModuleGenerator """
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(
os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(
prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def tearDown(self):
"""cleanup"""
super(ModuleGeneratorTest, self).tearDown()
os.remove(self.eb.logfile)
shutil.rmtree(self.modgen.app.installdir)
def test_descr(self):
"""Test generation of module description (which includes '#%Module' header)."""
gzip_txt = "gzip (GNU zip) is a popular data compression program as a replacement for compress "
gzip_txt += "- Homepage: http://www.gzip.org/"
expected = '\n'.join([
"#%Module",
"",
"proc ModulesHelp { } {",
" puts stderr { %s" % gzip_txt,
" }",
"}",
"",
"module-whatis {Description: %s}" % gzip_txt,
"",
"set root %s" % self.modgen.app.installdir,
"",
"conflict gzip",
"",
])
desc = self.modgen.get_description()
self.assertEqual(desc, expected)
def test_load(self):
"""Test load part in generated module file."""
expected = [
"",
"if { ![is-loaded mod_name] } {",
" module load mod_name",
"}",
"",
]
self.assertEqual('\n'.join(expected),
self.modgen.load_module("mod_name"))
# with recursive unloading: no if is-loaded guard
init_config(build_options={'recursive_mod_unload': True})
expected = [
"",
"module load mod_name",
"",
]
self.assertEqual('\n'.join(expected),
self.modgen.load_module("mod_name"))
def test_unload(self):
"""Test unload part in generated module file."""
expected = '\n'.join([
"",
"if { [is-loaded mod_name] } {",
" module unload mod_name",
"}",
"",
])
self.assertEqual(expected, self.modgen.unload_module("mod_name"))
def test_prepend_paths(self):
"""Test generating prepend-paths statements."""
# test prepend_paths
expected = ''.join([
"prepend-path\tkey\t\t$root/path1\n",
"prepend-path\tkey\t\t$root/path2\n",
])
self.assertEqual(expected,
self.modgen.prepend_paths("key", ["path1", "path2"]))
expected = "prepend-path\tbar\t\t$root/foo\n"
self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo"))
self.assertEqual(
"prepend-path\tkey\t\t/abs/path\n",
self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True))
self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to prepend_paths " \
"which only expects relative paths." % self.modgen.app.installdir,
self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
def test_use(self):
"""Test generating module use statements."""
expected = '\n'.join([
"module use /some/path",
"module use /foo/bar/baz",
])
self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]),
expected)
def test_env(self):
"""Test setting of environment variables."""
# test set_environment
self.assertEqual('setenv\tkey\t\t"value"\n',
self.modgen.set_environment("key", "value"))
self.assertEqual("setenv\tkey\t\t'va\"lue'\n",
self.modgen.set_environment("key", 'va"lue'))
self.assertEqual('setenv\tkey\t\t"va\'lue"\n',
self.modgen.set_environment("key", "va'lue"))
self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n',
self.modgen.set_environment("key", """va"l'ue"""))
def test_alias(self):
"""Test setting of alias in modulefiles."""
# test set_alias
self.assertEqual('set-alias\tkey\t\t"value"\n',
self.modgen.set_alias("key", "value"))
self.assertEqual("set-alias\tkey\t\t'va\"lue'\n",
self.modgen.set_alias("key", 'va"lue'))
self.assertEqual('set-alias\tkey\t\t"va\'lue"\n',
self.modgen.set_alias("key", "va'lue"))
self.assertEqual('set-alias\tkey\t\t"""va"l\'ue"""\n',
self.modgen.set_alias("key", """va"l'ue"""))
def test_load_msg(self):
"""Test including a load message in the module file."""
tcl_load_msg = '\n'.join([
'',
"if [ module-info mode load ] {",
" puts stderr \"test \\$test \\$test",
"test \\$foo \\$bar\"",
"}",
'',
])
self.assertEqual(
tcl_load_msg,
self.modgen.msg_on_load('test $test \\$test\ntest $foo \\$bar'))
def test_tcl_footer(self):
"""Test including a Tcl footer."""
tcltxt = 'puts stderr "foo"'
self.assertEqual(tcltxt, self.modgen.add_tcl_footer(tcltxt))
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [
os.path.join(subdir, fil)
for (subdir, _, files) in os.walk(ecs_dir) for fil in files
]
ec_files = [fil for fil in ec_files if not "v2.0" in fil
] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(
ec2mod_map[ec_fn],
ActiveMNS().det_full_module_name(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar',
ActiveMNS().det_full_module_name(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
mns_mods = [
'broken_module_naming_scheme', 'test_module_naming_scheme',
'test_module_naming_scheme_more'
]
for test_mns_mod in mns_mods:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
__import__(mns_path, globals(), locals(), [''])
init_config(build_options=build_options)
# verify that key errors in module naming scheme are reported properly
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = 'BrokenModuleNamingScheme'
init_config(build_options=build_options)
err_pattern = 'nosucheasyconfigparameteravailable'
self.assertErrorRegex(
EasyBuildError, err_pattern, EasyConfig,
os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(ec.toolchain.det_short_module_name(), 'goolf/1.4.10')
# test module naming scheme using all available easyconfig parameters
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeMore'
init_config(build_options=build_options)
# note: these checksums will change if another easyconfig parameter is added
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/9e9ab5a1e978f0843b5aedb63ac4f14c51efb859',
'gzip-1.4.eb': 'gzip/8805ec3152d2a4a08b6c06d740c23abe1a4d059f',
'gzip-1.4-GCC-4.6.3.eb':
'gzip/863557cc81811f8c3f4426a4b45aa269fa54130b',
'gzip-1.5-goolf-1.4.10.eb':
'gzip/b63c2b8cc518905473ccda023100b2d3cff52d55',
'gzip-1.5-ictce-4.1.13.eb':
'gzip/3d49f0e112708a95f79ed38b91b506366c0299ab',
'toy-0.0.eb': 'toy/44a206d9e8c14130cc9f79e061468303c6e91b53',
'toy-0.0-multiple.eb':
'toy/44a206d9e8c14130cc9f79e061468303c6e91b53',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
ec2mod_map[
'gzip-1.5-goolf-1.4.10.eb'] = 'gzip/.b63c2b8cc518905473ccda023100b2d3cff52d55'
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'hidden': False,
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {
'name': 'goolf',
'version': '1.4.10'
},
'hidden': True,
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'hidden': False,
}),
]:
# determine full module name
self.assertEqual(ActiveMNS().det_full_module_name(dep_spec),
ec2mod_map[dep_ec])
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'),
hidden=True)
self.assertEqual(ec.full_mod_name,
ec2mod_map['gzip-1.5-goolf-1.4.10.eb'])
self.assertEqual(ec.toolchain.det_short_module_name(),
'goolf/b7515d0efd346970f55e7aa8522e239a70007021')
# restore default module naming scheme, and retest
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def test_mod_name_validation(self):
"""Test module naming validation."""
# module name must be a string
self.assertTrue(not is_valid_module_name(('foo', 'bar')))
self.assertTrue(not is_valid_module_name(['foo', 'bar']))
self.assertTrue(not is_valid_module_name(123))
# module name must be relative
self.assertTrue(not is_valid_module_name('/foo/bar'))
# module name must only contain valid characters
self.assertTrue(not is_valid_module_name('foo\x0bbar'))
self.assertTrue(not is_valid_module_name('foo\x0cbar'))
self.assertTrue(not is_valid_module_name('foo\rbar'))
self.assertTrue(not is_valid_module_name('foo\0bar'))
# valid module name must be accepted
self.assertTrue(is_valid_module_name('gzip/goolf-1.4.10-suffix'))
self.assertTrue(is_valid_module_name('GCC/4.7.2'))
self.assertTrue(is_valid_module_name('foo-bar/1.2.3'))
self.assertTrue(is_valid_module_name('ictce'))
def test_is_short_modname_for(self):
"""Test is_short_modname_for method of module naming schemes."""
test_cases = [
('GCC/4.7.2', 'GCC', True),
('gzip/1.6-gompi-1.4.10', 'gzip', True),
('OpenMPI/1.6.4-GCC-4.7.2-no-OFED', 'OpenMPI', True),
('BLACS/1.1-gompi-1.1.0-no-OFED', 'BLACS', True),
('ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1',
'ScaLAPACK', True),
('netCDF-C++/4.2-goolf-1.4.10', 'netCDF-C++', True),
('gcc/4.7.2', 'GCC', False),
('ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1',
'BLACS', False),
('apps/blacs/1.1', 'BLACS', False),
('lib/math/BLACS-stable/1.1', 'BLACS', False),
]
for modname, softname, res in test_cases:
if res:
errormsg = "%s is recognised as a module for '%s'" % (modname,
softname)
else:
errormsg = "%s is NOT recognised as a module for '%s'" % (
modname, softname)
self.assertEqual(
ActiveMNS().is_short_modname_for(modname, softname), res,
errormsg)
def test_hierarchical_mns(self):
"""Test hierarchical module naming scheme."""
moduleclasses = [
'base', 'compiler', 'mpi', 'numlib', 'system', 'toolchain'
]
ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
'valid_module_classes': moduleclasses,
}
def test_ec(ecfile, short_modname, mod_subdir, modpath_exts,
init_modpaths):
"""Test whether active module naming scheme returns expected values."""
ec = EasyConfig(os.path.join(ecs_dir, ecfile))
self.assertEqual(ActiveMNS().det_full_module_name(ec),
os.path.join(mod_subdir, short_modname))
self.assertEqual(ActiveMNS().det_short_module_name(ec),
short_modname)
self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir)
self.assertEqual(ActiveMNS().det_modpath_extensions(ec),
modpath_exts)
self.assertEqual(ActiveMNS().det_init_modulepaths(ec),
init_modpaths)
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
# format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions, init_modpaths)
iccver = '2013.5.192-GCC-4.8.3'
impi_ec = 'impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb'
imkl_ec = 'imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb'
test_ecs = {
'GCC-4.7.2.eb':
('GCC/4.7.2', 'Core', ['Compiler/GCC/4.7.2'], ['Core']),
'OpenMPI-1.6.4-GCC-4.7.2.eb':
('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2',
['MPI/GCC/4.7.2/OpenMPI/1.6.4'], ['Core']),
'gzip-1.5-goolf-1.4.10.eb':
('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4', [], ['Core']),
'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core', [], ['Core']),
'icc-2013.5.192-GCC-4.8.3.eb':
('icc/%s' % iccver, 'Core', ['Compiler/intel/%s' % iccver],
['Core']),
'ifort-2013.3.163.eb': ('ifort/2013.3.163', 'Core',
['Compiler/intel/2013.3.163'], ['Core']),
'CUDA-5.5.22-GCC-4.8.2.eb': ('CUDA/5.5.22', 'Compiler/GCC/4.8.2',
['Compiler/GCC-CUDA/4.8.2-5.5.22'
], ['Core']),
impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s' % iccver,
['MPI/intel/%s/impi/4.1.3.049' % iccver], ['Core']),
imkl_ec: ('imkl/11.1.2.144',
'MPI/intel/%s/impi/4.1.3.049' % iccver, [], ['Core']),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile, *mns_vals)
# impi with dummy toolchain, which doesn't make sense in a hierarchical context
ec = EasyConfig(os.path.join(ecs_dir, 'impi-4.1.3.049.eb'))
self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib',
ActiveMNS().det_modpath_extensions, ec)
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
init_config(build_options=build_options)
# format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions)
test_ecs = {
'GCC-4.7.2.eb':
('GCC/4.7.2', 'Core/compiler',
['Compiler/GCC/4.7.2/%s' % c for c in moduleclasses]),
'OpenMPI-1.6.4-GCC-4.7.2.eb':
('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2/mpi',
['MPI/GCC/4.7.2/OpenMPI/1.6.4/%s' % c for c in moduleclasses]),
'gzip-1.5-goolf-1.4.10.eb':
('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4/base', []),
'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core/toolchain', []),
'icc-2013.5.192-GCC-4.8.3.eb':
('icc/%s' % iccver, 'Core/compiler',
['Compiler/intel/%s/%s' % (iccver, c) for c in moduleclasses]),
'ifort-2013.3.163.eb':
('ifort/2013.3.163', 'Core/compiler',
['Compiler/intel/2013.3.163/%s' % c for c in moduleclasses]),
'CUDA-5.5.22-GCC-4.8.2.eb':
('CUDA/5.5.22', 'Compiler/GCC/4.8.2/system',
['Compiler/GCC-CUDA/4.8.2-5.5.22/%s' % c for c in moduleclasses]),
impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s/mpi' % iccver, [
'MPI/intel/%s/impi/4.1.3.049/%s' % (iccver, c)
for c in moduleclasses
]),
imkl_ec: ('imkl/11.1.2.144',
'MPI/intel/%s/impi/4.1.3.049/numlib' % iccver, []),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile,
*mns_vals,
init_modpaths=['Core/%s' % c for c in moduleclasses])
# impi with dummy toolchain, which doesn't make sense in a hierarchical context
ec = EasyConfig(os.path.join(ecs_dir, 'impi-4.1.3.049.eb'))
self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib',
ActiveMNS().det_modpath_extensions, ec)
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
test_ecs = {
'GCC-4.7.2.eb': ('GCC/4.7.2', '', [], []),
'OpenMPI-1.6.4-GCC-4.7.2.eb':
('OpenMPI/1.6.4-GCC-4.7.2', '', [], []),
'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5-goolf-1.4.10', '', [], []),
'goolf-1.4.10.eb': ('goolf/1.4.10', '', [], []),
'impi-4.1.3.049.eb': ('impi/4.1.3.049', '', [], []),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile, *mns_vals)
class ModuleGeneratorTest(EnhancedTestCase):
""" testcase for ModuleGenerator """
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def tearDown(self):
"""cleanup"""
super(ModuleGeneratorTest, self).tearDown()
os.remove(self.eb.logfile)
shutil.rmtree(self.modgen.app.installdir)
def test_descr(self):
"""Test generation of module description (which includes '#%Module' header)."""
gzip_txt = "gzip (GNU zip) is a popular data compression program as a replacement for compress "
gzip_txt += "- Homepage: http://www.gzip.org/"
expected = '\n'.join([
"#%Module",
"",
"proc ModulesHelp { } {",
" puts stderr { %s" % gzip_txt,
" }",
"}",
"",
"module-whatis {Description: %s}" % gzip_txt,
"",
"set root %s" % self.modgen.app.installdir,
"",
"conflict gzip",
"",
])
desc = self.modgen.get_description()
self.assertEqual(desc, expected)
def test_load(self):
"""Test load part in generated module file."""
expected = [
"",
"if { ![is-loaded mod_name] } {",
" module load mod_name",
"}",
"",
]
self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name"))
# with recursive unloading: no if is-loaded guard
expected = [
"",
"module load mod_name",
"",
]
self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name", recursive_unload=True))
def test_unload(self):
"""Test unload part in generated module file."""
expected = '\n'.join([
"",
"if { [is-loaded mod_name] } {",
" module unload mod_name",
"}",
"",
])
self.assertEqual(expected, self.modgen.unload_module("mod_name"))
def test_prepend_paths(self):
"""Test generating prepend-paths statements."""
# test prepend_paths
expected = ''.join([
"prepend-path\tkey\t\t$root/path1\n",
"prepend-path\tkey\t\t$root/path2\n",
])
self.assertEqual(expected, self.modgen.prepend_paths("key", ["path1", "path2"]))
expected = "prepend-path\tbar\t\t$root/foo\n"
self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo"))
self.assertEqual("prepend-path\tkey\t\t/abs/path\n", self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True))
self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to prepend_paths " \
"which only expects relative paths." % self.modgen.app.installdir,
self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
def test_env(self):
"""Test setting of environment variables."""
# test set_environment
self.assertEqual('setenv\tkey\t\t"value"\n', self.modgen.set_environment("key", "value"))
self.assertEqual("setenv\tkey\t\t'va\"lue'\n", self.modgen.set_environment("key", 'va"lue'))
self.assertEqual('setenv\tkey\t\t"va\'lue"\n', self.modgen.set_environment("key", "va'lue"))
self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_environment("key", """va"l'ue"""))
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files]
ec_files = [fil for fil in ec_files if not "v2.0" in fil] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(ec2mod_map[ec_fn], det_full_module_name_mg(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar', det_full_module_name_ec(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
for test_mns_mod in ['test_module_naming_scheme', 'test_module_naming_scheme_all']:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
mns_mod = __import__(mns_path, globals(), locals(), [''])
test_mnss = dict([(x.__name__, x) for x in get_subclasses(mns_mod.ModuleNamingScheme)])
easybuild.tools.module_naming_scheme.AVAIL_MODULE_NAMING_SCHEMES.update(test_mnss)
init_config(build_options=build_options)
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
# test module naming scheme using all available easyconfig parameters
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeAll'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/afd4d25a1a2cdb1364c55274fb6929fab622f652',
'gzip-1.4.eb': 'gzip/b6306986fb95a06ad8bd2a09689d8997ff3e80dd',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/3c2d54583487828c21e17ed185eac372cabc5bb0',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1fb1e3787d6063e05a04b2c054faf00dbe1dfe97',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/78c9afa1ff09994fe38d796b7569ce4b175e3551',
'toy-0.0.eb': 'toy/494518267cc5ed64c4250c5fbd1730a6e48fde17',
'toy-0.0-multiple.eb': 'toy/02822d81743944e1c072fc3c717c666da70f1be6',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
]:
self.assertEqual(det_full_module_name_ec(dep_spec), ec2mod_map[dep_ec])
# restore default module naming scheme, and retest
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def test_mod_name_validation(self):
"""Test module naming validation."""
# module name must be a string
self.assertTrue(not is_valid_module_name(('foo', 'bar')))
self.assertTrue(not is_valid_module_name(['foo', 'bar']))
self.assertTrue(not is_valid_module_name(123))
# module name must be relative
self.assertTrue(not is_valid_module_name('/foo/bar'))
# module name must only contain valid characters
self.assertTrue(not is_valid_module_name('foo\x0bbar'))
self.assertTrue(not is_valid_module_name('foo\x0cbar'))
self.assertTrue(not is_valid_module_name('foo\rbar'))
self.assertTrue(not is_valid_module_name('foo\0bar'))
# valid module name must be accepted
self.assertTrue(is_valid_module_name('gzip/goolf-1.4.10-suffix'))
self.assertTrue(is_valid_module_name('GCC/4.7.2'))
self.assertTrue(is_valid_module_name('foo-bar/1.2.3'))
self.assertTrue(is_valid_module_name('ictce'))
class ModuleGeneratorTest(EnhancedTestCase):
""" testcase for ModuleGenerator """
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(
os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(
prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def tearDown(self):
"""cleanup"""
super(ModuleGeneratorTest, self).tearDown()
os.remove(self.eb.logfile)
shutil.rmtree(self.modgen.app.installdir)
def test_descr(self):
"""Test generation of module description (which includes '#%Module' header)."""
gzip_txt = "gzip (GNU zip) is a popular data compression program as a replacement for compress "
gzip_txt += "- Homepage: http://www.gzip.org/"
expected = '\n'.join([
"#%Module",
"",
"proc ModulesHelp { } {",
" puts stderr { %s" % gzip_txt,
" }",
"}",
"",
"module-whatis {Description: %s}" % gzip_txt,
"",
"set root %s" % self.modgen.app.installdir,
"",
"conflict gzip",
"",
])
desc = self.modgen.get_description()
self.assertEqual(desc, expected)
def test_load(self):
"""Test load part in generated module file."""
expected = [
"",
"if { ![is-loaded mod_name] } {",
" module load mod_name",
"}",
"",
]
self.assertEqual('\n'.join(expected),
self.modgen.load_module("mod_name"))
# with recursive unloading: no if is-loaded guard
expected = [
"",
"module load mod_name",
"",
]
self.assertEqual(
'\n'.join(expected),
self.modgen.load_module("mod_name", recursive_unload=True))
def test_unload(self):
"""Test unload part in generated module file."""
expected = '\n'.join([
"",
"if { [is-loaded mod_name] } {",
" module unload mod_name",
"}",
"",
])
self.assertEqual(expected, self.modgen.unload_module("mod_name"))
def test_prepend_paths(self):
"""Test generating prepend-paths statements."""
# test prepend_paths
expected = ''.join([
"prepend-path\tkey\t\t$root/path1\n",
"prepend-path\tkey\t\t$root/path2\n",
])
self.assertEqual(expected,
self.modgen.prepend_paths("key", ["path1", "path2"]))
expected = "prepend-path\tbar\t\t$root/foo\n"
self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo"))
self.assertEqual(
"prepend-path\tkey\t\t/abs/path\n",
self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True))
self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to prepend_paths " \
"which only expects relative paths." % self.modgen.app.installdir,
self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
def test_env(self):
"""Test setting of environment variables."""
# test set_environment
self.assertEqual('setenv\tkey\t\t"value"\n',
self.modgen.set_environment("key", "value"))
self.assertEqual("setenv\tkey\t\t'va\"lue'\n",
self.modgen.set_environment("key", 'va"lue'))
self.assertEqual('setenv\tkey\t\t"va\'lue"\n',
self.modgen.set_environment("key", "va'lue"))
self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n',
self.modgen.set_environment("key", """va"l'ue"""))
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [
os.path.join(subdir, fil)
for (subdir, _, files) in os.walk(ecs_dir) for fil in files
]
ec_files = [fil for fil in ec_files if not "v2.0" in fil
] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(ec2mod_map[ec_fn],
det_full_module_name_mg(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar',
det_full_module_name_ec(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
for test_mns_mod in [
'test_module_naming_scheme', 'test_module_naming_scheme_all'
]:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
mns_mod = __import__(mns_path, globals(), locals(), [''])
test_mnss = dict([
(x.__name__, x)
for x in get_subclasses(mns_mod.ModuleNamingScheme)
])
easybuild.tools.module_naming_scheme.AVAIL_MODULE_NAMING_SCHEMES.update(
test_mnss)
init_config(build_options=build_options)
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
# test module naming scheme using all available easyconfig parameters
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeAll'
init_config(build_options=build_options)
# note: these checksums will change if another easyconfig parameter is added
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/698cacc77167c6824f597f0b6371cad5e6749922',
'gzip-1.4.eb': 'gzip/d240a51c643ec42e709d405d958c7b26f5a25d5a',
'gzip-1.4-GCC-4.6.3.eb':
'gzip/cea02d332af7044ae5faf762cea2ef6ffed014d2',
'gzip-1.5-goolf-1.4.10.eb':
'gzip/f1dbb38c4518a15fc8bb1fbf797ceda02f0cacd0',
'gzip-1.5-ictce-4.1.13.eb':
'gzip/3ef9ac73b468c989f5a47b30098d340e92c3d0da',
'toy-0.0.eb': 'toy/778417f0e140ebbaebd60d0f98c8b2411f980edf',
'toy-0.0-multiple.eb':
'toy/2d45f3cde87dedf30662f4a005023d56d2532bf0',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {
'name': 'goolf',
'version': '1.4.10'
},
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
}),
]:
self.assertEqual(det_full_module_name_ec(dep_spec),
ec2mod_map[dep_ec])
# restore default module naming scheme, and retest
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def test_mod_name_validation(self):
"""Test module naming validation."""
# module name must be a string
self.assertTrue(not is_valid_module_name(('foo', 'bar')))
self.assertTrue(not is_valid_module_name(['foo', 'bar']))
self.assertTrue(not is_valid_module_name(123))
# module name must be relative
self.assertTrue(not is_valid_module_name('/foo/bar'))
# module name must only contain valid characters
self.assertTrue(not is_valid_module_name('foo\x0bbar'))
self.assertTrue(not is_valid_module_name('foo\x0cbar'))
self.assertTrue(not is_valid_module_name('foo\rbar'))
self.assertTrue(not is_valid_module_name('foo\0bar'))
# valid module name must be accepted
self.assertTrue(is_valid_module_name('gzip/goolf-1.4.10-suffix'))
self.assertTrue(is_valid_module_name('GCC/4.7.2'))
self.assertTrue(is_valid_module_name('foo-bar/1.2.3'))
self.assertTrue(is_valid_module_name('ictce'))
class ModuleGeneratorTest(EnhancedTestCase):
""" testcase for ModuleGenerator """
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def tearDown(self):
"""cleanup"""
super(ModuleGeneratorTest, self).tearDown()
os.remove(self.eb.logfile)
shutil.rmtree(self.modgen.app.installdir)
def test_descr(self):
"""Test generation of module description (which includes '#%Module' header)."""
gzip_txt = "gzip (GNU zip) is a popular data compression program as a replacement for compress "
gzip_txt += "- Homepage: http://www.gzip.org/"
expected = '\n'.join([
"#%Module",
"",
"proc ModulesHelp { } {",
" puts stderr { %s" % gzip_txt,
" }",
"}",
"",
"module-whatis {Description: %s}" % gzip_txt,
"",
"set root %s" % self.modgen.app.installdir,
"",
"conflict gzip",
"",
])
desc = self.modgen.get_description()
self.assertEqual(desc, expected)
def test_load(self):
"""Test load part in generated module file."""
expected = [
"",
"if { ![is-loaded mod_name] } {",
" module load mod_name",
"}",
"",
]
self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name"))
# with recursive unloading: no if is-loaded guard
expected = [
"",
"module load mod_name",
"",
]
self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name", recursive_unload=True))
def test_unload(self):
"""Test unload part in generated module file."""
expected = '\n'.join([
"",
"if { [is-loaded mod_name] } {",
" module unload mod_name",
"}",
"",
])
self.assertEqual(expected, self.modgen.unload_module("mod_name"))
def test_prepend_paths(self):
"""Test generating prepend-paths statements."""
# test prepend_paths
expected = ''.join([
"prepend-path\tkey\t\t$root/path1\n",
"prepend-path\tkey\t\t$root/path2\n",
])
self.assertEqual(expected, self.modgen.prepend_paths("key", ["path1", "path2"]))
expected = "prepend-path\tbar\t\t$root/foo\n"
self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo"))
self.assertEqual("prepend-path\tkey\t\t/abs/path\n", self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True))
self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to prepend_paths " \
"which only expects relative paths." % self.modgen.app.installdir,
self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])
def test_env(self):
"""Test setting of environment variables."""
# test set_environment
self.assertEqual('setenv\tkey\t\t"value"\n', self.modgen.set_environment("key", "value"))
self.assertEqual("setenv\tkey\t\t'va\"lue'\n", self.modgen.set_environment("key", 'va"lue'))
self.assertEqual('setenv\tkey\t\t"va\'lue"\n', self.modgen.set_environment("key", "va'lue"))
self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_environment("key", """va"l'ue"""))
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files]
ec_files = [fil for fil in ec_files if not "v2.0" in fil] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(ec2mod_map[ec_fn], det_full_module_name_mg(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar', det_full_module_name_ec(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
for test_mns_mod in ['test_module_naming_scheme', 'test_module_naming_scheme_all']:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
mns_mod = __import__(mns_path, globals(), locals(), [''])
test_mnss = dict([(x.__name__, x) for x in get_subclasses(mns_mod.ModuleNamingScheme)])
easybuild.tools.module_naming_scheme.AVAIL_MODULE_NAMING_SCHEMES.update(test_mnss)
init_config(build_options=build_options)
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
# test module naming scheme using all available easyconfig parameters
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeAll'
init_config(build_options=build_options)
# note: these checksums will change if another easyconfig parameter is added
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/698cacc77167c6824f597f0b6371cad5e6749922',
'gzip-1.4.eb': 'gzip/d240a51c643ec42e709d405d958c7b26f5a25d5a',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/cea02d332af7044ae5faf762cea2ef6ffed014d2',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/f1dbb38c4518a15fc8bb1fbf797ceda02f0cacd0',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/3ef9ac73b468c989f5a47b30098d340e92c3d0da',
'toy-0.0.eb': 'toy/778417f0e140ebbaebd60d0f98c8b2411f980edf',
'toy-0.0-multiple.eb': 'toy/2d45f3cde87dedf30662f4a005023d56d2532bf0',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
]:
self.assertEqual(det_full_module_name_ec(dep_spec), ec2mod_map[dep_ec])
# restore default module naming scheme, and retest
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def test_mod_name_validation(self):
"""Test module naming validation."""
# module name must be a string
self.assertTrue(not is_valid_module_name(('foo', 'bar')))
self.assertTrue(not is_valid_module_name(['foo', 'bar']))
self.assertTrue(not is_valid_module_name(123))
# module name must be relative
self.assertTrue(not is_valid_module_name('/foo/bar'))
# module name must only contain valid characters
self.assertTrue(not is_valid_module_name('foo\x0bbar'))
self.assertTrue(not is_valid_module_name('foo\x0cbar'))
self.assertTrue(not is_valid_module_name('foo\rbar'))
self.assertTrue(not is_valid_module_name('foo\0bar'))
# valid module name must be accepted
self.assertTrue(is_valid_module_name('gzip/goolf-1.4.10-suffix'))
self.assertTrue(is_valid_module_name('GCC/4.7.2'))
self.assertTrue(is_valid_module_name('foo-bar/1.2.3'))
self.assertTrue(is_valid_module_name('ictce'))
def setUp(self):
""" initialize ModuleGenerator with test Application """
self.modgen = ModuleGenerator(Application('easybuild/test/easyconfigs/gzip-1.4.eb'))
self.modgen.app.installdir = "/tmp"
