def test_check_readiness(self):
"""Test check_readiness method."""
init_config(build_options={'validate': False})
# check that check_readiness step works (adding dependencies, etc.)
ec_file = 'OpenMPI-1.6.4-GCC-4.6.4.eb'
ec_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'easyconfigs', ec_file)
ec = EasyConfig(ec_path)
eb = EasyBlock(ec)
eb.check_readiness_step()
# a proper error should be thrown for dependencies that can't be resolved (module should be there)
tmpdir = tempfile.mkdtemp()
shutil.copy2(ec_path, tmpdir)
ec_path = os.path.join(tmpdir, ec_file)
f = open(ec_path, 'a')
f.write("\ndependencies += [('nosuchsoftware', '1.2.3')]\n")
f.close()
ec = EasyConfig(ec_path)
eb = EasyBlock(ec)
try:
eb.check_readiness_step()
except EasyBuildError, err:
err_regex = re.compile(
"Missing modules for one or more dependencies: nosuchsoftware/1.2.3-GCC-4.6.4"
)
self.assertTrue(
err_regex.search(str(err)),
"Pattern '%s' found in '%s'" % (err_regex.pattern, err))
def test_extensions_step(self):
"""Test the extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1"]',
])
self.writeEC()
"""Testcase for extensions"""
# test for proper error message without the exts_defaultclass set
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
self.assertRaises(EasyBuildError, eb.extensions_step, fetch=True)
self.assertErrorRegex(EasyBuildError, "No default extension class set", eb.extensions_step, fetch=True)
# test if everything works fine if set
self.contents += "\nexts_defaultclass = 'DummyExtension'"
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.extensions_step(fetch=True)
# test for proper error message when skip is set, but no exts_filter is set
self.assertRaises(EasyBuildError, eb.skip_extensions)
self.assertErrorRegex(EasyBuildError, "no exts_filter set", eb.skip_extensions)
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_patch_step(self):
"""Test patch step."""
test_easyconfigs = os.path.join(
os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
ec = process_easyconfig(os.path.join(test_easyconfigs,
'toy-0.0.eb'))[0]
orig_sources = ec['ec']['sources'][:]
toy_patches = [
'toy-0.0_typo.patch', # test for applying patch
('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy
]
self.assertEqual(ec['ec']['patches'], toy_patches)
# test applying patches without sources
ec['ec']['sources'] = []
eb = EasyBlock(ec['ec'])
eb.fetch_step()
eb.extract_step()
self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step)
# test actual patching of unpacked sources
ec['ec']['sources'] = orig_sources
eb = EasyBlock(ec['ec'])
eb.fetch_step()
eb.extract_step()
eb.patch_step()
# verify that patches were applied
toydir = os.path.join(eb.builddir, 'toy-0.0')
self.assertEqual(sorted(os.listdir(toydir)),
['toy-extra.txt', 'toy.source', 'toy.source.orig'])
self.assertTrue("and very proud of it" in read_file(
os.path.join(toydir, 'toy.source')))
self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')),
'moar!\n')
def test_extensions_step(self):
"""Test the extensions_step"""
self.contents = """
name = "pi"
version = "3.14"
homepage = "http://google.com"
description = "test easyconfig"
toolchain = {"name":"dummy", "version": "dummy"}
exts_list = ['ext1']
"""
self.writeEC()
"""Testcase for extensions"""
# test for proper error message without the exts_defaultclass set
eb = EasyBlock(self.eb_file)
eb.installdir = config.variables['installpath']
self.assertRaises(EasyBuildError, eb.extensions_step)
self.assertErrorRegex(EasyBuildError, "No default extension class set",
eb.extensions_step)
# test if everything works fine if set
self.contents += "\nexts_defaultclass = ['easybuild.framework.extension', 'Extension']"
self.writeEC()
eb = EasyBlock(self.eb_file)
eb.builddir = config.variables['buildpath']
eb.installdir = config.variables['installpath']
eb.extensions_step()
# test for proper error message when skip is set, but no exts_filter is set
self.assertRaises(EasyBuildError, eb.skip_extensions)
self.assertErrorRegex(EasyBuildError, "no exts_filter set",
eb.skip_extensions)
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_make_module_extend_modpath(self):
"""Test for make_module_extend_modpath"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'moduleclass = "compiler"',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
self.assertEqual(eb.make_module_extend_modpath(), '')
usermodsdir = 'my/own/modules'
modclasses = ['compiler', 'tools']
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
build_options = {
'subdir_user_modules': usermodsdir,
'valid_module_classes': modclasses,
}
init_config(build_options=build_options)
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
txt = eb.make_module_extend_modpath()
if get_module_syntax() == 'Tcl':
regexs = [r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c for c in modclasses]
home = r'\$env\(HOME\)'
regexs.extend([
# extension for user modules is guarded
r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' % (home, usermodsdir),
])
elif get_module_syntax() == 'Lua':
regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$' % c for c in modclasses]
home = r'os.getenv\("HOME"\)'
regexs.extend([
# extension for user modules is guarded
r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)' % (home, usermodsdir),
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
for regex in regexs:
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def test_skip_extensions_step(self):
"""Test the skip_extensions_step"""
self.contents = """
name = "pi"
version = "3.14"
homepage = "http://google.com"
description = "test easyconfig"
toolchain = {"name":"dummy", "version": "dummy"}
exts_list = ['ext1', 'ext2']
exts_filter = ("if [ %(name)s == 'ext2' ]; then exit 0; else exit 1; fi", '')
exts_defaultclass = ['easybuild.framework.extension', 'Extension']
"""
# check if skip skips correct extensions
self.writeEC()
eb = EasyBlock(self.eb_file)
#self.assertTrue('ext1' in eb.exts.keys() and 'ext2' in eb.exts.keys())
eb.builddir = config.variables['buildpath']
eb.installdir = config.variables['installpath']
eb.skip = True
eb.extensions_step()
# 'ext1' should be in eb.exts
self.assertTrue('ext1' in [y for x in eb.exts for y in x.values()])
# 'ext2' should not
self.assertFalse('ext2' in [y for x in eb.exts for y in x.values()])
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search("^prepend-path\s+CPATH\s+.*$", guess, re.M))
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_exts_list(self):
"""Test handling of list of extensions."""
os.environ['EASYBUILD_SOURCEPATH'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
init_config()
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = [',
' ("ext1", "ext_ver1", {',
' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail
' "source_urls": ["http://example.com/"]',
' }),',
' ("ext2", "ext_ver2", {',
' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail
' "source_urls": [("http://example.com", "suffix")],'
' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail
' "checksums": [',
' "504c7036558938f997c1c269a01d7458",', # checksum for source (gzip-1.4.eb)
' "ddd5161154f5db67701525123129ff09",', # checksum for patch (toy-0.0.eb)
' ],',
' }),',
']',
])
self.prep()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
exts_sources = eb.fetch_extension_sources()
def test_prepare_step(self):
"""Test prepare step (setting up build environment)."""
test_easyconfigs = os.path.join(
os.path.abspath(os.path.dirname(__file__)), 'easyconfigs',
'test_ecs')
ec = process_easyconfig(
os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0]
mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'dummy-dummy'),
parents=True)
eb = EasyBlock(ec['ec'])
eb.silent = True
eb.prepare_step()
self.assertEqual(self.modtool.list(), [])
os.environ['THIS_IS_AN_UNWANTED_ENV_VAR'] = 'foo'
eb.cfg['unwanted_env_vars'] = ['THIS_IS_AN_UNWANTED_ENV_VAR']
eb.cfg['allow_system_deps'] = [('Python', '1.2.3')]
init_config(build_options={'extra_modules': ['GCC/4.7.2']})
eb.prepare_step()
self.assertEqual(os.environ.get('THIS_IS_AN_UNWANTED_ENV_VAR'), None)
self.assertEqual(os.environ.get('EBROOTPYTHON'), 'Python')
self.assertEqual(os.environ.get('EBVERSIONPYTHON'), '1.2.3')
self.assertEqual(len(self.modtool.list()), 1)
self.assertEqual(self.modtool.list()[0]['mod_name'], 'GCC/4.7.2')
def test_skip_extensions_step(self):
"""Test the skip_extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1", "ext2"]',
'exts_filter = ("if [ %(ext_name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")',
'exts_defaultclass = "DummyExtension"',
])
# check if skip skips correct extensions
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.skip = True
eb.extensions_step(fetch=True)
# 'ext1' should be in eb.exts
self.assertTrue('ext1' in [y for x in eb.exts for y in x.values()])
# 'ext2' should not
self.assertFalse('ext2' in [y for x in eb.exts for y in x.values()])
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://google.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = [('foo', '1.2.3')]",
"builddependencies = [('bar', '9.8.7')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
# overwrite installpath config setting
orig_installpath = config.variables['installpath']
installpath = tempfile.mkdtemp()
config.variables['installpath'] = installpath
# test if module is generated correctly
self.writeEC()
eb = EasyBlock(self.eb_file)
eb.installdir = os.path.join(config.variables['installpath'],
config.variables['subdir_software'], 'pi',
'3.14')
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath))
# verify contents of module
f = open(modpath, 'r')
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(
re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(
re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt,
re.M))
self.assertTrue(
re.search(
'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt,
re.M))
for (key, val) in modextravars.items():
self.assertTrue(
re.search('^setenv\s+%s\s+"%s"$' % (key, val), txt, re.M))
for (key, val) in modextrapaths.items():
self.assertTrue(
re.search('^prepend-path\s+%s\s+\$root/%s$' % (key, val), txt,
re.M))
# restore original settings
config.variables['installpath'] = orig_installpath
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
deps = [('GCC', '4.6.4')]
hiddendeps = [('toy', '0.0-deps')]
alldeps = deps + hiddendeps # hidden deps must be included in list of deps
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = %s" % str(alldeps),
"hiddendependencies = %s" % str(hiddendeps),
"builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
test_dir = os.path.dirname(os.path.abspath(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
# test if module is generated correctly
self.writeEC()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
f = open(modpath, 'r')
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M))
self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))
for (key, val) in modextravars.items():
regex = re.compile('^setenv\s+%s\s+"%s"$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, val) in modextrapaths.items():
regex = re.compile('^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in deps:
regex = re.compile('^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in hiddendeps:
regex = re.compile('^\s*module load %s/.%s\s*$' % (name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
def test_obtain_file(self):
"""Test obtain_file method."""
toy_tarball = 'toy-0.0.tar.gz'
testdir = os.path.abspath(os.path.dirname(__file__))
sandbox_sources = os.path.join(testdir, 'sandbox', 'sources')
toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball)
tmpdir = tempfile.mkdtemp()
tmpdir_subdir = os.path.join(tmpdir, 'testing')
mkdir(tmpdir_subdir, parents=True)
del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp
ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0]
eb = EasyBlock(ec['ec'])
# 'downloading' a file to (first) sourcepath works
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)])
shutil.copy2(toy_tarball_path, tmpdir_subdir)
res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball))
# finding a file in sourcepath works
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)])
res = eb.obtain_file(toy_tarball)
self.assertEqual(res, toy_tarball_path)
# sourcepath has preference over downloading
res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
self.assertEqual(res, toy_tarball_path)
# obtain_file yields error for non-existing files
fn = 'thisisclearlyanonexistingfile'
error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn
self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir])
# file specifications via URL also work, are downloaded to (first) sourcepath
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)])
urls = ["http://hpcugent.github.io/easybuild/index.html", "https://hpcugent.github.io/easybuild/index.html"]
for file_url in urls:
fn = os.path.basename(file_url)
res = None
try:
res = eb.obtain_file(file_url)
except EasyBuildError, err:
# if this fails, it should be because there's no online access
download_fail_regex = re.compile('socket error')
self.assertTrue(download_fail_regex.search(str(err)))
# result may be None during offline testing
if res is not None:
loc = os.path.join(tmpdir, 't', 'toy', fn)
self.assertEqual(res, loc)
self.assertTrue(os.path.exists(loc), "%s file is found at %s" % (fn, loc))
txt = open(loc, 'r').read()
eb_regex = re.compile("EasyBuild: building software with ease")
self.assertTrue(eb_regex.search(txt))
else:
print "ignoring failure to download %s in test_obtain_file, testing offline?" % file_url
def test_patch_step(self):
"""Test patch step."""
ec = process_easyconfig(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb'))[0]
orig_sources = ec['ec']['sources'][:]
# test applying patches without sources
ec['ec']['sources'] = []
eb = EasyBlock(ec['ec'])
eb.fetch_step()
eb.extract_step()
self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step)
# test actual patching of unpacked sources
ec['ec']['sources'] = orig_sources
eb = EasyBlock(ec['ec'])
eb.fetch_step()
eb.extract_step()
eb.patch_step()
def check_start_dir(expected_start_dir):
"""Check start dir."""
eb = EasyBlock(ec['ec'])
eb.silent = True
eb.cfg['stop'] = 'patch'
eb.run_all_steps(False)
eb.guess_start_dir()
abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir)
self.assertTrue(os.path.samefile(eb.cfg['start_dir'], abs_expected_start_dir))
self.assertTrue(os.path.samefile(os.getcwd(), abs_expected_start_dir))
def test_exclude_path_to_top_of_module_tree(self):
"""
Make sure that modules under the HierarchicalMNS are correct,
w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
"""
self.orig_module_naming_scheme = config.get_module_naming_scheme()
test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all')
mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True)
mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True)
impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049')
imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144')
if get_module_syntax() == 'Lua':
impi_modfile_path += '.lua'
imkl_modfile_path += '.lua'
# example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
# not for the toolchain or any of the toolchain components,
# since both icc/ifort and impi form the path to the top of the module tree
tests = [
('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']),
('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']),
]
for ec_file, modfile_path, excluded_deps in tests:
ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
eb = EasyBlock(ec)
eb.toolchain.prepare()
modpath = eb.make_module_step()
modfile_path = os.path.join(modpath, modfile_path)
modtxt = read_file(modfile_path)
for dep in excluded_deps:
tup = (dep, modfile_path, modtxt)
failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
if get_module_syntax() == 'Tcl':
self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg)
elif get_module_syntax() == 'Lua':
self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
def configure_step(self, name=None):
"""Configure build of toy."""
if name is None:
name = self.name
# make sure Python system dep is handled correctly when specified
if self.cfg['allow_system_deps']:
if get_software_root('Python') != 'Python' or get_software_version(
'Python') != platform.python_version():
raise EasyBlock(
"Sanity check on allowed Python system dep failed.")
os.rename('%s.source' % name, '%s.c' % name)
def test_make_module_extra(self):
"""Test for make_module_extra."""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
if get_module_syntax() == 'Tcl':
expected_default = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+\"\$root"',
r'setenv\s+EBVERSIONPI\s+"3.14"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+"/opt/software/tau/6.28"',
r'setenv\s+EBVERSIONPI\s+"6.28"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
elif get_module_syntax() == 'Lua':
expected_default = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", root\)',
r'setenv\("EBVERSIONPI", "3.14"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", "/opt/software/tau/6.28"\)',
r'setenv\("EBVERSIONPI", "6.28"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
defaulttxt = eb.make_module_extra().strip()
self.assertTrue(
expected_default.match(defaulttxt),
"Pattern %s found in %s" % (expected_default.pattern, defaulttxt))
alttxt = eb.make_module_extra(altroot='/opt/software/tau/6.28',
altversion='6.28').strip()
self.assertTrue(
expected_alt.match(alttxt),
"Pattern %s found in %s" % (expected_alt.pattern, alttxt))
def setUp(self):
"""Test setup."""
super(ModuleGeneratorTest, self).setUp()
# find .eb file
topdir = os.path.dirname(os.path.abspath(__file__))
eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = self.MODULE_GENERATOR_CLASS(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def test_gen_dirs(self):
"""Test methods that generate/set build/install directory names."""
self.contents = '\n'.join([
"name = 'pi'",
"version = '3.14'",
"homepage = 'http://example.com'",
"description = 'test easyconfig'",
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
])
self.writeEC()
stdoutorig = sys.stdout
sys.stdout = open("/dev/null", 'w')
eb = EasyBlock(EasyConfig(self.eb_file))
resb = eb.gen_builddir()
eb.mod_name = det_full_module_name(
eb.cfg) # required by gen_installdir()
resi = eb.gen_installdir()
eb.make_builddir()
eb.make_installdir()
# doesn't return anything
self.assertEqual(resb, None)
self.assertEqual(resi, None)
# directories are set, and exist
self.assertTrue(os.path.isdir(eb.builddir))
self.assertTrue(os.path.isdir(eb.installdir))
# make sure cleaning up old build dir is default
self.assertTrue(eb.cfg['cleanupoldbuild']
or eb.cfg.get('cleanupoldbuild', True))
builddir = eb.builddir
eb.gen_builddir()
self.assertEqual(builddir, eb.builddir)
eb.cfg['cleanupoldbuild'] = True
eb.gen_builddir()
self.assertEqual(builddir, eb.builddir)
# make sure build dir is unique
eb.cfg['cleanupoldbuild'] = False
builddir = eb.builddir
for i in range(3):
eb.gen_builddir()
self.assertEqual(eb.builddir, "%s.%d" % (builddir, i))
eb.make_builddir()
# cleanup
sys.stdout.close()
sys.stdout = stdoutorig
eb.close_log()
def test_make_module_dep_hmns(self):
"""Test for make_module_dep under HMNS"""
test_ecs_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.4.10'}",
'dependencies = [',
" ('FFTW', '3.3.3'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
eb.make_builddir()
eb.prepare_step()
# GCC, OpenMPI and hwloc modules should *not* be included in loads for dependencies
mod_dep_txt = eb.make_module_dep()
for mod in ['GCC/4.7.2', 'OpenMPI/1.6.4']:
regex = re.compile('load.*%s' % mod)
self.assertFalse(
regex.search(mod_dep_txt),
"Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
regex = re.compile('load.*FFTW/3.3.3')
self.assertTrue(
regex.search(mod_dep_txt),
"Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(
os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(
prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def test_fake_module_load(self):
"""Testcase for fake module load"""
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.build_path()
fake_mod_data = eb.load_fake_module()
eb.clean_up_fake_module(fake_mod_data)
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_fake_module_load(self):
"""Testcase for fake module load"""
self.contents = """
name = "pi"
version = "3.14"
homepage = "http://google.com"
description = "test easyconfig"
toolchain = {"name":"dummy", "version": "dummy"}
"""
self.writeEC()
eb = EasyBlock(self.eb_file)
eb.installdir = config.variables['installpath']
fake_mod_data = eb.load_fake_module()
eb.clean_up_fake_module(fake_mod_data)
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_obtain_file(self):
"""Test obtain_file method."""
toy_tarball = 'toy-0.0.tar.gz'
testdir = os.path.abspath(os.path.dirname(__file__))
sandbox_sources = os.path.join(testdir, 'sandbox', 'sources')
toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball)
tmpdir = tempfile.mkdtemp()
tmpdir_subdir = os.path.join(tmpdir, 'testing')
mkdir(tmpdir_subdir, parents=True)
del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp
ec = process_easyconfig(
os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0]
eb = EasyBlock(ec['ec'])
# 'downloading' a file to (first) sourcepath works
init_config(
args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)])
shutil.copy2(toy_tarball_path, tmpdir_subdir)
res = eb.obtain_file(toy_tarball,
urls=[os.path.join('file://', tmpdir_subdir)])
self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball))
# finding a file in sourcepath works
init_config(args=[
"--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)
])
res = eb.obtain_file(toy_tarball)
self.assertEqual(res, toy_tarball_path)
# sourcepath has preference over downloading
res = eb.obtain_file(toy_tarball,
urls=[os.path.join('file://', tmpdir_subdir)])
self.assertEqual(res, toy_tarball_path)
# obtain_file yields error for non-existing files
fn = 'thisisclearlyanonexistingfile'
try:
eb.obtain_file(fn, urls=[os.path.join('file://', tmpdir_subdir)])
except EasyBuildError, err:
fail_regex = re.compile(
"Couldn't find file %s anywhere, and downloading it didn't work either"
% fn)
self.assertTrue(fail_regex.search(str(err)))
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M))
self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_buildininstalldir(self):
"""Test specifying build in install dir."""
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'buildininstalldir = True',
])
self.prep()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.gen_builddir()
eb.gen_installdir()
eb.make_builddir()
eb.make_installdir()
self.assertEqual(eb.builddir, eb.installdir)
self.assertTrue(os.path.isdir(eb.builddir))
def test_parallel(self):
"""Test defining of parallellism."""
topdir = os.path.abspath(os.path.dirname(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
toytxt = read_file(toy_ec)
handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_',
suffix='.eb')
os.close(handle)
write_file(toy_ec1, toytxt + "\nparallel = 123")
handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_',
suffix='.eb')
os.close(handle)
write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67")
# default: parallellism is derived from # available cores + ulimit
test_eb = EasyBlock(EasyConfig(toy_ec))
test_eb.check_readiness_step()
self.assertTrue(
isinstance(test_eb.cfg['parallel'], int)
and test_eb.cfg['parallel'] > 0)
# only 'parallel' easyconfig parameter specified (no 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec1))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 123)
# both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 67)
# only 'parallel' build option specified
init_config(build_options={'parallel': '97', 'validate': False})
test_eb = EasyBlock(EasyConfig(toy_ec))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 97)
# both 'parallel' build option and easyconfig parameter specified (no 'maxparallel')
test_eb = EasyBlock(EasyConfig(toy_ec1))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 97)
# both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 67)
def test_easyblock(self):
""" make sure easyconfigs defining extensions work"""
self.contents = """
name = "pi"
version = "3.14"
homepage = "http://google.com"
description = "test easyconfig"
toolchain = {"name":"dummy", "version": "dummy"}
exts_list = ['ext1']
"""
self.writeEC()
stdoutorig = sys.stdout
sys.stdout = open("/dev/null", 'w')
eb = EasyBlock(self.eb_file)
self.assertRaises(NotImplementedError, eb.run_all_steps, True, False)
sys.stdout.close()
sys.stdout = stdoutorig
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_exts_list(self):
"""Test handling of list of extensions."""
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = [',
' ("ext1", "ext_ver1", {',
' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail
' "source_urls": ["http://example.com/"]',
' }),',
' ("ext2", "ext_ver2", {',
' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail
' "source_urls": [("http://example.com", "suffix")],'
' }),',
']',
])
self.prep()
eb = EasyBlock(self.eb_file)
exts_sources = eb.fetch_extension_sources()
