def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths):
"""Test whether active module naming scheme returns expected values."""
ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*','*', ecfile))[0])
self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname))
self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname)
self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir)
self.assertEqual(ActiveMNS().det_modpath_extensions(ec), modpath_exts)
self.assertEqual(ActiveMNS().det_user_modpath_extensions(ec), user_modpath_exts)
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths)
def test_parse_yeb(self):
"""Test parsing of .yeb easyconfigs."""
if 'yaml' not in sys.modules:
print "Skipping test_parse_yeb (no PyYAML available)"
return
build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
easybuild.tools.build_log.EXPERIMENTAL = True
testdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(testdir, 'easyconfigs')
test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
# test parsing
test_files = [
'bzip2-1.0.6-GCC-4.9.2',
'gzip-1.6-GCC-4.9.2',
'foss-2018a',
'intel-2018a',
'SQLite-3.8.10.2-foss-2018a',
'Python-2.7.10-intel-2018a',
'CrayCCE-5.1.29',
'toy-0.0',
]
for filename in test_files:
ec_yeb = EasyConfig(os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename))
# compare with parsed result of .eb easyconfig
ec_file = glob.glob(os.path.join(test_easyconfigs, 'test_ecs', '*', '*', '%s.eb' % filename))[0]
ec_eb = EasyConfig(ec_file)
for key in sorted(ec_yeb.asdict()):
eb_val = ec_eb[key]
yeb_val = ec_yeb[key]
if key == 'description':
# multi-line string is always terminated with '\n' in YAML, so strip it off
yeb_val = yeb_val.strip()
self.assertEqual(yeb_val, eb_val)
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
deps = [('GCC', '4.6.4')]
hiddendeps = [('toy', '0.0-deps')]
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = %s" % str(deps),
"hiddendependencies = %s" % str(hiddendeps),
"builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
test_dir = os.path.dirname(os.path.abspath(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
# test if module is generated correctly
self.writeEC()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
#eb.builddir = self.test_buildpath
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
f = open(modpath, 'r')
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M))
self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))
for (key, val) in modextravars.items():
regex = re.compile('^setenv\s+%s\s+"%s"$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, val) in modextrapaths.items():
regex = re.compile('^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in deps:
regex = re.compile('^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in hiddendeps:
regex = re.compile('^\s*module load %s/.%s\s*$' % (name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
def template_init_test(self, easyblock):
"""Test whether all easyconfigs can be initialized."""
def check_extra_options_format(extra_options):
"""Make sure extra_options value is of correct format."""
# EasyBuild v1.x
self.assertTrue(isinstance(extra_options, list))
for extra_option in extra_options:
self.assertTrue(isinstance(extra_option, tuple))
self.assertEqual(len(extra_option), 2)
self.assertTrue(isinstance(extra_option[0], basestring))
self.assertTrue(isinstance(extra_option[1], list))
self.assertEqual(len(extra_option[1]), 3)
# EasyBuild v2.0 (breaks backward compatibility compared to v1.x)
#self.assertTrue(isinstance(extra_options, dict))
#for key in extra_options:
# self.assertTrue(isinstance(extra_options[key], list))
# self.assertTrue(len(extra_options[key]), 3)
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# obtain easyblock class name using regex
f = open(easyblock, "r")
txt = f.read()
f.close()
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
extra_options = app_class.extra_options()
check_extra_options_format(extra_options)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_txt = ''
for (key, val) in extra_options:
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True):
"""
Process easyconfig, returning some information for each block
"""
blocks = retrieve_blocks_in_spec(path, onlyBlocks)
easyconfigs = []
for spec in blocks:
# process for dependencies and real installversionname
# - use mod? __init__ and importCfg are ignored.
_log.debug("Processing easyconfig %s" % spec)
# create easyconfig
try:
all_stops = [x[0] for x in EasyBlock.get_steps()]
ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops)
except EasyBuildError, err:
msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
_log.exception(msg)
name = ec['name']
# this app will appear as following module in the list
easyconfig = {
'spec': spec,
'module': (ec.name, ec.get_installversion()),
'dependencies': [],
'builddependencies': [],
}
if len(blocks) > 1:
easyconfig['originalSpec'] = path
# add build dependencies
for dep in ec.builddependencies():
deptup = (dep['name'], dep['tc'])
_log.debug("Adding build dependency %s for app %s." % (deptup, name))
easyconfig['builddependencies'].append(deptup)
# add dependencies (including build dependencies)
for dep in ec.dependencies():
deptup = (dep['name'], dep['tc'])
_log.debug("Adding dependency %s for app %s." % (deptup, name))
easyconfig['dependencies'].append(deptup)
# add toolchain as dependency too
if ec.toolchain.name != 'dummy':
dep = (ec.toolchain.name, ec.toolchain.version)
_log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)
del ec
# this is used by the parallel builder
easyconfig['unresolvedDependencies'] = copy.copy(easyconfig['dependencies'])
easyconfigs.append(easyconfig)
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'], det_full_ec_version(ec))
return fn
def test_exclude_path_to_top_of_module_tree(self):
"""
Make sure that modules under the HierarchicalMNS are correct,
w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
"""
self.orig_module_naming_scheme = config.get_module_naming_scheme()
test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all')
mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True)
mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True)
impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049')
imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144')
if get_module_syntax() == 'Lua':
impi_modfile_path += '.lua'
imkl_modfile_path += '.lua'
# example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
# not for the toolchain or any of the toolchain components,
# since both icc/ifort and impi form the path to the top of the module tree
tests = [
('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']),
('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']),
]
for ec_file, modfile_path, excluded_deps in tests:
ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
eb = EasyBlock(ec)
eb.toolchain.prepare()
modpath = eb.make_module_step()
modfile_path = os.path.join(modpath, modfile_path)
modtxt = read_file(modfile_path)
for dep in excluded_deps:
tup = (dep, modfile_path, modtxt)
failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
if get_module_syntax() == 'Tcl':
self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg)
elif get_module_syntax() == 'Lua':
self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
def template_module_only_test(self,
easyblock,
name='foo',
version='1.3.2',
extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, name=name, version=version, extratxt=extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
os.chdir(orig_workdir)
modfile = os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.3.2')
self.assertTrue(os.path.exists(modfile),
"Module file %s was generated" % modfile)
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def test_active_pns(self):
"""Test use of ActivePNS."""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
ec = EasyConfig(os.path.join(test_easyconfigs, 'OpenMPI-1.6.4-GCC-4.6.4.eb'), validate=False)
pns = ActivePNS()
# default: EasyBuild package naming scheme, pkg release 1
self.assertEqual(pns.name(ec), 'OpenMPI-1.6.4-GCC-4.6.4')
self.assertEqual(pns.version(ec), 'eb-%s' % EASYBUILD_VERSION)
self.assertEqual(pns.release(ec), '1')
def test_toolchain_inspection(self):
"""Test whether available toolchain inspection functionality is working."""
build_options = {
'robot_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'),
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(['/'.join([x['name'], x['version']]) for x in det_toolchain_compilers(ec)], ['GCC/4.7.2'])
self.assertEqual(det_toolchain_mpi(ec)['name'], 'OpenMPI')
ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'hwloc-1.6.2-GCC-4.6.4.eb'))
tc_comps = det_toolchain_compilers(ec)
self.assertEqual(['/'.join([x['name'], x['version']]) for x in tc_comps], ['GCC/4.6.4'])
self.assertEqual(det_toolchain_mpi(ec), None)
ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'))
self.assertEqual(det_toolchain_compilers(ec), None)
self.assertEqual(det_toolchain_mpi(ec), None)
def test_make_module_extra(self):
"""Test for make_module_extra."""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
if get_module_syntax() == 'Tcl':
expected_default = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+\"\$root"',
r'setenv\s+EBVERSIONPI\s+"3.14"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+"/opt/software/tau/6.28"',
r'setenv\s+EBVERSIONPI\s+"6.28"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
elif get_module_syntax() == 'Lua':
expected_default = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", root\)',
r'setenv\("EBVERSIONPI", "3.14"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", "/opt/software/tau/6.28"\)',
r'setenv\("EBVERSIONPI", "6.28"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
defaulttxt = eb.make_module_extra().strip()
self.assertTrue(
expected_default.match(defaulttxt),
"Pattern %s found in %s" % (expected_default.pattern, defaulttxt))
alttxt = eb.make_module_extra(altroot='/opt/software/tau/6.28',
altversion='6.28').strip()
self.assertTrue(
expected_alt.match(alttxt),
"Pattern %s found in %s" % (expected_alt.pattern, alttxt))
def test_package(self):
"""Test package function."""
init_config(build_options={'silent': True})
topdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy',
'toy-0.0-gompi-1.3.12-test.eb'),
validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# write a dummy log and report file to make sure they don't get packaged
logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log")
write_file(logfile, "I'm a logfile")
reportfile = os.path.join(easyblock.installdir, log_path(),
"report.md")
write_file(reportfile, "I'm a reportfile")
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(
pkgdir,
'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" %
easyblock.installdir)
self.assertTrue(
pkgtxt_regex.search(pkgtxt),
"Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
no_logfiles_regex = re.compile(
r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS',
re.DOTALL | re.MULTILINE)
self.assertFalse(
no_logfiles_regex.search(pkgtxt), "Pattern not '%s' found in: %s" %
(no_logfiles_regex.pattern, pkgtxt))
if DEBUG:
print "The FPM script debug output"
print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
print "The Package File"
print read_file(pkgfile)
def test_make_module_pythonpackage(self):
"""Test make_module_step of PythonPackage easyblock."""
app_class = get_easyblock_class('PythonPackage')
self.writeEC('PythonPackage', name='testpypkg', version='3.14')
app = app_class(EasyConfig(self.eb_file))
# install dir should not be there yet
self.assertFalse(os.path.exists(app.installdir))
# create install dir and populate it with subdirs/files
mkdir(app.installdir, parents=True)
# $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH
write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!')
write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar')
write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo')
write_file(os.path.join(app.installdir, 'lib', 'python2.7', 'site-packages', 'foo.egg'), 'foo egg')
write_file(os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'), 'libfoo: foo')
# create module file
app.make_module_step()
self.assertTrue(TMPDIR in app.installdir)
self.assertTrue(TMPDIR in app.installdir_mod)
modtxt = None
for cand_mod_filename in ['3.14', '3.14.lua']:
full_modpath = os.path.join(app.installdir_mod, 'testpypkg', cand_mod_filename)
if os.path.exists(full_modpath):
modtxt = read_file(full_modpath)
break
self.assertFalse(modtxt is None)
regexs = [
(r'^prepend.path.*\WCPATH\W.*include"?\W*$', True),
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WPATH\W.*bin"?\W*$', True),
(r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$', True),
(r'^prepend.path.*\WPYTHONPATH\W.*lib/python2.7/site-packages"?\W*$', True),
# lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False),
]
for (pattern, found) in regexs:
regex = re.compile(pattern, re.M)
if found:
assert_msg = "Pattern '%s' found in: %s" % (regex.pattern, modtxt)
else:
assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern, modtxt)
self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(first['tc'], '1.1-GCC-4.6.3')
self.assertEqual(second['tc'], '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(first['tc'], '1.1-GCC-4.6.3')
self.assertEqual(second['tc'], '2.2-GCC-4.6.3')
eb['dependencies'] = ["wrong type"]
self.assertErrorRegex(EasyBuildError,
"wrong type from unsupported type",
eb.dependencies)
eb['dependencies'] = [()]
self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies)
eb['dependencies'] = [{'name': "test"}]
self.assertErrorRegex(EasyBuildError, "without version",
eb.dependencies)
def test_shared_lib_ext(self):
""" inside easyconfigs shared_lib_ext should be set """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'sanity_check_paths = { "files": ["lib/lib.%s" % shared_lib_ext] }',
])
self.prep()
eb = EasyConfig(self.eb_file)
self.assertEqual(eb['sanity_check_paths']['files'][0], "lib/lib.%s" % get_shared_lib_ext())
def test_extensions_step(self):
"""Test the extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1"]',
])
self.writeEC()
"""Testcase for extensions"""
# test for proper error message without the exts_defaultclass set
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
self.assertRaises(EasyBuildError, eb.extensions_step, fetch=True)
self.assertErrorRegex(EasyBuildError,
"No default extension class set",
eb.extensions_step,
fetch=True)
# test if everything works fine if set
self.contents += "\nexts_defaultclass = 'DummyExtension'"
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.extensions_step(fetch=True)
# test for proper error message when skip is set, but no exts_filter is set
self.assertRaises(EasyBuildError, eb.skip_extensions)
self.assertErrorRegex(EasyBuildError, "no exts_filter set",
eb.skip_extensions)
# cleanup
eb.close_log()
os.remove(eb.logfile)
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'],
det_installversion(
ec['version'], ec['toolchain']['name'],
ec['toolchain']['version'], ec['versionprefix'],
ec['versionsuffix']))
return fn
def test_toy_archived_easyconfig(self):
"""Test archived easyconfig for a succesful build."""
repositorypath = os.path.join(self.test_installpath, 'easyconfigs_archive')
extra_args = [
'--repository=FileRepository',
'--repositorypath=%s' % repositorypath,
]
self.test_toy_build(raise_error=True, extra_args=extra_args)
archived_ec = os.path.join(repositorypath, 'toy', 'toy-0.0.eb')
self.assertTrue(os.path.exists(archived_ec))
ec = EasyConfig(archived_ec)
self.assertEqual(ec.name, 'toy')
self.assertEqual(ec.version, '0.0')
def test_yeb_get_config_obj(self):
"""Test get_config_dict method."""
testdir = os.path.dirname(os.path.abspath(__file__))
test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
ec = EasyConfig(os.path.join(test_yeb_easyconfigs, 'toy-0.0.yeb'))
ecdict = ec.parser.get_config_dict()
# changes to this dict should not affect the return value of the next call to get_config_dict
fn = 'test.tar.gz'
ecdict['sources'].append(fn)
ecdict_bis = ec.parser.get_config_dict()
self.assertTrue(fn in ecdict['sources'])
self.assertFalse(fn in ecdict_bis['sources'])
def test_parse_yeb(self):
"""Test parsing of .yeb easyconfigs."""
if 'yaml' not in sys.modules:
print "Skipping test_parse_yeb (no PyYAML available)"
return
testdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(testdir, 'easyconfigs')
test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
# test parsing
test_files = [
'bzip2-1.0.6-GCC-4.9.2',
'gzip-1.6-GCC-4.9.2',
'goolf-1.4.10',
'ictce-4.1.13',
'SQLite-3.8.10.2-goolf-1.4.10',
'Python-2.7.10-ictce-4.1.13',
'CrayCCE-5.1.29',
]
for filename in test_files:
ec_yeb = EasyConfig(
os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename))
# compare with parsed result of .eb easyconfig
ec_eb = EasyConfig(
os.path.join(test_easyconfigs, '%s.eb' % filename))
no_match = False
for key in sorted(ec_yeb.asdict()):
eb_val = ec_eb[key]
yeb_val = ec_yeb[key]
if key == 'description':
# multi-line string is always terminated with '\n' in YAML, so strip it off
yeb_val = yeb_val.strip()
self.assertEqual(yeb_val, eb_val)
def setUp(self):
"""Test setup."""
super(ModuleGeneratorTest, self).setUp()
# find .eb file
topdir = os.path.dirname(os.path.abspath(__file__))
eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = self.MODULE_GENERATOR_CLASS(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def test_extensions_sanity_check(self):
"""Test sanity check aspect of extensions."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb'))
# purposely put sanity check command in place that breaks the build,
# to check whether sanity check is only run once;
# sanity check commands are checked after checking sanity check paths, so this should work
toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')])
# this import only works here, since EB_toy is a test easyblock
from easybuild.easyblocks.toy import EB_toy
eb = EB_toy(toy_ec)
eb.silent = True
eb.run_all_steps(True)
def test_active_pns(self):
"""Test use of ActivePNS."""
init_config(build_options={'silent': True})
topdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
test_ec = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb')
ec = EasyConfig(test_ec, validate=False)
pns = ActivePNS()
# default: EasyBuild package naming scheme, pkg release 1
self.assertEqual(pns.name(ec), 'OpenMPI-2.1.2-GCC-6.4.0-2.28')
self.assertEqual(pns.version(ec), 'eb-%s' % EASYBUILD_VERSION)
self.assertEqual(pns.release(ec), '1')
def test_gen_dirs(self):
"""Test methods that generate/set build/install directory names."""
self.contents = '\n'.join([
"name = 'pi'",
"version = '3.14'",
"homepage = 'http://example.com'",
"description = 'test easyconfig'",
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
])
self.writeEC()
stdoutorig = sys.stdout
sys.stdout = open("/dev/null", 'w')
eb = EasyBlock(EasyConfig(self.eb_file))
resb = eb.gen_builddir()
eb.mod_name = det_full_module_name(
eb.cfg) # required by gen_installdir()
resi = eb.gen_installdir()
eb.make_builddir()
eb.make_installdir()
# doesn't return anything
self.assertEqual(resb, None)
self.assertEqual(resi, None)
# directories are set, and exist
self.assertTrue(os.path.isdir(eb.builddir))
self.assertTrue(os.path.isdir(eb.installdir))
# make sure cleaning up old build dir is default
self.assertTrue(eb.cfg['cleanupoldbuild']
or eb.cfg.get('cleanupoldbuild', True))
builddir = eb.builddir
eb.gen_builddir()
self.assertEqual(builddir, eb.builddir)
eb.cfg['cleanupoldbuild'] = True
eb.gen_builddir()
self.assertEqual(builddir, eb.builddir)
# make sure build dir is unique
eb.cfg['cleanupoldbuild'] = False
builddir = eb.builddir
for i in range(3):
eb.gen_builddir()
self.assertEqual(eb.builddir, "%s.%d" % (builddir, i))
eb.make_builddir()
# cleanup
sys.stdout.close()
sys.stdout = stdoutorig
eb.close_log()
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = [('foo', '1.2.3')]",
"builddependencies = [('bar', '9.8.7')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
# test if module is generated correctly
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
f = open(modpath, 'r')
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(
re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(
re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt,
re.M))
self.assertTrue(
re.search(
'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt,
re.M))
for (key, val) in modextravars.items():
self.assertTrue(
re.search('^setenv\s+%s\s+"%s"$' % (key, val), txt, re.M))
for (key, val) in modextrapaths.items():
self.assertTrue(
re.search('^prepend-path\s+%s\s+\$root/%s$' % (key, val), txt,
re.M))
def test_replaced_easyconfig_parameters(self):
"""Test handling of replaced easyconfig parameters."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
ec = EasyConfig(os.path.join(test_ecs_dir, 'toy-0.0.eb'))
replaced_parameters = {
'license': ('license_file', '2.0'),
'makeopts': ('buildopts', '2.0'),
'premakeopts': ('prebuildopts', '2.0'),
}
for key, (newkey, ver) in replaced_parameters.items():
error_regex = "NO LONGER SUPPORTED since v%s.*'%s' is replaced by '%s'" % (ver, key, newkey)
self.assertErrorRegex(EasyBuildError, error_regex, ec.get, key)
self.assertErrorRegex(EasyBuildError, error_regex, lambda k: ec[k], key)
def foo(key):
ec[key] = 'foo'
self.assertErrorRegex(EasyBuildError, error_regex, foo, key)
def get_buildstats(self, name, ec_version):
"""
return the build statistics
"""
full_path = os.path.join(self.wc, self.subdir, name)
if not os.path.isdir(full_path):
self.log.debug("module (%s) has not been found in the repo" % name)
return []
dest = os.path.join(full_path, "%s.eb" % ec_version)
if not os.path.isfile(dest):
self.log.debug("version %s for %s has not been found in the repo" % (ec_version, name))
return []
eb = EasyConfig(dest, validate=False)
return eb['buildstats']
def test_make_module_dep_hmns(self):
"""Test for make_module_dep under HMNS"""
test_ecs_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.4.10'}",
'dependencies = [',
" ('FFTW', '3.3.3'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
eb.make_builddir()
eb.prepare_step()
# GCC, OpenMPI and hwloc modules should *not* be included in loads for dependencies
mod_dep_txt = eb.make_module_dep()
for mod in ['GCC/4.7.2', 'OpenMPI/1.6.4']:
regex = re.compile('load.*%s' % mod)
self.assertFalse(
regex.search(mod_dep_txt),
"Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
regex = re.compile('load.*FFTW/3.3.3')
self.assertTrue(
regex.search(mod_dep_txt),
"Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
def setUp(self):
""" initialize ModuleGenerator with test Application """
super(ModuleGeneratorTest, self).setUp()
# find .eb file
eb_path = os.path.join(
os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'gzip-1.4.eb')
eb_full_path = find_full_path(eb_path)
self.assertTrue(eb_full_path)
ec = EasyConfig(eb_full_path)
self.eb = EasyBlock(ec)
self.modgen = ModuleGenerator(self.eb)
self.modgen.app.installdir = tempfile.mkdtemp(
prefix='easybuild-modgen-test-')
self.orig_module_naming_scheme = config.get_module_naming_scheme()
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo")
self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ())
self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'})
