def avail_easyblocks(self):
"""Get a class tree for easyblocks."""
detailed = self.options.list_easyblocks == "detailed"
module_regexp = re.compile(r"^([^_].*)\.py$")
# finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
search_toolchain('')
for package in [
"easybuild.easyblocks", "easybuild.easyblocks.generic"
]:
__import__(package)
# determine paths for this package
paths = sys.modules[package].__path__
# import all modules in these paths
for path in paths:
if os.path.exists(path):
for f in os.listdir(path):
res = module_regexp.match(f)
if res:
__import__("%s.%s" % (package, res.group(1)))
def add_class(classes, cls):
"""Add a new class, and all of its subclasses."""
children = cls.__subclasses__()
classes.update({
cls.__name__: {
'module': cls.__module__,
'children': [x.__name__ for x in children]
}
})
for child in children:
add_class(classes, child)
roots = [EasyBlock, Extension]
classes = {}
for root in roots:
add_class(classes, root)
# Print the tree, start with the roots
txt = []
for root in roots:
root = root.__name__
if detailed:
txt.append("%s (%s)" % (root, classes[root]['module']))
else:
txt.append("%s" % root)
if 'children' in classes[root]:
txt.extend(
self.avail_classes_tree(classes, classes[root]['children'],
detailed))
txt.append("")
return "\n".join(txt)
def avail_easyblocks():
"""Return a list of all available easyblocks."""
module_regexp = re.compile(r"^([^_].*)\.py$")
class_regex = re.compile(r"^class ([^(]*)\(", re.M)
# finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
search_toolchain('')
easyblocks = {}
for pkg in ['easybuild.easyblocks', 'easybuild.easyblocks.generic']:
__import__(pkg)
# determine paths for this package
paths = sys.modules[pkg].__path__
# import all modules in these paths
for path in paths:
if os.path.exists(path):
for fn in os.listdir(path):
res = module_regexp.match(fn)
if res:
easyblock_mod_name = '%s.%s' % (pkg, res.group(1))
if easyblock_mod_name not in easyblocks:
__import__(easyblock_mod_name)
easyblock_loc = os.path.join(path, fn)
class_names = class_regex.findall(
read_file(easyblock_loc))
if len(class_names) == 1:
easyblock_class = class_names[0]
elif class_names:
raise EasyBuildError(
"Found multiple class names for easyblock %s: %s",
easyblock_loc, class_names)
else:
raise EasyBuildError(
"Failed to determine easyblock class name for %s",
easyblock_loc)
easyblocks[easyblock_mod_name] = {
'class': easyblock_class,
'loc': easyblock_loc
}
else:
_log.debug(
"%s already imported from %s, ignoring %s",
easyblock_mod_name,
easyblocks[easyblock_mod_name]['loc'], path)
return easyblocks
def avail_easyblocks(self):
"""Get a class tree for easyblocks."""
detailed = self.options.list_easyblocks == "detailed"
module_regexp = re.compile(r"^([^_].*)\.py$")
# finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
search_toolchain('')
for package in ["easybuild.easyblocks", "easybuild.easyblocks.generic"]:
__import__(package)
# determine paths for this package
paths = sys.modules[package].__path__
# import all modules in these paths
for path in paths:
if os.path.exists(path):
for f in os.listdir(path):
res = module_regexp.match(f)
if res:
__import__("%s.%s" % (package, res.group(1)))
def add_class(classes, cls):
"""Add a new class, and all of its subclasses."""
children = cls.__subclasses__()
classes.update({cls.__name__: {
'module': cls.__module__,
'children': [x.__name__ for x in children]
}
})
for child in children:
add_class(classes, child)
roots = [EasyBlock, Extension]
classes = {}
for root in roots:
add_class(classes, root)
# Print the tree, start with the roots
txt = []
for root in roots:
root = root.__name__
if detailed:
txt.append("%s (%s)" % (root, classes[root]['module']))
else:
txt.append("%s" % root)
if 'children' in classes[root]:
txt.extend(self.avail_classes_tree(classes, classes[root]['children'], detailed))
txt.append("")
return "\n".join(txt)
def test_goolfc(self):
"""Test whether goolfc is handled properly."""
tc_class, _ = search_toolchain("goolfc")
tc = tc_class(version="1.3.12")
opts = {'cuda_gencode': ['arch=compute_35,code=sm_35', 'arch=compute_10,code=compute_10']}
tc.set_options(opts)
tc.prepare()
nvcc_flags = r' '.join([
r'-Xcompiler="-O2 -march=native"',
# the use of -lcudart in -Xlinker is a bit silly but hard to avoid
r'-Xlinker=".* -lm -lrt -lcudart -lpthread"',
r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]),
])
self.assertEqual(tc.get_variable('CUDA_CC'), 'nvcc -ccbin="g++"')
self.assertEqual(tc.get_variable('CUDA_CXX'), 'nvcc -ccbin="g++"')
# -L/path flags will not be there if the software installations are not available
val = tc.get_variable('CUDA_CFLAGS')
self.assertTrue(re.compile(nvcc_flags).match(val), "'%s' matches '%s'" % (val, nvcc_flags))
val = tc.get_variable('CUDA_CXXFLAGS')
self.assertTrue(re.compile(nvcc_flags).match(val), "'%s' matches '%s'" % (val, nvcc_flags))
# check compiler prefixes
self.assertEqual(tc.comp_family(prefix='CUDA'), "CUDA")
# check CUDA runtime lib
self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS'))
def test_get_variable_compilers(self):
"""Test get_variable function to obtain compiler variables."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
cc = tc.get_variable('CC')
self.assertEqual(cc, "gcc")
cxx = tc.get_variable('CXX')
self.assertEqual(cxx, "g++")
f77 = tc.get_variable('F77')
self.assertEqual(f77, "gfortran")
f90 = tc.get_variable('F90')
self.assertEqual(f90, "gfortran")
mpicc = tc.get_variable('MPICC')
self.assertEqual(mpicc, "mpicc")
mpicxx = tc.get_variable('MPICXX')
self.assertEqual(mpicxx, "mpicxx")
mpif77 = tc.get_variable('MPIF77')
self.assertEqual(mpif77, "mpif77")
mpif90 = tc.get_variable('MPIF90')
self.assertEqual(mpif90, "mpif90")
ompi_cc = tc.get_variable('OMPI_CC')
self.assertEqual(ompi_cc, "gcc")
ompi_cxx = tc.get_variable('OMPI_CXX')
self.assertEqual(ompi_cxx, "g++")
ompi_f77 = tc.get_variable('OMPI_F77')
self.assertEqual(ompi_f77, "gfortran")
ompi_fc = tc.get_variable('OMPI_FC')
self.assertEqual(ompi_fc, "gfortran")
def is_valid(self):
"""Check if this is a valid ToolchainVersionOperator"""
_, all_tcs = search_toolchain('')
tc_names = [x.NAME for x in all_tcs]
known_tc_name = self.tc_name in tc_names
return known_tc_name and super(ToolchainVersionOperator,
self).is_valid()
def test_optimization_flags_combos(self):
"""Test whether combining optimization levels works as expected."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# check combining of optimization flags (doesn't make much sense)
# lowest optimization should always be picked
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'lowopt': True, 'opt':True})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['lowopt']
self.assertTrue(flag in flags)
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'noopt': True, 'lowopt':True})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt']
self.assertTrue(flag in flags)
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'noopt':True, 'lowopt': True, 'opt':True})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt']
self.assertTrue(flag in flags)
def test_get_variable_compilers(self):
"""Test get_variable function to obtain compiler variables."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
cc = tc.get_variable('CC')
self.assertEqual(cc, "gcc")
cxx = tc.get_variable('CXX')
self.assertEqual(cxx, "g++")
f77 = tc.get_variable('F77')
self.assertEqual(f77, "gfortran")
f90 = tc.get_variable('F90')
self.assertEqual(f90, "gfortran")
mpicc = tc.get_variable('MPICC')
self.assertEqual(mpicc, "mpicc")
mpicxx = tc.get_variable('MPICXX')
self.assertEqual(mpicxx, "mpicxx")
mpif77 = tc.get_variable('MPIF77')
self.assertEqual(mpif77, "mpif77")
mpif90 = tc.get_variable('MPIF90')
self.assertEqual(mpif90, "mpif90")
ompi_cc = tc.get_variable('OMPI_CC')
self.assertEqual(ompi_cc, "gcc")
ompi_cxx = tc.get_variable('OMPI_CXX')
self.assertEqual(ompi_cxx, "g++")
ompi_f77 = tc.get_variable('OMPI_F77')
self.assertEqual(ompi_f77, "gfortran")
ompi_fc = tc.get_variable('OMPI_FC')
self.assertEqual(ompi_fc, "gfortran")
def test_optimization_flags_combos(self):
"""Test whether combining optimization levels works as expected."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# check combining of optimization flags (doesn't make much sense)
# lowest optimization should always be picked
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'lowopt': True, 'opt': True})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['lowopt']
self.assertTrue(flag in flags)
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'noopt': True, 'lowopt': True})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt']
self.assertTrue(flag in flags)
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'noopt': True, 'lowopt': True, 'opt': True})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt']
self.assertTrue(flag in flags)
def test_parser_toolchain_regex(self):
"""Test the ToolchainVersionOperator parser"""
top = ToolchainVersionOperator()
_, tcs = search_toolchain('')
tc_names = [x.NAME for x in tcs]
for tc in tc_names: # test all known toolchain names
# test version expressions with optional version operator
ok_tests = [
("%s >= 1.2.3" % tc, None), # only dict repr for == operator
("%s == 1.2.3" % tc, {'name': tc, 'version': '1.2.3'}),
(tc, None), # only toolchain name, no dict repr (default operator is >=, version is 0.0.0)
]
for txt, as_dict in ok_tests:
self.assertTrue(top.regex.search(txt), "%s matches toolchain section marker regex" % txt)
tcversop = ToolchainVersionOperator(txt)
self.assertTrue(tcversop)
self.assertEqual(tcversop.as_dict(), as_dict)
# only accept known toolchain names
fail_tests = [
"x%s >= 1.2.3" % tc,
"%sx >= 1.2.3" % tc,
"foo",
">= 1.2.3",
]
for txt in fail_tests:
self.assertFalse(top.regex.search(txt), "%s doesn't match toolchain section marker regex" % txt)
tcv = ToolchainVersionOperator(txt)
self.assertEqual(tcv.tc_name, None)
self.assertEqual(tcv.tcversop_str, None)
def test_optimization_flags(self):
"""Test whether optimization flags are being set correctly."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# check default optimization flag (e.g. -O2)
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
self.assertTrue(
tc.COMPILER_SHARED_OPTION_MAP['defaultopt'] in flags)
# check other optimization flags
for opt in ['noopt', 'lowopt', 'opt']:
tc = tc_class(version="1.1.0-no-OFED")
for enable in [True, False]:
tc.set_options({opt: enable})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(
tc.COMPILER_SHARED_OPTION_MAP[opt] in flags)
else:
self.assertTrue(
tc.COMPILER_SHARED_OPTION_MAP[opt] in flags)
def test_parser_toolchain_regex(self):
"""Test the ToolchainVersionOperator parser"""
top = ToolchainVersionOperator()
_, tcs = search_toolchain('')
tc_names = [x.NAME for x in tcs]
for tc in tc_names: # test all known toolchain names
# test version expressions with optional version operator
ok_tests = [
"%s >= 1.2.3" % tc,
"%s 1.2.3" % tc,
tc,
]
for txt in ok_tests:
self.assertTrue(top.regex.search(txt), "%s matches toolchain section marker regex" % txt)
self.assertTrue(ToolchainVersionOperator(txt))
# only accept known toolchain names
fail_tests = [
"x%s >= 1.2.3" % tc,
"%sx >= 1.2.3" % tc,
"foo",
">= 1.2.3",
]
for txt in fail_tests:
self.assertFalse(top.regex.search(txt), "%s doesn't match toolchain section marker regex" % txt)
self.assertFalse(ToolchainVersionOperator(txt))
def test_optimization_flags(self):
"""Test whether optimization flags are being set correctly."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# check default optimization flag (e.g. -O2)
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP['defaultopt'] in flags)
# check other optimization flags
for opt in ['noopt', 'lowopt', 'opt']:
tc = tc_class(version="1.1.0-no-OFED")
for enable in [True, False]:
tc.set_options({opt: enable})
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP[opt] in flags)
else:
self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP[opt] in flags)
def test_precision_flags(self):
"""Test whether precision flags are being set correctly."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# check default precision flag
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
val = ' '.join([
'-%s' % f for f in tc.COMPILER_UNIQUE_OPTION_MAP['defaultprec']
])
self.assertTrue(val in flags)
# check other precision flags
for opt in ['strict', 'precise', 'loose', 'veryloose']:
for enable in [True, False]:
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({opt: enable})
tc.prepare()
val = ' '.join(
['-%s' % f for f in tc.COMPILER_UNIQUE_OPTION_MAP[opt]])
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(val in flags)
else:
self.assertTrue(val not in flags)
def test_precision_flags(self):
"""Test whether precision flags are being set correctly."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# check default precision flag
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
for var in flag_vars:
flags = tc.get_variable(var)
val = ' '.join(['-%s' % f for f in tc.COMPILER_UNIQUE_OPTION_MAP['defaultprec']])
self.assertTrue(val in flags)
# check other precision flags
for opt in ['strict', 'precise', 'loose', 'veryloose']:
for enable in [True, False]:
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({opt: enable})
tc.prepare()
val = ' '.join(['-%s' % f for f in tc.COMPILER_UNIQUE_OPTION_MAP[opt]])
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(val in flags)
else:
self.assertTrue(val not in flags)
def test_comp_family(self):
"""Test determining compiler family."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
self.assertEqual(tc.comp_family(), "GCC")
def test_comp_family(self):
"""Test determining compiler family."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
self.assertEqual(tc.comp_family(), "GCC")
def disable_use_mpi_for_non_mpi_toolchains(ec):
toolchain = ec.get('toolchain')
toolchain_class, _ = search_toolchain(toolchain['name'])
if ec['toolchainopts'] and ec['toolchainopts'].get(
'usempi') and not toolchain_class(
version=toolchain['version']).mpi_family():
print(
"usempi option found, but using a non-MPI toolchain. Disabling it")
del ec['toolchainopts']['usempi']
print("New toolchainopts:%s" % str(ec['toolchainopts']))
def test_get_variable_libs_list(self):
"""Test get_variable function to obtain list of libraries."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
ldflags = tc.get_variable('LDFLAGS', typ=list)
self.assertTrue(isinstance(ldflags, list))
if len(ldflags) > 0:
self.assertTrue(isinstance(ldflags[0], basestring))
def avail_easyblocks():
"""Return a list of all available easyblocks."""
module_regexp = re.compile(r"^([^_].*)\.py$")
class_regex = re.compile(r"^class ([^(]*)\(", re.M)
# finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
search_toolchain('')
easyblocks = {}
for pkg in ['easybuild.easyblocks', 'easybuild.easyblocks.generic']:
__import__(pkg)
# determine paths for this package
paths = sys.modules[pkg].__path__
# import all modules in these paths
for path in paths:
if os.path.exists(path):
for fn in os.listdir(path):
res = module_regexp.match(fn)
if res:
easyblock_mod_name = '%s.%s' % (pkg, res.group(1))
if easyblock_mod_name not in easyblocks:
__import__(easyblock_mod_name)
easyblock_loc = os.path.join(path, fn)
class_names = class_regex.findall(read_file(easyblock_loc))
if len(class_names) == 1:
easyblock_class = class_names[0]
elif class_names:
raise EasyBuildError("Found multiple class names for easyblock %s: %s",
easyblock_loc, class_names)
else:
raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc)
easyblocks[easyblock_mod_name] = {'class': easyblock_class, 'loc': easyblock_loc}
else:
_log.debug("%s already imported from %s, ignoring %s",
easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path)
return easyblocks
def test_get_variable_libs_list(self):
"""Test get_variable function to obtain list of libraries."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
ldflags = tc.get_variable('LDFLAGS', typ=list)
self.assertTrue(isinstance(ldflags, list))
if len(ldflags) > 0:
self.assertTrue(isinstance(ldflags[0], basestring))
def list_toolchains(output_format=FORMAT_TXT):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
tcs = dict()
for (tcname, tcc) in tclist:
tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there
tcs[tcname] = tc.definition()
return generate_doc('list_toolchains_%s' % output_format, [tcs])
def list_toolchains(output_format=FORMAT_TXT):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
tcs = dict()
for (tcname, tcc) in tclist:
tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there
tcs[tcname] = tc.definition()
return generate_doc('list_toolchains_%s' % output_format, [tcs])
def set_modaltsoftname(ec):
if ec['name'] in modaltsoftnames:
ec['modaltsoftname'] = modaltsoftnames[ec['name']]
# add -mpi to module name for various modules with both -mpi and no-mpi varieties
toolchain = ec.get('toolchain')
toolchain_class, _ = search_toolchain(toolchain['name'])
if (ec['name'].lower() in mpi_modaltsoftname
and (toolchain_class(version=toolchain['version']).mpi_family() or
(ec['toolchainopts'] and ec['toolchainopts'].get('usempi')))):
ec['modaltsoftname'] = ec['name'].lower() + '-mpi'
ec['versionsuffix'] = '-mpi'
def test_cgoolf_toolchain(self):
"""Test for cgoolf toolchain."""
name = "cgoolf"
tc_class, _ = search_toolchain(name)
self.assertEqual(tc_class.NAME, name)
tc = tc_class(version="1.1.6")
tc.prepare()
self.assertEqual(tc.get_variable('CC'), 'clang')
self.assertEqual(tc.get_variable('CXX'), 'clang++')
self.assertEqual(tc.get_variable('F77'), 'gfortran')
self.assertEqual(tc.get_variable('F90'), 'gfortran')
def test_cgoolf_toolchain(self):
"""Test for cgoolf toolchain."""
name = "cgoolf"
tc_class, _ = search_toolchain(name)
self.assertEqual(tc_class.NAME, name)
tc = tc_class(version="1.1.6")
tc.prepare()
self.assertEqual(tc.get_variable('CC'), 'clang')
self.assertEqual(tc.get_variable('CXX'), 'clang++')
self.assertEqual(tc.get_variable('F77'), 'gfortran')
self.assertEqual(tc.get_variable('F90'), 'gfortran')
def _get_all_toolchain_names(self, search_string=''):
"""
Initialise each search_toolchain request, save in module constant TOOLCHAIN_NAMES.
@param search_string: passed to search_toolchain function.
"""
global TOOLCHAIN_NAMES
if not search_string in TOOLCHAIN_NAMES:
_, all_tcs = search_toolchain(search_string)
self.log.debug('Found all toolchains for "%s" to %s' % (search_string, all_tcs))
TOOLCHAIN_NAMES[search_string] = [x.NAME for x in all_tcs]
self.log.debug('Set TOOLCHAIN_NAMES for "%s" to %s' % (search_string, TOOLCHAIN_NAMES[search_string]))
return TOOLCHAIN_NAMES[search_string]
def setUp(self):
"""Set some convenience attributes"""
super(TestEBConfigObj, self).setUp()
_, tcs = search_toolchain('')
self.tc_names = [x.NAME for x in tcs]
self.tcmax = min(len(self.tc_names), 3)
if len(self.tc_names) < self.tcmax:
self.tcmax = len(self.tc_names)
self.tc_namesmax = self.tc_names[:self.tcmax]
self.tc_first = self.tc_names[0]
self.tc_last = self.tc_names[-1]
self.tc_lastmax = self.tc_namesmax[-1]
def avail_toolchains(self):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
txt = ["List of known toolchains (toolchainname: module[,module...]):"]
for (tcname, tcc) in tclist:
tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there
tc_elems = nub(sorted([e for es in tc.definition().values() for e in es]))
txt.append("\t%s: %s" % (tcname, ', '.join(tc_elems)))
return '\n'.join(txt)
def setUp(self):
"""Set some convenience attributes"""
super(TestEBConfigObj, self).setUp()
_, tcs = search_toolchain('')
self.tc_names = [x.NAME for x in tcs]
self.tcmax = min(len(self.tc_names), 3)
if len(self.tc_names) < self.tcmax:
self.tcmax = len(self.tc_names)
self.tc_namesmax = self.tc_names[:self.tcmax]
self.tc_first = self.tc_names[0]
self.tc_last = self.tc_names[-1]
self.tc_lastmax = self.tc_namesmax[-1]
def avail_toolchains(self):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain("")
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
txt = ["List of known toolchains (toolchainname: module[,module...]):"]
for (tcname, tcc) in tclist:
tc = tcc(version="1.2.3") # version doesn't matter here, but something needs to be there
tc_elems = nub(sorted([e for es in tc.definition().values() for e in es]))
txt.append("\t%s: %s" % (tcname, ", ".join(tc_elems)))
return "\n".join(txt)
def test_get_variable_seq_compilers(self):
"""Test get_variable function to obtain compiler variables."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'usempi': True})
tc.prepare()
cc_seq = tc.get_variable('CC_SEQ')
self.assertEqual(cc_seq, "gcc")
cxx_seq = tc.get_variable('CXX_SEQ')
self.assertEqual(cxx_seq, "g++")
f77_seq = tc.get_variable('F77_SEQ')
self.assertEqual(f77_seq, "gfortran")
f90_seq = tc.get_variable('F90_SEQ')
self.assertEqual(f90_seq, "gfortran")
def avail_toolchains(self):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
txt = ["List of known toolchains (toolchainname: module[,module...]):"]
for (tcname, tcc) in tclist:
tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there
tc_elems = set([y for x in dir(tc) if x.endswith('_MODULE_NAME') for y in eval("tc.%s" % x)])
txt.append("\t%s: %s" % (tcname, ', '.join(sorted(tc_elems))))
return '\n'.join(txt)
def test_get_variable_seq_compilers(self):
"""Test get_variable function to obtain compiler variables."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'usempi': True})
tc.prepare()
cc_seq = tc.get_variable('CC_SEQ')
self.assertEqual(cc_seq, "gcc")
cxx_seq = tc.get_variable('CXX_SEQ')
self.assertEqual(cxx_seq, "g++")
f77_seq = tc.get_variable('F77_SEQ')
self.assertEqual(f77_seq, "gfortran")
f90_seq = tc.get_variable('F90_SEQ')
self.assertEqual(f90_seq, "gfortran")
def avail_toolchain_opts(name, output_format=FORMAT_TXT):
"""Show list of known options for given toolchain."""
tc_class, _ = search_toolchain(name)
if not tc_class:
raise EasyBuildError("Couldn't find toolchain: '%s'. To see available toolchains, use --list-toolchains" % name)
tc = tc_class(version='1.0') # version doesn't matter here, but needs to be defined
tc_dict = {}
for cst in ['COMPILER_SHARED_OPTS', 'COMPILER_UNIQUE_OPTS', 'MPI_SHARED_OPTS', 'MPI_UNIQUE_OPTS']:
if hasattr(tc, cst):
opts = getattr(tc, cst)
if opts is not None:
tc_dict.update(opts)
return generate_doc('avail_toolchain_opts_%s' % output_format, [name, tc_dict])
def avail_toolchains(self):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain("")
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
txt = ["List of known toolchains (toolchainname: module[,module...]):"]
for (tcname, tcc) in tclist:
tc = tcc(version="1.2.3") # version doesn't matter here, but something needs to be there
tc_elems = set([y for x in dir(tc) if x.endswith("_MODULE_NAME") for y in eval("tc.%s" % x)])
txt.append("\t%s: %s" % (tcname, ", ".join(sorted(tc_elems))))
return "\n".join(txt)
def avail_toolchain_opts(name, output_format=FORMAT_TXT):
"""Show list of known options for given toolchain."""
tc_class, _ = search_toolchain(name)
if not tc_class:
raise EasyBuildError("Couldn't find toolchain: '%s'. To see available toolchains, use --list-toolchains" % name)
tc = tc_class(version='1.0') # version doesn't matter here, but needs to be defined
tc_dict = {}
for cst in ['COMPILER_SHARED_OPTS', 'COMPILER_UNIQUE_OPTS', 'MPI_SHARED_OPTS', 'MPI_UNIQUE_OPTS']:
if hasattr(tc, cst):
opts = getattr(tc, cst)
if opts is not None:
tc_dict.update(opts)
return generate_doc('avail_toolchain_opts_%s' % output_format, [name, tc_dict])
def versop_regex(self):
"""
Create the regular expression for toolchain support of format ^<toolchain> <versop_expr>$ ,
with <toolchain> the name of one of the supported toolchains and <versop_expr> in '<operator> <version>' syntax
"""
_, all_tcs = search_toolchain('')
tc_names = [x.NAME for x in all_tcs]
self.log.debug("found toolchain names %s" % tc_names)
versop_regex = super(ToolchainVersionOperator, self).versop_regex(begin_end=False)
versop_pattern = r'(?P<versop_str>%s)' % versop_regex.pattern
tc_names_regex = r'(?P<tc_name>(?:%s))' % '|'.join(tc_names)
tc_regex = re.compile(r'^%s(?:%s%s)?$' % (tc_names_regex, self.SEPARATOR, versop_pattern))
self.log.debug("toolchain versop regex pattern %s " % tc_regex.pattern)
return tc_regex
def list_toolchains(output_format=FORMAT_TXT):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
# filter deprecated 'dummy' toolchain
all_tcs = [x for x in all_tcs if x.NAME != DUMMY_TOOLCHAIN_NAME]
all_tcs_names = [x.NAME for x in all_tcs]
# start with dict that maps toolchain name to corresponding subclass of Toolchain
tcs = dict(zip(all_tcs_names, all_tcs))
for tcname in sorted(tcs):
tcc = tcs[tcname]
tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there
tcs[tcname] = tc.definition()
return generate_doc('list_toolchains_%s' % output_format, [tcs])
def versop_regex(self):
"""
Create the regular expression for toolchain support of format ^<toolchain> <versop_expr>$ ,
with <toolchain> the name of one of the supported toolchains and <versop_expr> in '<operator> <version>' syntax
"""
_, all_tcs = search_toolchain('')
tc_names = [x.NAME for x in all_tcs]
self.log.debug("found toolchain names %s" % tc_names)
versop_regex = super(ToolchainVersionOperator,
self).versop_regex(begin_end=False)
versop_pattern = r'(?P<versop_str>%s)' % versop_regex.pattern
tc_names_regex = r'(?P<tc_name>(?:%s))' % '|'.join(tc_names)
tc_regex = re.compile(r'^%s(?:%s%s)?$' %
(tc_names_regex, self.SEPARATOR, versop_pattern))
self.log.debug("toolchain versop regex pattern %s " % tc_regex.pattern)
return tc_regex
def toolchain(self):
"""
returns the Toolchain used
"""
if self._toolchain:
return self._toolchain
tcname = self['toolchain']['name']
tc, all_tcs = search_toolchain(tcname)
if not tc:
all_tcs_names = ",".join([x.NAME for x in all_tcs])
self.log.error("Toolchain %s not found, available toolchains: %s" % (tcname, all_tcs_names))
tc = tc(version=self['toolchain']['version'])
if self['toolchainopts']:
tc.set_options(self['toolchainopts'])
self._toolchain = tc
return self._toolchain
def test_toolchain_versop_test(self):
"""Test the ToolchainVersionOperator test"""
_, tcs = search_toolchain('')
tc_names = [x.NAME for x in tcs]
for tc in tc_names: # test all known toolchain names
# test version expressions with optional version operator
tests = [
("%s >= 1.2.3" % tc, (
(tc, '1.2.3', True), # version ok, name ok
(tc, '1.2.4', True), # version ok, name ok
(tc, '1.2.2', False), # version not ok, name ok
('x' + tc, '1.2.3', False), # version ok, name not ok
('x' + tc, '1.2.2', False), # version not ok, name not ok
)),
]
for txt, subtests in tests:
tcversop = ToolchainVersionOperator(txt)
for name, version, res in subtests:
self.assertEqual(tcversop.test(name, version), res)
def test_toolchain_versop_test(self):
"""Test the ToolchainVersionOperator test"""
_, tcs = search_toolchain('')
tc_names = [x.NAME for x in tcs]
for tc in tc_names: # test all known toolchain names
# test version expressions with optional version operator
tests = [
("%s >= 1.2.3" % tc, (
(tc, '1.2.3', True), # version ok, name ok
(tc, '1.2.4', True), # version ok, name ok
(tc, '1.2.2', False), # version not ok, name ok
('x' + tc, '1.2.3', False), # version ok, name not ok
('x' + tc, '1.2.2', False), # version not ok, name not ok
)),
]
for txt, subtests in tests:
tcversop = ToolchainVersionOperator(txt)
for name, version, res in subtests:
self.assertEqual(tcversop.test(name, version), res)
def test_misc_flags_unique_fortran(self):
"""Test whether unique Fortran compiler flags are set correctly."""
flag_vars = ['FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# setting option should result in corresponding flag to be set (Fortran unique options)
for opt in ['i8', 'r8']:
for enable in [True, False]:
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({opt: enable})
tc.prepare()
flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt]
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags))
else:
self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags))
def test_misc_flags_shared(self):
"""Test whether shared compiler flags are set correctly."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# setting option should result in corresponding flag to be set (shared options)
for opt in ['pic', 'verbose', 'debug', 'static', 'shared']:
for enable in [True, False]:
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({opt: enable})
tc.prepare()
# we need to make sure we check for flags, not letter (e.g. 'v' vs '-v')
flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP[opt]
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags))
else:
self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags))
def test_configobj(self):
"""Test configobj sort"""
_, tcs = search_toolchain('')
tc_names = [x.NAME for x in tcs]
tcmax = min(len(tc_names), 3)
if len(tc_names) < tcmax:
tcmax = len(tc_names)
tc = tc_names[0]
configobj_txt = [
'[DEFAULT]',
'toolchains=%s >= 7.8.9' % ','.join(tc_names[:tcmax]),
'versions=1.2.3,2.3.4,3.4.5',
'[>= 2.3.4]',
'foo=bar',
'[== 3.4.5]',
'baz=biz',
'[!= %s 5.6.7]' % tc,
'[%s > 7.8.9]' % tc_names[tcmax - 1],
]
co = ConfigObj(configobj_txt)
cov = ConfigObjVersion()
def toolchain(self):
"""
returns the Toolchain used
"""
if self._toolchain:
return self._toolchain
tcname = self['toolchain']['name']
tc, all_tcs = search_toolchain(tcname)
if not tc:
all_tcs_names = ",".join([x.NAME for x in all_tcs])
self.log.error("Toolchain %s not found, available toolchains: %s" % (tcname, all_tcs_names))
tc = tc(version=self['toolchain']['version'])
if self['toolchainopts'] is None:
# set_options should always be called, even if no toolchain options are specified
# this is required to set the default options
tc.set_options({})
else:
tc.set_options(self['toolchainopts'])
self._toolchain = tc
return self._toolchain
def toolchain(self):
"""
returns the Toolchain used
"""
if self._toolchain:
return self._toolchain
tcname = self['toolchain']['name']
tc, all_tcs = search_toolchain(tcname)
if not tc:
all_tcs_names = ",".join([x.NAME for x in all_tcs])
self.log.error("Toolchain %s not found, available toolchains: %s" % (tcname, all_tcs_names))
tc = tc(version=self['toolchain']['version'])
if self['toolchainopts'] is None:
# set_options should always be called, even if no toolchain options are specified
# this is required to set the default options
tc.set_options({})
else:
tc.set_options(self['toolchainopts'])
self._toolchain = tc
return self._toolchain
def test_validate_pass_by_value(self):
"""
Check that elements of variables are passed by value, not by reference,
which is required to ensure correctness.
"""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
pass_by_value = True
ids = []
for k, v in tc.variables.items():
for x in v:
idx = id(x)
if not idx in ids:
ids.append(idx)
else:
pass_by_value = False
break
if not pass_by_value:
break
self.assertTrue(pass_by_value)
def test_get_variable_mpi_compilers(self):
"""Test get_variable function to obtain compiler variables."""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({'usempi': True})
tc.prepare()
cc = tc.get_variable('CC')
self.assertEqual(cc, "mpicc")
cxx = tc.get_variable('CXX')
self.assertEqual(cxx, "mpicxx")
f77 = tc.get_variable('F77')
self.assertEqual(f77, "mpif77")
f90 = tc.get_variable('F90')
self.assertEqual(f90, "mpif90")
mpicc = tc.get_variable('MPICC')
self.assertEqual(mpicc, "mpicc")
mpicxx = tc.get_variable('MPICXX')
self.assertEqual(mpicxx, "mpicxx")
mpif77 = tc.get_variable('MPIF77')
self.assertEqual(mpif77, "mpif77")
mpif90 = tc.get_variable('MPIF90')
self.assertEqual(mpif90, "mpif90")
def test_validate_pass_by_value(self):
"""
Check that elements of variables are passed by value, not by reference,
which is required to ensure correctness.
"""
tc_class, _ = search_toolchain("goalf")
tc = tc_class(version="1.1.0-no-OFED")
tc.prepare()
pass_by_value = True
ids = []
for k, v in tc.variables.items():
for x in v:
idx = id(x)
if not idx in ids:
ids.append(idx)
else:
pass_by_value = False
break
if not pass_by_value:
break
self.assertTrue(pass_by_value)
def test_misc_flags_unique_fortran(self):
"""Test whether unique Fortran compiler flags are set correctly."""
flag_vars = ['FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# setting option should result in corresponding flag to be set (Fortran unique options)
for opt in ['i8', 'r8']:
for enable in [True, False]:
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({opt: enable})
tc.prepare()
flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt]
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(
flag in flags,
"%s: True means %s in %s" % (opt, flag, flags))
else:
self.assertTrue(
flag not in flags,
"%s: False means no %s in %s" % (opt, flag, flags))
def test_misc_flags_shared(self):
"""Test whether shared compiler flags are set correctly."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
tc_class, _ = search_toolchain("goalf")
# setting option should result in corresponding flag to be set (shared options)
for opt in ['pic', 'verbose', 'debug', 'static', 'shared']:
for enable in [True, False]:
tc = tc_class(version="1.1.0-no-OFED")
tc.set_options({opt: enable})
tc.prepare()
# we need to make sure we check for flags, not letter (e.g. 'v' vs '-v')
flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP[opt]
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(
flag in flags,
"%s: True means %s in %s" % (opt, flag, flags))
else:
self.assertTrue(
flag not in flags,
"%s: False means no %s in %s" % (opt, flag, flags))
def test_goolfc(self):
"""Test whether goolfc is handled properly."""
tc_class, _ = search_toolchain("goolfc")
tc = tc_class(version="1.3.12")
opts = {
'cuda_gencode':
['arch=compute_35,code=sm_35', 'arch=compute_10,code=compute_10']
}
tc.set_options(opts)
tc.prepare()
nvcc_flags = r' '.join([
r'-Xcompiler="-O2 -march=native"',
# the use of -lcudart in -Xlinker is a bit silly but hard to avoid
r'-Xlinker=".* -lm -lcudart -lpthread"',
r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]),
])
self.assertEqual(tc.get_variable('CUDA_CC'), 'nvcc -ccbin="g++"')
self.assertEqual(tc.get_variable('CUDA_CXX'), 'nvcc -ccbin="g++"')
# -L/path flags will not be there if the software installations are not available
val = tc.get_variable('CUDA_CFLAGS')
self.assertTrue(
re.compile(nvcc_flags).match(val),
"'%s' matches '%s'" % (val, nvcc_flags))
val = tc.get_variable('CUDA_CXXFLAGS')
self.assertTrue(
re.compile(nvcc_flags).match(val),
"'%s' matches '%s'" % (val, nvcc_flags))
# check compiler prefixes
self.assertEqual(tc.comp_family(prefix='CUDA'), "CUDA")
# check CUDA runtime lib
self.assertTrue("-lcudart" in tc.get_variable('LIBS'))
def test_unknown_toolchain(self):
"""Test search_toolchain function for not available toolchains."""
tc, all_tcs = search_toolchain("NOSUCHTOOLKIT")
self.assertEqual(tc, None)
self.assertTrue(len(all_tcs) > 0) # list of available toolchains
def get_toolchain(self, name, version=None):
"""Get a toolchain object instance to test with."""
tc_class, _ = search_toolchain(name)
self.assertEqual(tc_class.NAME, name)
tc = tc_class(version=version, mns=ActiveMNS())
return tc
