def avail_easyblocks(self): """Get a class tree for easyblocks.""" detailed = self.options.list_easyblocks == "detailed" module_regexp = re.compile(r"^([^_].*)\.py$") # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants) search_toolchain('') for package in [ "easybuild.easyblocks", "easybuild.easyblocks.generic" ]: __import__(package) # determine paths for this package paths = sys.modules[package].__path__ # import all modules in these paths for path in paths: if os.path.exists(path): for f in os.listdir(path): res = module_regexp.match(f) if res: __import__("%s.%s" % (package, res.group(1))) def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() classes.update({ cls.__name__: { 'module': cls.__module__, 'children': [x.__name__ for x in children] } }) for child in children: add_class(classes, child) roots = [EasyBlock, Extension] classes = {} for root in roots: add_class(classes, root) # Print the tree, start with the roots txt = [] for root in roots: root = root.__name__ if detailed: txt.append("%s (%s)" % (root, classes[root]['module'])) else: txt.append("%s" % root) if 'children' in classes[root]: txt.extend( self.avail_classes_tree(classes, classes[root]['children'], detailed)) txt.append("") return "\n".join(txt)
def avail_easyblocks(): """Return a list of all available easyblocks.""" module_regexp = re.compile(r"^([^_].*)\.py$") class_regex = re.compile(r"^class ([^(]*)\(", re.M) # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants) search_toolchain('') easyblocks = {} for pkg in ['easybuild.easyblocks', 'easybuild.easyblocks.generic']: __import__(pkg) # determine paths for this package paths = sys.modules[pkg].__path__ # import all modules in these paths for path in paths: if os.path.exists(path): for fn in os.listdir(path): res = module_regexp.match(fn) if res: easyblock_mod_name = '%s.%s' % (pkg, res.group(1)) if easyblock_mod_name not in easyblocks: __import__(easyblock_mod_name) easyblock_loc = os.path.join(path, fn) class_names = class_regex.findall( read_file(easyblock_loc)) if len(class_names) == 1: easyblock_class = class_names[0] elif class_names: raise EasyBuildError( "Found multiple class names for easyblock %s: %s", easyblock_loc, class_names) else: raise EasyBuildError( "Failed to determine easyblock class name for %s", easyblock_loc) easyblocks[easyblock_mod_name] = { 'class': easyblock_class, 'loc': easyblock_loc } else: _log.debug( "%s already imported from %s, ignoring %s", easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path) return easyblocks
def avail_easyblocks(self): """Get a class tree for easyblocks.""" detailed = self.options.list_easyblocks == "detailed" module_regexp = re.compile(r"^([^_].*)\.py$") # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants) search_toolchain('') for package in ["easybuild.easyblocks", "easybuild.easyblocks.generic"]: __import__(package) # determine paths for this package paths = sys.modules[package].__path__ # import all modules in these paths for path in paths: if os.path.exists(path): for f in os.listdir(path): res = module_regexp.match(f) if res: __import__("%s.%s" % (package, res.group(1))) def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() classes.update({cls.__name__: { 'module': cls.__module__, 'children': [x.__name__ for x in children] } }) for child in children: add_class(classes, child) roots = [EasyBlock, Extension] classes = {} for root in roots: add_class(classes, root) # Print the tree, start with the roots txt = [] for root in roots: root = root.__name__ if detailed: txt.append("%s (%s)" % (root, classes[root]['module'])) else: txt.append("%s" % root) if 'children' in classes[root]: txt.extend(self.avail_classes_tree(classes, classes[root]['children'], detailed)) txt.append("") return "\n".join(txt)
def test_goolfc(self): """Test whether goolfc is handled properly.""" tc_class, _ = search_toolchain("goolfc") tc = tc_class(version="1.3.12") opts = {'cuda_gencode': ['arch=compute_35,code=sm_35', 'arch=compute_10,code=compute_10']} tc.set_options(opts) tc.prepare() nvcc_flags = r' '.join([ r'-Xcompiler="-O2 -march=native"', # the use of -lcudart in -Xlinker is a bit silly but hard to avoid r'-Xlinker=".* -lm -lrt -lcudart -lpthread"', r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]), ]) self.assertEqual(tc.get_variable('CUDA_CC'), 'nvcc -ccbin="g++"') self.assertEqual(tc.get_variable('CUDA_CXX'), 'nvcc -ccbin="g++"') # -L/path flags will not be there if the software installations are not available val = tc.get_variable('CUDA_CFLAGS') self.assertTrue(re.compile(nvcc_flags).match(val), "'%s' matches '%s'" % (val, nvcc_flags)) val = tc.get_variable('CUDA_CXXFLAGS') self.assertTrue(re.compile(nvcc_flags).match(val), "'%s' matches '%s'" % (val, nvcc_flags)) # check compiler prefixes self.assertEqual(tc.comp_family(prefix='CUDA'), "CUDA") # check CUDA runtime lib self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS'))
def test_get_variable_compilers(self): """Test get_variable function to obtain compiler variables.""" tc_class, _ = search_toolchain("goalf") tc = tc_class(version="1.1.0-no-OFED") tc.prepare() cc = tc.get_variable('CC') self.assertEqual(cc, "gcc") cxx = tc.get_variable('CXX') self.assertEqual(cxx, "g++") f77 = tc.get_variable('F77') self.assertEqual(f77, "gfortran") f90 = tc.get_variable('F90') self.assertEqual(f90, "gfortran") mpicc = tc.get_variable('MPICC') self.assertEqual(mpicc, "mpicc") mpicxx = tc.get_variable('MPICXX') self.assertEqual(mpicxx, "mpicxx") mpif77 = tc.get_variable('MPIF77') self.assertEqual(mpif77, "mpif77") mpif90 = tc.get_variable('MPIF90') self.assertEqual(mpif90, "mpif90") ompi_cc = tc.get_variable('OMPI_CC') self.assertEqual(ompi_cc, "gcc") ompi_cxx = tc.get_variable('OMPI_CXX') self.assertEqual(ompi_cxx, "g++") ompi_f77 = tc.get_variable('OMPI_F77') self.assertEqual(ompi_f77, "gfortran") ompi_fc = tc.get_variable('OMPI_FC') self.assertEqual(ompi_fc, "gfortran")
def is_valid(self): """Check if this is a valid ToolchainVersionOperator""" _, all_tcs = search_toolchain('') tc_names = [x.NAME for x in all_tcs] known_tc_name = self.tc_name in tc_names return known_tc_name and super(ToolchainVersionOperator, self).is_valid()
def test_optimization_flags_combos(self): """Test whether combining optimization levels works as expected.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # check combining of optimization flags (doesn't make much sense) # lowest optimization should always be picked tc = tc_class(version="1.1.0-no-OFED") tc.set_options({'lowopt': True, 'opt':True}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['lowopt'] self.assertTrue(flag in flags) tc = tc_class(version="1.1.0-no-OFED") tc.set_options({'noopt': True, 'lowopt':True}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt'] self.assertTrue(flag in flags) tc = tc_class(version="1.1.0-no-OFED") tc.set_options({'noopt':True, 'lowopt': True, 'opt':True}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt'] self.assertTrue(flag in flags)
def test_optimization_flags_combos(self): """Test whether combining optimization levels works as expected.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # check combining of optimization flags (doesn't make much sense) # lowest optimization should always be picked tc = tc_class(version="1.1.0-no-OFED") tc.set_options({'lowopt': True, 'opt': True}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['lowopt'] self.assertTrue(flag in flags) tc = tc_class(version="1.1.0-no-OFED") tc.set_options({'noopt': True, 'lowopt': True}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt'] self.assertTrue(flag in flags) tc = tc_class(version="1.1.0-no-OFED") tc.set_options({'noopt': True, 'lowopt': True, 'opt': True}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt'] self.assertTrue(flag in flags)
def test_parser_toolchain_regex(self): """Test the ToolchainVersionOperator parser""" top = ToolchainVersionOperator() _, tcs = search_toolchain('') tc_names = [x.NAME for x in tcs] for tc in tc_names: # test all known toolchain names # test version expressions with optional version operator ok_tests = [ ("%s >= 1.2.3" % tc, None), # only dict repr for == operator ("%s == 1.2.3" % tc, {'name': tc, 'version': '1.2.3'}), (tc, None), # only toolchain name, no dict repr (default operator is >=, version is 0.0.0) ] for txt, as_dict in ok_tests: self.assertTrue(top.regex.search(txt), "%s matches toolchain section marker regex" % txt) tcversop = ToolchainVersionOperator(txt) self.assertTrue(tcversop) self.assertEqual(tcversop.as_dict(), as_dict) # only accept known toolchain names fail_tests = [ "x%s >= 1.2.3" % tc, "%sx >= 1.2.3" % tc, "foo", ">= 1.2.3", ] for txt in fail_tests: self.assertFalse(top.regex.search(txt), "%s doesn't match toolchain section marker regex" % txt) tcv = ToolchainVersionOperator(txt) self.assertEqual(tcv.tc_name, None) self.assertEqual(tcv.tcversop_str, None)
def test_optimization_flags(self): """Test whether optimization flags are being set correctly.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # check default optimization flag (e.g. -O2) tc = tc_class(version="1.1.0-no-OFED") tc.set_options({}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) self.assertTrue( tc.COMPILER_SHARED_OPTION_MAP['defaultopt'] in flags) # check other optimization flags for opt in ['noopt', 'lowopt', 'opt']: tc = tc_class(version="1.1.0-no-OFED") for enable in [True, False]: tc.set_options({opt: enable}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue( tc.COMPILER_SHARED_OPTION_MAP[opt] in flags) else: self.assertTrue( tc.COMPILER_SHARED_OPTION_MAP[opt] in flags)
def test_parser_toolchain_regex(self): """Test the ToolchainVersionOperator parser""" top = ToolchainVersionOperator() _, tcs = search_toolchain('') tc_names = [x.NAME for x in tcs] for tc in tc_names: # test all known toolchain names # test version expressions with optional version operator ok_tests = [ "%s >= 1.2.3" % tc, "%s 1.2.3" % tc, tc, ] for txt in ok_tests: self.assertTrue(top.regex.search(txt), "%s matches toolchain section marker regex" % txt) self.assertTrue(ToolchainVersionOperator(txt)) # only accept known toolchain names fail_tests = [ "x%s >= 1.2.3" % tc, "%sx >= 1.2.3" % tc, "foo", ">= 1.2.3", ] for txt in fail_tests: self.assertFalse(top.regex.search(txt), "%s doesn't match toolchain section marker regex" % txt) self.assertFalse(ToolchainVersionOperator(txt))
def test_optimization_flags(self): """Test whether optimization flags are being set correctly.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # check default optimization flag (e.g. -O2) tc = tc_class(version="1.1.0-no-OFED") tc.set_options({}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP['defaultopt'] in flags) # check other optimization flags for opt in ['noopt', 'lowopt', 'opt']: tc = tc_class(version="1.1.0-no-OFED") for enable in [True, False]: tc.set_options({opt: enable}) tc.prepare() for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP[opt] in flags) else: self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP[opt] in flags)
def test_precision_flags(self): """Test whether precision flags are being set correctly.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # check default precision flag tc = tc_class(version="1.1.0-no-OFED") tc.prepare() for var in flag_vars: flags = tc.get_variable(var) val = ' '.join([ '-%s' % f for f in tc.COMPILER_UNIQUE_OPTION_MAP['defaultprec'] ]) self.assertTrue(val in flags) # check other precision flags for opt in ['strict', 'precise', 'loose', 'veryloose']: for enable in [True, False]: tc = tc_class(version="1.1.0-no-OFED") tc.set_options({opt: enable}) tc.prepare() val = ' '.join( ['-%s' % f for f in tc.COMPILER_UNIQUE_OPTION_MAP[opt]]) for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue(val in flags) else: self.assertTrue(val not in flags)
def test_precision_flags(self): """Test whether precision flags are being set correctly.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # check default precision flag tc = tc_class(version="1.1.0-no-OFED") tc.prepare() for var in flag_vars: flags = tc.get_variable(var) val = ' '.join(['-%s' % f for f in tc.COMPILER_UNIQUE_OPTION_MAP['defaultprec']]) self.assertTrue(val in flags) # check other precision flags for opt in ['strict', 'precise', 'loose', 'veryloose']: for enable in [True, False]: tc = tc_class(version="1.1.0-no-OFED") tc.set_options({opt: enable}) tc.prepare() val = ' '.join(['-%s' % f for f in tc.COMPILER_UNIQUE_OPTION_MAP[opt]]) for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue(val in flags) else: self.assertTrue(val not in flags)
def test_comp_family(self): """Test determining compiler family.""" tc_class, _ = search_toolchain("goalf") tc = tc_class(version="1.1.0-no-OFED") tc.prepare() self.assertEqual(tc.comp_family(), "GCC")
def disable_use_mpi_for_non_mpi_toolchains(ec): toolchain = ec.get('toolchain') toolchain_class, _ = search_toolchain(toolchain['name']) if ec['toolchainopts'] and ec['toolchainopts'].get( 'usempi') and not toolchain_class( version=toolchain['version']).mpi_family(): print( "usempi option found, but using a non-MPI toolchain. Disabling it") del ec['toolchainopts']['usempi'] print("New toolchainopts:%s" % str(ec['toolchainopts']))
def test_get_variable_libs_list(self): """Test get_variable function to obtain list of libraries.""" tc_class, _ = search_toolchain("goalf") tc = tc_class(version="1.1.0-no-OFED") tc.prepare() ldflags = tc.get_variable('LDFLAGS', typ=list) self.assertTrue(isinstance(ldflags, list)) if len(ldflags) > 0: self.assertTrue(isinstance(ldflags[0], basestring))
def avail_easyblocks(): """Return a list of all available easyblocks.""" module_regexp = re.compile(r"^([^_].*)\.py$") class_regex = re.compile(r"^class ([^(]*)\(", re.M) # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants) search_toolchain('') easyblocks = {} for pkg in ['easybuild.easyblocks', 'easybuild.easyblocks.generic']: __import__(pkg) # determine paths for this package paths = sys.modules[pkg].__path__ # import all modules in these paths for path in paths: if os.path.exists(path): for fn in os.listdir(path): res = module_regexp.match(fn) if res: easyblock_mod_name = '%s.%s' % (pkg, res.group(1)) if easyblock_mod_name not in easyblocks: __import__(easyblock_mod_name) easyblock_loc = os.path.join(path, fn) class_names = class_regex.findall(read_file(easyblock_loc)) if len(class_names) == 1: easyblock_class = class_names[0] elif class_names: raise EasyBuildError("Found multiple class names for easyblock %s: %s", easyblock_loc, class_names) else: raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc) easyblocks[easyblock_mod_name] = {'class': easyblock_class, 'loc': easyblock_loc} else: _log.debug("%s already imported from %s, ignoring %s", easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path) return easyblocks
def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') all_tcs_names = [x.NAME for x in all_tcs] tclist = sorted(zip(all_tcs_names, all_tcs)) tcs = dict() for (tcname, tcc) in tclist: tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there tcs[tcname] = tc.definition() return generate_doc('list_toolchains_%s' % output_format, [tcs])
def set_modaltsoftname(ec): if ec['name'] in modaltsoftnames: ec['modaltsoftname'] = modaltsoftnames[ec['name']] # add -mpi to module name for various modules with both -mpi and no-mpi varieties toolchain = ec.get('toolchain') toolchain_class, _ = search_toolchain(toolchain['name']) if (ec['name'].lower() in mpi_modaltsoftname and (toolchain_class(version=toolchain['version']).mpi_family() or (ec['toolchainopts'] and ec['toolchainopts'].get('usempi')))): ec['modaltsoftname'] = ec['name'].lower() + '-mpi' ec['versionsuffix'] = '-mpi'
def test_cgoolf_toolchain(self): """Test for cgoolf toolchain.""" name = "cgoolf" tc_class, _ = search_toolchain(name) self.assertEqual(tc_class.NAME, name) tc = tc_class(version="1.1.6") tc.prepare() self.assertEqual(tc.get_variable('CC'), 'clang') self.assertEqual(tc.get_variable('CXX'), 'clang++') self.assertEqual(tc.get_variable('F77'), 'gfortran') self.assertEqual(tc.get_variable('F90'), 'gfortran')
def _get_all_toolchain_names(self, search_string=''): """ Initialise each search_toolchain request, save in module constant TOOLCHAIN_NAMES. @param search_string: passed to search_toolchain function. """ global TOOLCHAIN_NAMES if not search_string in TOOLCHAIN_NAMES: _, all_tcs = search_toolchain(search_string) self.log.debug('Found all toolchains for "%s" to %s' % (search_string, all_tcs)) TOOLCHAIN_NAMES[search_string] = [x.NAME for x in all_tcs] self.log.debug('Set TOOLCHAIN_NAMES for "%s" to %s' % (search_string, TOOLCHAIN_NAMES[search_string])) return TOOLCHAIN_NAMES[search_string]
def setUp(self): """Set some convenience attributes""" super(TestEBConfigObj, self).setUp() _, tcs = search_toolchain('') self.tc_names = [x.NAME for x in tcs] self.tcmax = min(len(self.tc_names), 3) if len(self.tc_names) < self.tcmax: self.tcmax = len(self.tc_names) self.tc_namesmax = self.tc_names[:self.tcmax] self.tc_first = self.tc_names[0] self.tc_last = self.tc_names[-1] self.tc_lastmax = self.tc_namesmax[-1]
def avail_toolchains(self): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') all_tcs_names = [x.NAME for x in all_tcs] tclist = sorted(zip(all_tcs_names, all_tcs)) txt = ["List of known toolchains (toolchainname: module[,module...]):"] for (tcname, tcc) in tclist: tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there tc_elems = nub(sorted([e for es in tc.definition().values() for e in es])) txt.append("\t%s: %s" % (tcname, ', '.join(tc_elems))) return '\n'.join(txt)
def avail_toolchains(self): """Show list of known toolchains.""" _, all_tcs = search_toolchain("") all_tcs_names = [x.NAME for x in all_tcs] tclist = sorted(zip(all_tcs_names, all_tcs)) txt = ["List of known toolchains (toolchainname: module[,module...]):"] for (tcname, tcc) in tclist: tc = tcc(version="1.2.3") # version doesn't matter here, but something needs to be there tc_elems = nub(sorted([e for es in tc.definition().values() for e in es])) txt.append("\t%s: %s" % (tcname, ", ".join(tc_elems))) return "\n".join(txt)
def test_get_variable_seq_compilers(self): """Test get_variable function to obtain compiler variables.""" tc_class, _ = search_toolchain("goalf") tc = tc_class(version="1.1.0-no-OFED") tc.set_options({'usempi': True}) tc.prepare() cc_seq = tc.get_variable('CC_SEQ') self.assertEqual(cc_seq, "gcc") cxx_seq = tc.get_variable('CXX_SEQ') self.assertEqual(cxx_seq, "g++") f77_seq = tc.get_variable('F77_SEQ') self.assertEqual(f77_seq, "gfortran") f90_seq = tc.get_variable('F90_SEQ') self.assertEqual(f90_seq, "gfortran")
def avail_toolchains(self): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') all_tcs_names = [x.NAME for x in all_tcs] tclist = sorted(zip(all_tcs_names, all_tcs)) txt = ["List of known toolchains (toolchainname: module[,module...]):"] for (tcname, tcc) in tclist: tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there tc_elems = set([y for x in dir(tc) if x.endswith('_MODULE_NAME') for y in eval("tc.%s" % x)]) txt.append("\t%s: %s" % (tcname, ', '.join(sorted(tc_elems)))) return '\n'.join(txt)
def avail_toolchain_opts(name, output_format=FORMAT_TXT): """Show list of known options for given toolchain.""" tc_class, _ = search_toolchain(name) if not tc_class: raise EasyBuildError("Couldn't find toolchain: '%s'. To see available toolchains, use --list-toolchains" % name) tc = tc_class(version='1.0') # version doesn't matter here, but needs to be defined tc_dict = {} for cst in ['COMPILER_SHARED_OPTS', 'COMPILER_UNIQUE_OPTS', 'MPI_SHARED_OPTS', 'MPI_UNIQUE_OPTS']: if hasattr(tc, cst): opts = getattr(tc, cst) if opts is not None: tc_dict.update(opts) return generate_doc('avail_toolchain_opts_%s' % output_format, [name, tc_dict])
def avail_toolchains(self): """Show list of known toolchains.""" _, all_tcs = search_toolchain("") all_tcs_names = [x.NAME for x in all_tcs] tclist = sorted(zip(all_tcs_names, all_tcs)) txt = ["List of known toolchains (toolchainname: module[,module...]):"] for (tcname, tcc) in tclist: tc = tcc(version="1.2.3") # version doesn't matter here, but something needs to be there tc_elems = set([y for x in dir(tc) if x.endswith("_MODULE_NAME") for y in eval("tc.%s" % x)]) txt.append("\t%s: %s" % (tcname, ", ".join(sorted(tc_elems)))) return "\n".join(txt)
def versop_regex(self): """ Create the regular expression for toolchain support of format ^<toolchain> <versop_expr>$ , with <toolchain> the name of one of the supported toolchains and <versop_expr> in '<operator> <version>' syntax """ _, all_tcs = search_toolchain('') tc_names = [x.NAME for x in all_tcs] self.log.debug("found toolchain names %s" % tc_names) versop_regex = super(ToolchainVersionOperator, self).versop_regex(begin_end=False) versop_pattern = r'(?P<versop_str>%s)' % versop_regex.pattern tc_names_regex = r'(?P<tc_name>(?:%s))' % '|'.join(tc_names) tc_regex = re.compile(r'^%s(?:%s%s)?$' % (tc_names_regex, self.SEPARATOR, versop_pattern)) self.log.debug("toolchain versop regex pattern %s " % tc_regex.pattern) return tc_regex
def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') # filter deprecated 'dummy' toolchain all_tcs = [x for x in all_tcs if x.NAME != DUMMY_TOOLCHAIN_NAME] all_tcs_names = [x.NAME for x in all_tcs] # start with dict that maps toolchain name to corresponding subclass of Toolchain tcs = dict(zip(all_tcs_names, all_tcs)) for tcname in sorted(tcs): tcc = tcs[tcname] tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there tcs[tcname] = tc.definition() return generate_doc('list_toolchains_%s' % output_format, [tcs])
def toolchain(self): """ returns the Toolchain used """ if self._toolchain: return self._toolchain tcname = self['toolchain']['name'] tc, all_tcs = search_toolchain(tcname) if not tc: all_tcs_names = ",".join([x.NAME for x in all_tcs]) self.log.error("Toolchain %s not found, available toolchains: %s" % (tcname, all_tcs_names)) tc = tc(version=self['toolchain']['version']) if self['toolchainopts']: tc.set_options(self['toolchainopts']) self._toolchain = tc return self._toolchain
def test_toolchain_versop_test(self): """Test the ToolchainVersionOperator test""" _, tcs = search_toolchain('') tc_names = [x.NAME for x in tcs] for tc in tc_names: # test all known toolchain names # test version expressions with optional version operator tests = [ ("%s >= 1.2.3" % tc, ( (tc, '1.2.3', True), # version ok, name ok (tc, '1.2.4', True), # version ok, name ok (tc, '1.2.2', False), # version not ok, name ok ('x' + tc, '1.2.3', False), # version ok, name not ok ('x' + tc, '1.2.2', False), # version not ok, name not ok )), ] for txt, subtests in tests: tcversop = ToolchainVersionOperator(txt) for name, version, res in subtests: self.assertEqual(tcversop.test(name, version), res)
def test_misc_flags_unique_fortran(self): """Test whether unique Fortran compiler flags are set correctly.""" flag_vars = ['FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # setting option should result in corresponding flag to be set (Fortran unique options) for opt in ['i8', 'r8']: for enable in [True, False]: tc = tc_class(version="1.1.0-no-OFED") tc.set_options({opt: enable}) tc.prepare() flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt] for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) else: self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags))
def test_misc_flags_shared(self): """Test whether shared compiler flags are set correctly.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # setting option should result in corresponding flag to be set (shared options) for opt in ['pic', 'verbose', 'debug', 'static', 'shared']: for enable in [True, False]: tc = tc_class(version="1.1.0-no-OFED") tc.set_options({opt: enable}) tc.prepare() # we need to make sure we check for flags, not letter (e.g. 'v' vs '-v') flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP[opt] for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) else: self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags))
def test_configobj(self): """Test configobj sort""" _, tcs = search_toolchain('') tc_names = [x.NAME for x in tcs] tcmax = min(len(tc_names), 3) if len(tc_names) < tcmax: tcmax = len(tc_names) tc = tc_names[0] configobj_txt = [ '[DEFAULT]', 'toolchains=%s >= 7.8.9' % ','.join(tc_names[:tcmax]), 'versions=1.2.3,2.3.4,3.4.5', '[>= 2.3.4]', 'foo=bar', '[== 3.4.5]', 'baz=biz', '[!= %s 5.6.7]' % tc, '[%s > 7.8.9]' % tc_names[tcmax - 1], ] co = ConfigObj(configobj_txt) cov = ConfigObjVersion()
def toolchain(self): """ returns the Toolchain used """ if self._toolchain: return self._toolchain tcname = self['toolchain']['name'] tc, all_tcs = search_toolchain(tcname) if not tc: all_tcs_names = ",".join([x.NAME for x in all_tcs]) self.log.error("Toolchain %s not found, available toolchains: %s" % (tcname, all_tcs_names)) tc = tc(version=self['toolchain']['version']) if self['toolchainopts'] is None: # set_options should always be called, even if no toolchain options are specified # this is required to set the default options tc.set_options({}) else: tc.set_options(self['toolchainopts']) self._toolchain = tc return self._toolchain
def test_validate_pass_by_value(self): """ Check that elements of variables are passed by value, not by reference, which is required to ensure correctness. """ tc_class, _ = search_toolchain("goalf") tc = tc_class(version="1.1.0-no-OFED") tc.prepare() pass_by_value = True ids = [] for k, v in tc.variables.items(): for x in v: idx = id(x) if not idx in ids: ids.append(idx) else: pass_by_value = False break if not pass_by_value: break self.assertTrue(pass_by_value)
def test_get_variable_mpi_compilers(self): """Test get_variable function to obtain compiler variables.""" tc_class, _ = search_toolchain("goalf") tc = tc_class(version="1.1.0-no-OFED") tc.set_options({'usempi': True}) tc.prepare() cc = tc.get_variable('CC') self.assertEqual(cc, "mpicc") cxx = tc.get_variable('CXX') self.assertEqual(cxx, "mpicxx") f77 = tc.get_variable('F77') self.assertEqual(f77, "mpif77") f90 = tc.get_variable('F90') self.assertEqual(f90, "mpif90") mpicc = tc.get_variable('MPICC') self.assertEqual(mpicc, "mpicc") mpicxx = tc.get_variable('MPICXX') self.assertEqual(mpicxx, "mpicxx") mpif77 = tc.get_variable('MPIF77') self.assertEqual(mpif77, "mpif77") mpif90 = tc.get_variable('MPIF90') self.assertEqual(mpif90, "mpif90")
def test_misc_flags_unique_fortran(self): """Test whether unique Fortran compiler flags are set correctly.""" flag_vars = ['FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # setting option should result in corresponding flag to be set (Fortran unique options) for opt in ['i8', 'r8']: for enable in [True, False]: tc = tc_class(version="1.1.0-no-OFED") tc.set_options({opt: enable}) tc.prepare() flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt] for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue( flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) else: self.assertTrue( flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags))
def test_misc_flags_shared(self): """Test whether shared compiler flags are set correctly.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] tc_class, _ = search_toolchain("goalf") # setting option should result in corresponding flag to be set (shared options) for opt in ['pic', 'verbose', 'debug', 'static', 'shared']: for enable in [True, False]: tc = tc_class(version="1.1.0-no-OFED") tc.set_options({opt: enable}) tc.prepare() # we need to make sure we check for flags, not letter (e.g. 'v' vs '-v') flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP[opt] for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue( flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) else: self.assertTrue( flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags))
def test_goolfc(self): """Test whether goolfc is handled properly.""" tc_class, _ = search_toolchain("goolfc") tc = tc_class(version="1.3.12") opts = { 'cuda_gencode': ['arch=compute_35,code=sm_35', 'arch=compute_10,code=compute_10'] } tc.set_options(opts) tc.prepare() nvcc_flags = r' '.join([ r'-Xcompiler="-O2 -march=native"', # the use of -lcudart in -Xlinker is a bit silly but hard to avoid r'-Xlinker=".* -lm -lcudart -lpthread"', r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]), ]) self.assertEqual(tc.get_variable('CUDA_CC'), 'nvcc -ccbin="g++"') self.assertEqual(tc.get_variable('CUDA_CXX'), 'nvcc -ccbin="g++"') # -L/path flags will not be there if the software installations are not available val = tc.get_variable('CUDA_CFLAGS') self.assertTrue( re.compile(nvcc_flags).match(val), "'%s' matches '%s'" % (val, nvcc_flags)) val = tc.get_variable('CUDA_CXXFLAGS') self.assertTrue( re.compile(nvcc_flags).match(val), "'%s' matches '%s'" % (val, nvcc_flags)) # check compiler prefixes self.assertEqual(tc.comp_family(prefix='CUDA'), "CUDA") # check CUDA runtime lib self.assertTrue("-lcudart" in tc.get_variable('LIBS'))
def test_unknown_toolchain(self): """Test search_toolchain function for not available toolchains.""" tc, all_tcs = search_toolchain("NOSUCHTOOLKIT") self.assertEqual(tc, None) self.assertTrue(len(all_tcs) > 0) # list of available toolchains
def get_toolchain(self, name, version=None): """Get a toolchain object instance to test with.""" tc_class, _ = search_toolchain(name) self.assertEqual(tc_class.NAME, name) tc = tc_class(version=version, mns=ActiveMNS()) return tc