def test_squash_simple(self): """Test toolchain filter""" tc_first = {'version': '10', 'name': self.tc_first} tc_last = {'version': '100', 'name': self.tc_last} tc_tmpl = '%(name)s == %(version)s' default_version = '1.0' all_versions = [default_version, '0.0', '1.0'] txt = [ '[SUPPORTED]', 'versions = %s' % ', '.join(all_versions), 'toolchains = %s,%s' % (tc_tmpl % tc_first, tc_tmpl % tc_last), ] co = ConfigObj(txt) cov = EBConfigObj(co) found_tcs = [tmptc.as_dict() for tmptc in cov.sections['toolchains']] self.assertEqual(found_tcs, [tc_first, tc_last]) for tc in [tc_first, tc_last]: for version in all_versions: co = ConfigObj(txt) cov = EBConfigObj(co) res = cov.squash(version, tc['name'], tc['version']) self.assertEqual(res, {}) # very simple
def test_squash_invalid(self): """Try to squash invalid files. Should trigger error""" tc_first = {'version': '10', 'name': self.tc_first} tc_last = {'version': '100', 'name': self.tc_last} tc_tmpl = '%(name)s == %(version)s' default_version = '1.0' all_wrong_versions = [default_version, '>= 0.0', '< 1.0'] # all txt should have default version and first toolchain unmodified txt_wrong_versions = [ '[SUPPORTED]', 'versions = %s' % ', '.join( all_wrong_versions), # there's a conflict in the versions list 'toolchains = %s,%s' % (tc_tmpl % tc_first, tc_tmpl % tc_last), ] txt_conflict_nested_versions = [ '[SUPPORTED]', 'versions = %s' % default_version, 'toolchains = %s,%s' % (tc_tmpl % tc_first, tc_tmpl % tc_last), '[> 1]', '[[< 2]]', # although this makes sense, it's considered a conflict ] for txt in [ txt_wrong_versions, txt_conflict_nested_versions, ]: co = ConfigObj(txt) cov = EBConfigObj(co) self.assertErrorRegex(EasyBuildError, r'conflict', cov.squash, default_version, tc_first['name'], tc_first['version'])
def test_ebconfigobj(self): """Test configobj sort""" # the as_dict method is crap # tc >= 0.0.0 returns empty as_dict, although the boundary can be used # anyway, will go away with proper defaults tcfirst = ",".join([ '%s == 0.0.0' % self.tc_namesmax[0], '%s > 0.0.0' % self.tc_namesmax[0] ]) configobj_txt = [ '[SUPPORTED]', 'toolchains=%s,%s >= 7.8.9' % (tcfirst, ','.join(self.tc_namesmax[1:])), 'versions=1.2.3,2.3.4,3.4.5', '[>= 2.3.4]', 'foo=bar', '[== 3.4.5]', 'baz=biz', '[%s == 5.6.7]' % self.tc_first, '[%s > 7.8.9]' % self.tc_lastmax, ] co = ConfigObj(configobj_txt) cov = EBConfigObj(co) # default tc is cgoolf -> cgoolf > 0.0.0 res = cov.get_specs_for(version='2.3.4', tcname=self.tc_first, tcversion='1.0.0') self.assertEqual(res, {'foo': 'bar'})
def init_config(args=None, build_options=None, with_include=True): """(re)initialize configuration""" cleanup() # initialize configuration so config.get_modules_tool function works eb_go = eboptions.parse_options(args=args, with_include=with_include) config.init(eb_go.options, eb_go.get_options_by_section('config')) # initialize build options if build_options is None: build_options = {} default_build_options = { 'extended_dry_run': False, 'external_modules_metadata': ConfigObj(), 'local_var_naming_check': 'error', 'silence_deprecation_warnings': eb_go.options.silence_deprecation_warnings, 'suffix_modules_path': GENERAL_CLASS, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } for key in default_build_options: if key not in build_options: build_options[key] = default_build_options[key] config.init_build_options(build_options=build_options) return eb_go.options
def parse_section_block(self, section): """Parse the section block by trying to convert it into a ConfigObj instance""" try: self.configobj = ConfigObj(section.split('\n')) except SyntaxError as err: raise EasyBuildError('Failed to convert section text %s: %s', section, err) self.log.debug("Found ConfigObj instance %s" % self.configobj)
def _postprocess_external_modules_metadata(self): """Parse file(s) specifying metadata for external modules.""" # leave external_modules_metadata untouched if no files are provided if not self.options.external_modules_metadata: self.log.debug("No metadata provided for external modules.") return parsed_external_modules_metadata = ConfigObj() for path in self.options.external_modules_metadata: if os.path.exists(path): self.log.debug("Parsing %s with external modules metadata", path) try: parsed_external_modules_metadata.merge(ConfigObj(path)) except ConfigObjError, err: raise EasyBuildError("Failed to parse %s with external modules metadata: %s", path, err) else: raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", path)
def test_nested_version(self): """Test nested config""" tc = {'version': '10', 'name': self.tc_first} default_version = '1.0' txt = [ '[SUPPORTED]', 'versions = %s, 0.0, 1.1, 1.5, 1.6, 2.0, 3.0' % default_version, 'toolchains = %(name)s == %(version)s' % tc, # set tc, don't use it '[> 1.0]', 'versionprefix = stable-', '[[>= 1.5]]', 'versionsuffix = -early', '[> 2.0]', 'versionprefix = production-', 'versionsuffix = -mature', ] # version string, attributes without version and toolchain data = [ (None, {}), (default_version, {}), ('0.0', {}), ('1.1', { 'versionprefix': 'stable-' }), ('1.5', { 'versionprefix': 'stable-', 'versionsuffix': '-early' }), ('1.6', { 'versionprefix': 'stable-', 'versionsuffix': '-early' }), ('2.0', { 'versionprefix': 'stable-', 'versionsuffix': '-early' }), ('3.0', { 'versionprefix': 'production-', 'versionsuffix': '-mature' }), ] for version, res in data: # yes, redo this for each test, even if it's static text # some of the data is modified in place co = ConfigObj(txt) cov = EBConfigObj(co) specs = cov.get_specs_for(version=version) self.assertEqual(specs, res)
def _postprocess_external_modules_metadata(self): """Parse file(s) specifying metadata for external modules.""" # leave external_modules_metadata untouched if no files are provided if not self.options.external_modules_metadata: self.log.debug("No metadata provided for external modules.") return parsed_external_modules_metadata = ConfigObj() for path in self.options.external_modules_metadata: if os.path.exists(path): self.log.debug("Parsing %s with external modules metadata", path) try: parsed_external_modules_metadata.merge(ConfigObj(path)) except ConfigObjError, err: raise EasyBuildError( "Failed to parse %s with external modules metadata: %s", path, err) else: raise EasyBuildError( "Specified path for file with external modules metadata does not exist: %s", path)
def test_toolchain_squash_nested(self): """Test toolchain filter on nested sections""" tc_first = {'version': '10', 'name': self.tc_first} tc_last = {'version': '100', 'name': self.tc_last} tc_tmpl = '%(name)s == %(version)s' tc_section_first = tc_tmpl % tc_first tc_section_last = tc_tmpl % tc_last txt = [ '[SUPPORTED]', 'versions = 1.0, 0.0, 1.1, 1.6, 2.1', 'toolchains = %s,%s' % (tc_section_first, tc_tmpl % tc_last), '[DEFAULT]', 'y=a', '[> 1.0]', 'y=b', 'x = 1', '[[>= 1.5]]', 'x = 2', 'y=c', '[[[%s]]]' % tc_section_first, 'y=z2', '[[>= 1.6]]', 'z=3', '[> 2.0]', 'x = 3', 'y=d', '[%s]' % tc_section_first, 'y=z1', ] # tests tests = [ (tc_last, '1.0', {'y':'a'}), (tc_last, '1.1', {'y':'b', 'x':'1'}), (tc_last, '1.5', {}), # not a supported version (tc_last, '1.6', {'y':'c', 'x':'2', 'z':'3'}), # nested (tc_last, '2.1', {'y':'d', 'x':'3', 'z':'3'}), # values from most precise versop (tc_first, '1.0', {'y':'z1'}), # toolchain section, not default (tc_first, '1.1', {'y':'b', 'x':'1'}), # the version section precedes the toolchain section (tc_first, '1.5', {}), # not a supported version (tc_first, '1.6', {'y':'z2', 'x':'2', 'z':'3'}), # nested (tc_first, '2.1', {'y':'d', 'x':'3', 'z':'3'}), # values from most precise versop ] for tc, version, res in tests: co = ConfigObj(txt) cov = EBConfigObj(co) squashed = cov.squash(version, tc['name'], tc['version']) self.assertEqual(squashed, res, 'Test for tc %s version %s' % (tc, version))
def test_ebconfigobj_default(self): """Tests wrt ebconfigobj default parsing""" data = [ ('versions=1', {'version': '1'}), # == is usable ('toolchains=%s == 1' % self.tc_first, {'toolchain':{'name': self.tc_first, 'version': '1'}}), ] for val, res in data: configobj_txt = ['[SUPPORTED]', val] co = ConfigObj(configobj_txt) cov = EBConfigObj(co) self.assertEqual(cov.default, res)
def test_ebconfigobj_unusable_default(self): """Tests wrt ebconfigobj handling of unusable defaults""" # TODO implement proper default as per JSC meeting, remove this test # these will not raise error forever # the defaults will be interpreted with dedicated default_version and default_toochain data = [ # default operator > and/or version 0.0.0 are not usable for default ('toolchains=%s' % self.tc_first, {}), # > not usable for default ('toolchains=%s > 1' % self.tc_first, {}), ] for val, res in data: configobj_txt = ['[SUPPORTED]', val] co = ConfigObj(configobj_txt) self.assertErrorRegex(EasyBuildError, r'First\s+(toolchain|version)\s.*?\scan\'t\s+be\s+used\s+as\s+default', EBConfigObj, co)
def test_find_patches(self): """ Test for find_software_name_for_patch """ testdir = os.path.dirname(os.path.abspath(__file__)) ec_path = os.path.join(testdir, 'easyconfigs') init_config(build_options={ 'allow_modules_tool_mismatch': True, 'minimal_toolchains': True, 'use_existing_modules': True, 'external_modules_metadata': ConfigObj(), 'silent': True, 'valid_module_classes': module_classes(), 'validate': False, }) self.mock_stdout(True) ec = gh.find_software_name_for_patch('toy-0.0_fix-silly-typo-in-printf-statement.patch', [ec_path]) txt = self.get_stdout() self.mock_stdout(False) self.assertTrue(ec == 'toy') reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked') self.assertTrue(re.search(reg, txt))
def init_config(args=None, build_options=None): """(re)initialize configuration""" cleanup() # initialize configuration so config.get_modules_tool function works eb_go = eboptions.parse_options(args=args) config.init(eb_go.options, eb_go.get_options_by_section('config')) # initialize build options if build_options is None: build_options = { 'external_modules_metadata': ConfigObj(), 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } if 'suffix_modules_path' not in build_options: build_options.update({'suffix_modules_path': GENERAL_CLASS}) config.init_build_options(build_options=build_options) return eb_go.options
def test_configobj(self): """Test configobj sort""" _, tcs = search_toolchain('') tc_names = [x.NAME for x in tcs] tcmax = min(len(tc_names), 3) if len(tc_names) < tcmax: tcmax = len(tc_names) tc = tc_names[0] configobj_txt = [ '[DEFAULT]', 'toolchains=%s >= 7.8.9' % ','.join(tc_names[:tcmax]), 'versions=1.2.3,2.3.4,3.4.5', '[>= 2.3.4]', 'foo=bar', '[== 3.4.5]', 'baz=biz', '[!= %s 5.6.7]' % tc, '[%s > 7.8.9]' % tc_names[tcmax - 1], ] co = ConfigObj(configobj_txt) cov = ConfigObjVersion()
def test_resolve_dependencies_minimal(self): """Test resolved dependencies with minimal toolchain.""" # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning test_easyconfigs = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') install_mock_module() init_config( build_options={ 'allow_modules_tool_mismatch': True, 'external_modules_metadata': ConfigObj(), 'robot_path': test_easyconfigs, 'valid_module_classes': module_classes(), 'validate': False, }) barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_lines = [ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2 " ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('SQLite', '3.8.10.2'),", "]", # toolchain as list line, for easy modification later "toolchain = {'name': 'goolf', 'version': '1.4.10'}", ] write_file(barec, '\n'.join(barec_lines)) bar = process_easyconfig(barec)[0] # all modules in the dep graph, in order all_mods_ordered = [ 'GCC/4.7.2', 'hwloc/1.6.2-GCC-4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'gompi/1.4.10', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', 'SQLite/3.8.10.2-GCC-4.7.2', 'FFTW/3.3.3-gompi-1.4.10', 'goolf/1.4.10', 'bar/1.2.3-goolf-1.4.10', ] # no modules available, so all dependencies are retained MockModule.avail_modules = [] res = resolve_dependencies([bar], minimal_toolchains=True) self.assertEqual(len(res), 10) self.assertEqual([x['full_mod_name'] for x in res], all_mods_ordered) # cleanup shutil.rmtree( os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs')) MockModule.avail_modules = [ 'GCC/4.7.2', 'gompi/1.4.10', 'goolf/1.4.10', 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', 'SQLite/3.8.10.2-GCC-4.7.2', ] # test resolving dependencies with minimal toolchain (rather than using goolf/1.4.10 for all of them) # existing modules are *not* taken into account when determining minimal subtoolchain, by default res = resolve_dependencies([bar], minimal_toolchains=True) self.assertEqual(len(res), 1) self.assertEqual(res[0]['full_mod_name'], bar['ec'].full_mod_name) # cleanup shutil.rmtree( os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs')) # test retaining all dependencies, regardless of whether modules are available or not res = resolve_dependencies([bar], minimal_toolchains=True, retain_all_deps=True) self.assertEqual(len(res), 10) mods = [x['full_mod_name'] for x in res] self.assertEqual(mods, all_mods_ordered) self.assertTrue('SQLite/3.8.10.2-GCC-4.7.2' in mods) # cleanup shutil.rmtree( os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs')) # test taking into account existing modules # with an SQLite module with goolf/1.4.10 in place, this toolchain should be used rather than GCC/4.7.2 MockModule.avail_modules = [ 'SQLite/3.8.10.2-goolf-1.4.10', ] res = resolve_dependencies([bar], minimal_toolchains=True, retain_all_deps=True, use_existing_modules=True) self.assertEqual(len(res), 10) mods = [x['full_mod_name'] for x in res] self.assertTrue('SQLite/3.8.10.2-goolf-1.4.10' in mods) self.assertFalse('SQLite/3.8.10.2-GCC-4.7.2' in mods)
def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ self.install_mock_module() base_easyconfig_dir = find_full_path(os.path.join('test', 'framework', 'easyconfigs', 'test_ecs')) self.assertTrue(base_easyconfig_dir) easyconfig = { 'spec': '_', 'full_mod_name': 'name/version', 'short_mod_name': 'name/version', 'dependencies': [] } build_options = { 'allow_modules_tool_mismatch': True, 'external_modules_metadata': ConfigObj(), 'robot_path': None, 'validate': False, } init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig)], self.modtool) self.assertEqual([easyconfig], res) easyconfig_dep = { 'ec': { 'name': 'foo', 'version': '1.2.3', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, }, 'spec': '_', 'short_mod_name': 'foo/1.2.3', 'full_mod_name': 'foo/1.2.3', 'dependencies': [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }], 'parsed': True, } build_options.update({'robot': True, 'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool) # dependency should be found, order should be correct self.assertEqual(len(res), 2) self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) # hidden dependencies are found too, but only retained if they're not available (or forced to be retained hidden_dep = { 'name': 'toy', 'version': '0.0', 'versionsuffix': '-deps', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': True, } easyconfig_moredeps = deepcopy(easyconfig_dep) easyconfig_moredeps['dependencies'].append(hidden_dep) easyconfig_moredeps['hiddendependencies'] = [hidden_dep] # toy/.0.0-deps is available and thus should be omitted res = resolve_dependencies([deepcopy(easyconfig_moredeps)], self.modtool) self.assertEqual(len(res), 2) full_mod_names = [ec['full_mod_name'] for ec in res] self.assertFalse('toy/.0.0-deps' in full_mod_names) res = resolve_dependencies([deepcopy(easyconfig_moredeps)], self.modtool, retain_all_deps=True) self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1) self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) full_mod_names = [ec['full_mod_name'] for ec in res] self.assertTrue('toy/.0.0-deps' in full_mod_names) self.assertTrue('ictce/4.1.13' in full_mod_names) # here we have included a dependency in the easyconfig list easyconfig['full_mod_name'] = 'gzip/1.4' ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] build_options.update({'robot_path': None}) init_config(build_options=build_options) res = resolve_dependencies(ecs, self.modtool) # all dependencies should be resolved self.assertEqual(0, sum(len(ec['dependencies']) for ec in res)) # this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled ecs = [deepcopy(easyconfig_dep)] msg = "Irresolvable dependencies encountered" self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs, self.modtool) # test if dependencies of an automatically found file are also loaded easyconfig_dep['dependencies'] = [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] build_options.update({'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool) # GCC should be first (required by gzip dependency) self.assertEqual('GCC/4.6.3', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) # make sure that only missing stuff is built, and that available modules are not rebuilt # monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'FFTW/3.3.3-gompi-1.4.10', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs, self.modtool) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 2) # goolf should be first, the software itself second self.assertEqual('goolf/1.4.10', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[1]['full_mod_name']) # force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt build_options.update({'force': True}) init_config(build_options=build_options) easyconfig['full_mod_name'] = 'this/is/already/there' MockModule.avail_modules.append('this/is/already/there') ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] res = resolve_dependencies(ecs, self.modtool) # there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available) self.assertEqual(len(res), 3) # goolf should be first, the software itself second self.assertEqual('this/is/already/there', res[0]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[1]['full_mod_name']) self.assertEqual('foo/1.2.3', res[2]['full_mod_name']) # build that are listed but already have a module available are not retained without force build_options.update({'force': False}) init_config(build_options=build_options) newecs = skip_available(ecs, self.modtool) # skip available builds since force is not enabled res = resolve_dependencies(newecs, self.modtool) self.assertEqual(len(res), 2) self.assertEqual('goolf/1.4.10', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[1]['full_mod_name']) # with retain_all_deps enabled, all dependencies ae retained build_options.update({'retain_all_deps': True}) init_config(build_options=build_options) ecs = [deepcopy(easyconfig_dep)] newecs = skip_available(ecs, self.modtool) # skip available builds since force is not enabled res = resolve_dependencies(newecs, self.modtool) self.assertEqual(len(res), 9) self.assertEqual('GCC/4.7.2', res[0]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[-2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) build_options.update({'retain_all_deps': False}) init_config(build_options=build_options) # provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'gompi/1.4.10', 'FFTW/3.3.3-gompi-1.4.10', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['full_mod_name']) self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[3]['full_mod_name'])
def test_resolve_dependencies_minimal(self): """Test resolved dependencies with minimal toolchain.""" # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') self.install_mock_module() init_config(build_options={ 'allow_modules_tool_mismatch': True, 'minimal_toolchains': True, 'use_existing_modules': True, 'external_modules_metadata': ConfigObj(), 'robot_path': test_easyconfigs, 'valid_module_classes': module_classes(), 'validate': False, }) barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_lines = [ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2 " ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('SQLite', '3.8.10.2'),", "]", # toolchain as list line, for easy modification later; # the use of %(version_major)s here is mainly to check if templates are being handled correctly # (it doesn't make much sense, but it serves the purpose) "toolchain = {'name': 'goolf', 'version': '%(version_major)s.4.10'}", ] write_file(barec, '\n'.join(barec_lines)) bar = process_easyconfig(barec)[0] # all modules in the dep graph, in order all_mods_ordered = [ 'GCC/4.7.2', 'hwloc/1.6.2-GCC-4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'gompi/1.4.10', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', 'SQLite/3.8.10.2-GCC-4.7.2', 'FFTW/3.3.3-gompi-1.4.10', 'goolf/1.4.10', 'bar/1.2.3-goolf-1.4.10', ] # no modules available, so all dependencies are retained MockModule.avail_modules = [] res = resolve_dependencies([bar], self.modtool) self.assertEqual(len(res), 10) self.assertEqual([x['full_mod_name'] for x in res], all_mods_ordered) MockModule.avail_modules = [ 'GCC/4.7.2', 'gompi/1.4.10', 'goolf/1.4.10', 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', 'SQLite/3.8.10.2-GCC-4.7.2', ] # test resolving dependencies with minimal toolchain (rather than using goolf/1.4.10 for all of them) # existing modules are *not* taken into account when determining minimal subtoolchain, by default res = resolve_dependencies([bar], self.modtool) self.assertEqual(len(res), 1) self.assertEqual(res[0]['full_mod_name'], bar['ec'].full_mod_name) # test retaining all dependencies, regardless of whether modules are available or not res = resolve_dependencies([bar], self.modtool, retain_all_deps=True) self.assertEqual(len(res), 10) mods = [x['full_mod_name'] for x in res] self.assertEqual(mods, all_mods_ordered) self.assertTrue('SQLite/3.8.10.2-GCC-4.7.2' in mods) # test taking into account existing modules # with an SQLite module with goolf/1.4.10 in place, this toolchain should be used rather than GCC/4.7.2 MockModule.avail_modules = [ 'SQLite/3.8.10.2-goolf-1.4.10', ] # parsed easyconfigs are cached, so clear the cache before reprocessing easyconfigs ecec._easyconfigs_cache.clear() bar = process_easyconfig(barec)[0] res = resolve_dependencies([bar], self.modtool, retain_all_deps=True) self.assertEqual(len(res), 10) mods = [x['full_mod_name'] for x in res] self.assertTrue('SQLite/3.8.10.2-goolf-1.4.10' in mods) self.assertFalse('SQLite/3.8.10.2-GCC-4.7.2' in mods) # Check whether having 2 version of dummy toolchain is ok # Clear easyconfig and toolchain caches ecec._easyconfigs_cache.clear() get_toolchain_hierarchy.clear() init_config(build_options={ 'allow_modules_tool_mismatch': True, 'minimal_toolchains': True, 'add_dummy_to_minimal_toolchains': True, 'external_modules_metadata': ConfigObj(), 'robot_path': test_easyconfigs, 'valid_module_classes': module_classes(), 'validate': False, }) impi_txt = read_file(os.path.join(test_easyconfigs, 'i', 'impi', 'impi-4.1.3.049.eb')) self.assertTrue(re.search("^toolchain = {'name': 'dummy', 'version': ''}", impi_txt, re.M)) gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4.eb')) self.assertTrue(re.search("^toolchain = {'name': 'dummy', 'version': 'dummy'}", gzip_txt, re.M)) barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_lines = [ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('impi', '4.1.3.049'),", # has toolchain ('dummy', '') " ('gzip', '1.4'),", # has toolchain ('dummy', 'dummy') "]", # toolchain as list line, for easy modification later "toolchain = {'name': 'goolf', 'version': '1.4.10'}", ] write_file(barec, '\n'.join(barec_lines)) bar = process_easyconfig(barec)[0] res = resolve_dependencies([bar], self.modtool, retain_all_deps=True) self.assertEqual(len(res), 11) mods = [x['full_mod_name'] for x in res] self.assertTrue('impi/4.1.3.049' in mods) self.assertTrue('gzip/1.4' in mods)
def test_get_toolchain_hierarchy(self): """Test get_toolchain_hierarchy function.""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}) self.assertEqual(goolf_hierarchy, [ {'name': 'GCC', 'version': '4.7.2'}, {'name': 'gompi', 'version': '1.4.10'}, {'name': 'goolf', 'version': '1.4.10'}, ]) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}) self.assertEqual(iimpi_hierarchy, [ {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'}, {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, ]) # test also including dummy init_config(build_options={ 'add_dummy_to_minimal_toolchains': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) get_toolchain_hierarchy.clear() gompi_hierarchy = get_toolchain_hierarchy({'name': 'gompi', 'version': '1.4.10'}) self.assertEqual(gompi_hierarchy, [ {'name': 'dummy', 'version': ''}, {'name': 'GCC', 'version': '4.7.2'}, {'name': 'gompi', 'version': '1.4.10'}, ]) get_toolchain_hierarchy.clear() # check whether GCCcore is considered as subtoolchain, even if it's only listed as a dep gcc_hierarchy = get_toolchain_hierarchy({'name': 'GCC', 'version': '4.9.3-2.25'}) self.assertEqual(gcc_hierarchy, [ {'name': 'dummy', 'version': ''}, {'name': 'GCCcore', 'version': '4.9.3'}, {'name': 'GCC', 'version': '4.9.3-2.25'}, ]) get_toolchain_hierarchy.clear() iccifort_hierarchy = get_toolchain_hierarchy({'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}) self.assertEqual(iccifort_hierarchy, [ {'name': 'dummy', 'version': ''}, {'name': 'GCCcore', 'version': '4.9.3'}, {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, ]) get_toolchain_hierarchy.clear() build_options = { 'add_dummy_to_minimal_toolchains': True, 'external_modules_metadata': ConfigObj(), 'robot_path': test_easyconfigs, } init_config(build_options=build_options) craycce_hierarchy = get_toolchain_hierarchy({'name': 'CrayCCE', 'version': '5.1.29'}) self.assertEqual(craycce_hierarchy, [ {'name': 'dummy', 'version': ''}, {'name': 'CrayCCE', 'version': '5.1.29'}, ])