def test_squash_simple(self):
"""Test toolchain filter"""
tc_first = {'version': '10', 'name': self.tc_first}
tc_last = {'version': '100', 'name': self.tc_last}
tc_tmpl = '%(name)s == %(version)s'
default_version = '1.0'
all_versions = [default_version, '0.0', '1.0']
txt = [
'[SUPPORTED]',
'versions = %s' % ', '.join(all_versions),
'toolchains = %s,%s' % (tc_tmpl % tc_first, tc_tmpl % tc_last),
]
co = ConfigObj(txt)
cov = EBConfigObj(co)
found_tcs = [tmptc.as_dict() for tmptc in cov.sections['toolchains']]
self.assertEqual(found_tcs, [tc_first, tc_last])
for tc in [tc_first, tc_last]:
for version in all_versions:
co = ConfigObj(txt)
cov = EBConfigObj(co)
res = cov.squash(version, tc['name'], tc['version'])
self.assertEqual(res, {}) # very simple
def test_squash_invalid(self):
"""Try to squash invalid files. Should trigger error"""
tc_first = {'version': '10', 'name': self.tc_first}
tc_last = {'version': '100', 'name': self.tc_last}
tc_tmpl = '%(name)s == %(version)s'
default_version = '1.0'
all_wrong_versions = [default_version, '>= 0.0', '< 1.0']
# all txt should have default version and first toolchain unmodified
txt_wrong_versions = [
'[SUPPORTED]',
'versions = %s' % ', '.join(
all_wrong_versions), # there's a conflict in the versions list
'toolchains = %s,%s' % (tc_tmpl % tc_first, tc_tmpl % tc_last),
]
txt_conflict_nested_versions = [
'[SUPPORTED]',
'versions = %s' % default_version,
'toolchains = %s,%s' % (tc_tmpl % tc_first, tc_tmpl % tc_last),
'[> 1]',
'[[< 2]]', # although this makes sense, it's considered a conflict
]
for txt in [
txt_wrong_versions,
txt_conflict_nested_versions,
]:
co = ConfigObj(txt)
cov = EBConfigObj(co)
self.assertErrorRegex(EasyBuildError, r'conflict', cov.squash,
default_version, tc_first['name'],
tc_first['version'])
def test_ebconfigobj(self):
"""Test configobj sort"""
# the as_dict method is crap
# tc >= 0.0.0 returns empty as_dict, although the boundary can be used
# anyway, will go away with proper defaults
tcfirst = ",".join([
'%s == 0.0.0' % self.tc_namesmax[0],
'%s > 0.0.0' % self.tc_namesmax[0]
])
configobj_txt = [
'[SUPPORTED]',
'toolchains=%s,%s >= 7.8.9' %
(tcfirst, ','.join(self.tc_namesmax[1:])),
'versions=1.2.3,2.3.4,3.4.5',
'[>= 2.3.4]',
'foo=bar',
'[== 3.4.5]',
'baz=biz',
'[%s == 5.6.7]' % self.tc_first,
'[%s > 7.8.9]' % self.tc_lastmax,
]
co = ConfigObj(configobj_txt)
cov = EBConfigObj(co)
# default tc is cgoolf -> cgoolf > 0.0.0
res = cov.get_specs_for(version='2.3.4',
tcname=self.tc_first,
tcversion='1.0.0')
self.assertEqual(res, {'foo': 'bar'})
def init_config(args=None, build_options=None, with_include=True):
"""(re)initialize configuration"""
cleanup()
# initialize configuration so config.get_modules_tool function works
eb_go = eboptions.parse_options(args=args, with_include=with_include)
config.init(eb_go.options, eb_go.get_options_by_section('config'))
# initialize build options
if build_options is None:
build_options = {}
default_build_options = {
'extended_dry_run': False,
'external_modules_metadata': ConfigObj(),
'local_var_naming_check': 'error',
'silence_deprecation_warnings': eb_go.options.silence_deprecation_warnings,
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
for key in default_build_options:
if key not in build_options:
build_options[key] = default_build_options[key]
config.init_build_options(build_options=build_options)
return eb_go.options
def parse_section_block(self, section):
"""Parse the section block by trying to convert it into a ConfigObj instance"""
try:
self.configobj = ConfigObj(section.split('\n'))
except SyntaxError as err:
raise EasyBuildError('Failed to convert section text %s: %s', section, err)
self.log.debug("Found ConfigObj instance %s" % self.configobj)
def _postprocess_external_modules_metadata(self):
"""Parse file(s) specifying metadata for external modules."""
# leave external_modules_metadata untouched if no files are provided
if not self.options.external_modules_metadata:
self.log.debug("No metadata provided for external modules.")
return
parsed_external_modules_metadata = ConfigObj()
for path in self.options.external_modules_metadata:
if os.path.exists(path):
self.log.debug("Parsing %s with external modules metadata", path)
try:
parsed_external_modules_metadata.merge(ConfigObj(path))
except ConfigObjError, err:
raise EasyBuildError("Failed to parse %s with external modules metadata: %s", path, err)
else:
raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", path)
def test_nested_version(self):
"""Test nested config"""
tc = {'version': '10', 'name': self.tc_first}
default_version = '1.0'
txt = [
'[SUPPORTED]',
'versions = %s, 0.0, 1.1, 1.5, 1.6, 2.0, 3.0' % default_version,
'toolchains = %(name)s == %(version)s' %
tc, # set tc, don't use it
'[> 1.0]',
'versionprefix = stable-',
'[[>= 1.5]]',
'versionsuffix = -early',
'[> 2.0]',
'versionprefix = production-',
'versionsuffix = -mature',
]
# version string, attributes without version and toolchain
data = [
(None, {}),
(default_version, {}),
('0.0', {}),
('1.1', {
'versionprefix': 'stable-'
}),
('1.5', {
'versionprefix': 'stable-',
'versionsuffix': '-early'
}),
('1.6', {
'versionprefix': 'stable-',
'versionsuffix': '-early'
}),
('2.0', {
'versionprefix': 'stable-',
'versionsuffix': '-early'
}),
('3.0', {
'versionprefix': 'production-',
'versionsuffix': '-mature'
}),
]
for version, res in data:
# yes, redo this for each test, even if it's static text
# some of the data is modified in place
co = ConfigObj(txt)
cov = EBConfigObj(co)
specs = cov.get_specs_for(version=version)
self.assertEqual(specs, res)
def _postprocess_external_modules_metadata(self):
"""Parse file(s) specifying metadata for external modules."""
# leave external_modules_metadata untouched if no files are provided
if not self.options.external_modules_metadata:
self.log.debug("No metadata provided for external modules.")
return
parsed_external_modules_metadata = ConfigObj()
for path in self.options.external_modules_metadata:
if os.path.exists(path):
self.log.debug("Parsing %s with external modules metadata",
path)
try:
parsed_external_modules_metadata.merge(ConfigObj(path))
except ConfigObjError, err:
raise EasyBuildError(
"Failed to parse %s with external modules metadata: %s",
path, err)
else:
raise EasyBuildError(
"Specified path for file with external modules metadata does not exist: %s",
path)
def test_toolchain_squash_nested(self):
"""Test toolchain filter on nested sections"""
tc_first = {'version': '10', 'name': self.tc_first}
tc_last = {'version': '100', 'name': self.tc_last}
tc_tmpl = '%(name)s == %(version)s'
tc_section_first = tc_tmpl % tc_first
tc_section_last = tc_tmpl % tc_last
txt = [
'[SUPPORTED]',
'versions = 1.0, 0.0, 1.1, 1.6, 2.1',
'toolchains = %s,%s' % (tc_section_first, tc_tmpl % tc_last),
'[DEFAULT]',
'y=a',
'[> 1.0]',
'y=b',
'x = 1',
'[[>= 1.5]]',
'x = 2',
'y=c',
'[[[%s]]]' % tc_section_first,
'y=z2',
'[[>= 1.6]]',
'z=3',
'[> 2.0]',
'x = 3',
'y=d',
'[%s]' % tc_section_first,
'y=z1',
]
# tests
tests = [
(tc_last, '1.0', {'y':'a'}),
(tc_last, '1.1', {'y':'b', 'x':'1'}),
(tc_last, '1.5', {}), # not a supported version
(tc_last, '1.6', {'y':'c', 'x':'2', 'z':'3'}), # nested
(tc_last, '2.1', {'y':'d', 'x':'3', 'z':'3'}), # values from most precise versop
(tc_first, '1.0', {'y':'z1'}), # toolchain section, not default
(tc_first, '1.1', {'y':'b', 'x':'1'}), # the version section precedes the toolchain section
(tc_first, '1.5', {}), # not a supported version
(tc_first, '1.6', {'y':'z2', 'x':'2', 'z':'3'}), # nested
(tc_first, '2.1', {'y':'d', 'x':'3', 'z':'3'}), # values from most precise versop
]
for tc, version, res in tests:
co = ConfigObj(txt)
cov = EBConfigObj(co)
squashed = cov.squash(version, tc['name'], tc['version'])
self.assertEqual(squashed, res, 'Test for tc %s version %s' % (tc, version))
def test_ebconfigobj_default(self):
"""Tests wrt ebconfigobj default parsing"""
data = [
('versions=1', {'version': '1'}),
# == is usable
('toolchains=%s == 1' % self.tc_first, {'toolchain':{'name': self.tc_first, 'version': '1'}}),
]
for val, res in data:
configobj_txt = ['[SUPPORTED]', val]
co = ConfigObj(configobj_txt)
cov = EBConfigObj(co)
self.assertEqual(cov.default, res)
def test_ebconfigobj_unusable_default(self):
"""Tests wrt ebconfigobj handling of unusable defaults"""
# TODO implement proper default as per JSC meeting, remove this test
# these will not raise error forever
# the defaults will be interpreted with dedicated default_version and default_toochain
data = [
# default operator > and/or version 0.0.0 are not usable for default
('toolchains=%s' % self.tc_first, {}),
# > not usable for default
('toolchains=%s > 1' % self.tc_first, {}),
]
for val, res in data:
configobj_txt = ['[SUPPORTED]', val]
co = ConfigObj(configobj_txt)
self.assertErrorRegex(EasyBuildError,
r'First\s+(toolchain|version)\s.*?\scan\'t\s+be\s+used\s+as\s+default',
EBConfigObj, co)
def test_find_patches(self):
""" Test for find_software_name_for_patch """
testdir = os.path.dirname(os.path.abspath(__file__))
ec_path = os.path.join(testdir, 'easyconfigs')
init_config(build_options={
'allow_modules_tool_mismatch': True,
'minimal_toolchains': True,
'use_existing_modules': True,
'external_modules_metadata': ConfigObj(),
'silent': True,
'valid_module_classes': module_classes(),
'validate': False,
})
self.mock_stdout(True)
ec = gh.find_software_name_for_patch('toy-0.0_fix-silly-typo-in-printf-statement.patch', [ec_path])
txt = self.get_stdout()
self.mock_stdout(False)
self.assertTrue(ec == 'toy')
reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked')
self.assertTrue(re.search(reg, txt))
def init_config(args=None, build_options=None):
"""(re)initialize configuration"""
cleanup()
# initialize configuration so config.get_modules_tool function works
eb_go = eboptions.parse_options(args=args)
config.init(eb_go.options, eb_go.get_options_by_section('config'))
# initialize build options
if build_options is None:
build_options = {
'external_modules_metadata': ConfigObj(),
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
if 'suffix_modules_path' not in build_options:
build_options.update({'suffix_modules_path': GENERAL_CLASS})
config.init_build_options(build_options=build_options)
return eb_go.options
def test_configobj(self):
"""Test configobj sort"""
_, tcs = search_toolchain('')
tc_names = [x.NAME for x in tcs]
tcmax = min(len(tc_names), 3)
if len(tc_names) < tcmax:
tcmax = len(tc_names)
tc = tc_names[0]
configobj_txt = [
'[DEFAULT]',
'toolchains=%s >= 7.8.9' % ','.join(tc_names[:tcmax]),
'versions=1.2.3,2.3.4,3.4.5',
'[>= 2.3.4]',
'foo=bar',
'[== 3.4.5]',
'baz=biz',
'[!= %s 5.6.7]' % tc,
'[%s > 7.8.9]' % tc_names[tcmax - 1],
]
co = ConfigObj(configobj_txt)
cov = ConfigObjVersion()
def test_resolve_dependencies_minimal(self):
"""Test resolved dependencies with minimal toolchain."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
install_mock_module()
init_config(
build_options={
'allow_modules_tool_mismatch': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
'valid_module_classes': module_classes(),
'validate': False,
})
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_lines = [
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),",
"]",
# toolchain as list line, for easy modification later
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
]
write_file(barec, '\n'.join(barec_lines))
bar = process_easyconfig(barec)[0]
# all modules in the dep graph, in order
all_mods_ordered = [
'GCC/4.7.2',
'hwloc/1.6.2-GCC-4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
'FFTW/3.3.3-gompi-1.4.10',
'goolf/1.4.10',
'bar/1.2.3-goolf-1.4.10',
]
# no modules available, so all dependencies are retained
MockModule.avail_modules = []
res = resolve_dependencies([bar], minimal_toolchains=True)
self.assertEqual(len(res), 10)
self.assertEqual([x['full_mod_name'] for x in res], all_mods_ordered)
# cleanup
shutil.rmtree(
os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
MockModule.avail_modules = [
'GCC/4.7.2',
'gompi/1.4.10',
'goolf/1.4.10',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
]
# test resolving dependencies with minimal toolchain (rather than using goolf/1.4.10 for all of them)
# existing modules are *not* taken into account when determining minimal subtoolchain, by default
res = resolve_dependencies([bar], minimal_toolchains=True)
self.assertEqual(len(res), 1)
self.assertEqual(res[0]['full_mod_name'], bar['ec'].full_mod_name)
# cleanup
shutil.rmtree(
os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
# test retaining all dependencies, regardless of whether modules are available or not
res = resolve_dependencies([bar],
minimal_toolchains=True,
retain_all_deps=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertEqual(mods, all_mods_ordered)
self.assertTrue('SQLite/3.8.10.2-GCC-4.7.2' in mods)
# cleanup
shutil.rmtree(
os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
# test taking into account existing modules
# with an SQLite module with goolf/1.4.10 in place, this toolchain should be used rather than GCC/4.7.2
MockModule.avail_modules = [
'SQLite/3.8.10.2-goolf-1.4.10',
]
res = resolve_dependencies([bar],
minimal_toolchains=True,
retain_all_deps=True,
use_existing_modules=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertTrue('SQLite/3.8.10.2-goolf-1.4.10' in mods)
self.assertFalse('SQLite/3.8.10.2-GCC-4.7.2' in mods)
def test_resolve_dependencies(self):
""" Test with some basic testcases (also check if he can find dependencies inside the given directory """
self.install_mock_module()
base_easyconfig_dir = find_full_path(os.path.join('test', 'framework', 'easyconfigs', 'test_ecs'))
self.assertTrue(base_easyconfig_dir)
easyconfig = {
'spec': '_',
'full_mod_name': 'name/version',
'short_mod_name': 'name/version',
'dependencies': []
}
build_options = {
'allow_modules_tool_mismatch': True,
'external_modules_metadata': ConfigObj(),
'robot_path': None,
'validate': False,
}
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig)], self.modtool)
self.assertEqual([easyconfig], res)
easyconfig_dep = {
'ec': {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
},
'spec': '_',
'short_mod_name': 'foo/1.2.3',
'full_mod_name': 'foo/1.2.3',
'dependencies': [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': False,
}],
'parsed': True,
}
build_options.update({'robot': True, 'robot_path': base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool)
# dependency should be found, order should be correct
self.assertEqual(len(res), 2)
self.assertEqual('gzip/1.4', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
# hidden dependencies are found too, but only retained if they're not available (or forced to be retained
hidden_dep = {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-deps',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': True,
}
easyconfig_moredeps = deepcopy(easyconfig_dep)
easyconfig_moredeps['dependencies'].append(hidden_dep)
easyconfig_moredeps['hiddendependencies'] = [hidden_dep]
# toy/.0.0-deps is available and thus should be omitted
res = resolve_dependencies([deepcopy(easyconfig_moredeps)], self.modtool)
self.assertEqual(len(res), 2)
full_mod_names = [ec['full_mod_name'] for ec in res]
self.assertFalse('toy/.0.0-deps' in full_mod_names)
res = resolve_dependencies([deepcopy(easyconfig_moredeps)], self.modtool, retain_all_deps=True)
self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1)
self.assertEqual('gzip/1.4', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
full_mod_names = [ec['full_mod_name'] for ec in res]
self.assertTrue('toy/.0.0-deps' in full_mod_names)
self.assertTrue('ictce/4.1.13' in full_mod_names)
# here we have included a dependency in the easyconfig list
easyconfig['full_mod_name'] = 'gzip/1.4'
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
build_options.update({'robot_path': None})
init_config(build_options=build_options)
res = resolve_dependencies(ecs, self.modtool)
# all dependencies should be resolved
self.assertEqual(0, sum(len(ec['dependencies']) for ec in res))
# this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled
ecs = [deepcopy(easyconfig_dep)]
msg = "Irresolvable dependencies encountered"
self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs, self.modtool)
# test if dependencies of an automatically found file are also loaded
easyconfig_dep['dependencies'] = [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'GCC', 'version': '4.6.3'},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
build_options.update({'robot_path': base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool)
# GCC should be first (required by gzip dependency)
self.assertEqual('GCC/4.6.3', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
# make sure that only missing stuff is built, and that available modules are not rebuilt
# monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'FFTW/3.3.3-gompi-1.4.10',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies(ecs, self.modtool)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 2)
# goolf should be first, the software itself second
self.assertEqual('goolf/1.4.10', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[1]['full_mod_name'])
# force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt
build_options.update({'force': True})
init_config(build_options=build_options)
easyconfig['full_mod_name'] = 'this/is/already/there'
MockModule.avail_modules.append('this/is/already/there')
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
res = resolve_dependencies(ecs, self.modtool)
# there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available)
self.assertEqual(len(res), 3)
# goolf should be first, the software itself second
self.assertEqual('this/is/already/there', res[0]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[1]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[2]['full_mod_name'])
# build that are listed but already have a module available are not retained without force
build_options.update({'force': False})
init_config(build_options=build_options)
newecs = skip_available(ecs, self.modtool) # skip available builds since force is not enabled
res = resolve_dependencies(newecs, self.modtool)
self.assertEqual(len(res), 2)
self.assertEqual('goolf/1.4.10', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[1]['full_mod_name'])
# with retain_all_deps enabled, all dependencies ae retained
build_options.update({'retain_all_deps': True})
init_config(build_options=build_options)
ecs = [deepcopy(easyconfig_dep)]
newecs = skip_available(ecs, self.modtool) # skip available builds since force is not enabled
res = resolve_dependencies(newecs, self.modtool)
self.assertEqual(len(res), 9)
self.assertEqual('GCC/4.7.2', res[0]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[-2]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
build_options.update({'retain_all_deps': False})
init_config(build_options=build_options)
# provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'FFTW/3.3.3-gompi-1.4.10',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 4)
# goolf should be first, the software itself second
self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['full_mod_name'])
self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[2]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[3]['full_mod_name'])
def test_resolve_dependencies_minimal(self):
"""Test resolved dependencies with minimal toolchain."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
self.install_mock_module()
init_config(build_options={
'allow_modules_tool_mismatch': True,
'minimal_toolchains': True,
'use_existing_modules': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
'valid_module_classes': module_classes(),
'validate': False,
})
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_lines = [
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),",
"]",
# toolchain as list line, for easy modification later;
# the use of %(version_major)s here is mainly to check if templates are being handled correctly
# (it doesn't make much sense, but it serves the purpose)
"toolchain = {'name': 'goolf', 'version': '%(version_major)s.4.10'}",
]
write_file(barec, '\n'.join(barec_lines))
bar = process_easyconfig(barec)[0]
# all modules in the dep graph, in order
all_mods_ordered = [
'GCC/4.7.2',
'hwloc/1.6.2-GCC-4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
'FFTW/3.3.3-gompi-1.4.10',
'goolf/1.4.10',
'bar/1.2.3-goolf-1.4.10',
]
# no modules available, so all dependencies are retained
MockModule.avail_modules = []
res = resolve_dependencies([bar], self.modtool)
self.assertEqual(len(res), 10)
self.assertEqual([x['full_mod_name'] for x in res], all_mods_ordered)
MockModule.avail_modules = [
'GCC/4.7.2',
'gompi/1.4.10',
'goolf/1.4.10',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
]
# test resolving dependencies with minimal toolchain (rather than using goolf/1.4.10 for all of them)
# existing modules are *not* taken into account when determining minimal subtoolchain, by default
res = resolve_dependencies([bar], self.modtool)
self.assertEqual(len(res), 1)
self.assertEqual(res[0]['full_mod_name'], bar['ec'].full_mod_name)
# test retaining all dependencies, regardless of whether modules are available or not
res = resolve_dependencies([bar], self.modtool, retain_all_deps=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertEqual(mods, all_mods_ordered)
self.assertTrue('SQLite/3.8.10.2-GCC-4.7.2' in mods)
# test taking into account existing modules
# with an SQLite module with goolf/1.4.10 in place, this toolchain should be used rather than GCC/4.7.2
MockModule.avail_modules = [
'SQLite/3.8.10.2-goolf-1.4.10',
]
# parsed easyconfigs are cached, so clear the cache before reprocessing easyconfigs
ecec._easyconfigs_cache.clear()
bar = process_easyconfig(barec)[0]
res = resolve_dependencies([bar], self.modtool, retain_all_deps=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertTrue('SQLite/3.8.10.2-goolf-1.4.10' in mods)
self.assertFalse('SQLite/3.8.10.2-GCC-4.7.2' in mods)
# Check whether having 2 version of dummy toolchain is ok
# Clear easyconfig and toolchain caches
ecec._easyconfigs_cache.clear()
get_toolchain_hierarchy.clear()
init_config(build_options={
'allow_modules_tool_mismatch': True,
'minimal_toolchains': True,
'add_dummy_to_minimal_toolchains': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
'valid_module_classes': module_classes(),
'validate': False,
})
impi_txt = read_file(os.path.join(test_easyconfigs, 'i', 'impi', 'impi-4.1.3.049.eb'))
self.assertTrue(re.search("^toolchain = {'name': 'dummy', 'version': ''}", impi_txt, re.M))
gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4.eb'))
self.assertTrue(re.search("^toolchain = {'name': 'dummy', 'version': 'dummy'}", gzip_txt, re.M))
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_lines = [
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('impi', '4.1.3.049'),", # has toolchain ('dummy', '')
" ('gzip', '1.4'),", # has toolchain ('dummy', 'dummy')
"]",
# toolchain as list line, for easy modification later
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
]
write_file(barec, '\n'.join(barec_lines))
bar = process_easyconfig(barec)[0]
res = resolve_dependencies([bar], self.modtool, retain_all_deps=True)
self.assertEqual(len(res), 11)
mods = [x['full_mod_name'] for x in res]
self.assertTrue('impi/4.1.3.049' in mods)
self.assertTrue('gzip/1.4' in mods)
def test_get_toolchain_hierarchy(self):
"""Test get_toolchain_hierarchy function."""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'})
self.assertEqual(goolf_hierarchy, [
{'name': 'GCC', 'version': '4.7.2'},
{'name': 'gompi', 'version': '1.4.10'},
{'name': 'goolf', 'version': '1.4.10'},
])
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'})
self.assertEqual(iimpi_hierarchy, [
{'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'},
{'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
])
# test also including dummy
init_config(build_options={
'add_dummy_to_minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
get_toolchain_hierarchy.clear()
gompi_hierarchy = get_toolchain_hierarchy({'name': 'gompi', 'version': '1.4.10'})
self.assertEqual(gompi_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCC', 'version': '4.7.2'},
{'name': 'gompi', 'version': '1.4.10'},
])
get_toolchain_hierarchy.clear()
# check whether GCCcore is considered as subtoolchain, even if it's only listed as a dep
gcc_hierarchy = get_toolchain_hierarchy({'name': 'GCC', 'version': '4.9.3-2.25'})
self.assertEqual(gcc_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCCcore', 'version': '4.9.3'},
{'name': 'GCC', 'version': '4.9.3-2.25'},
])
get_toolchain_hierarchy.clear()
iccifort_hierarchy = get_toolchain_hierarchy({'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'})
self.assertEqual(iccifort_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCCcore', 'version': '4.9.3'},
{'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
])
get_toolchain_hierarchy.clear()
build_options = {
'add_dummy_to_minimal_toolchains': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
}
init_config(build_options=build_options)
craycce_hierarchy = get_toolchain_hierarchy({'name': 'CrayCCE', 'version': '5.1.29'})
self.assertEqual(craycce_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'CrayCCE', 'version': '5.1.29'},
])