def test_map_common_versionsuffixes(self):
"""Test mapping between two toolchain hierarchies"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'robot_path': [test_easyconfigs],
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
toolchain_mapping = map_toolchain_hierarchies(iccifort_binutils_tc, gcc_binutils_tc, self.modtool)
possible_mappings = map_common_versionsuffixes('binutils', iccifort_binutils_tc, toolchain_mapping)
self.assertEqual(possible_mappings, {'-binutils-2.25': '-binutils-2.26'})
# Make sure we only map upwards, here it's gzip 1.4 in gcc and 1.6 in iccifort
possible_mappings = map_common_versionsuffixes('gzip', iccifort_binutils_tc, toolchain_mapping)
self.assertEqual(possible_mappings, {})
# newer gzip is picked up other way around (GCC -> iccifort)
toolchain_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool)
possible_mappings = map_common_versionsuffixes('gzip', gcc_binutils_tc, toolchain_mapping)
self.assertEqual(possible_mappings, {'-gzip-1.4': '-gzip-1.6'})
def test_map_easyconfig_to_target_tc_hierarchy(self):
"""Test mapping of easyconfig to target hierarchy"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
# The below mapping includes a binutils mapping (2.26 to 2.25)
tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool)
ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb')
tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
tweaked_dict = tweaked_ec['ec'].asdict()
# First check the mapped toolchain
key, value = 'toolchain', iccifort_binutils_tc
self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
# Also check that binutils has been mapped
for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items():
self.assertTrue(key in tweaked_dict['builddependencies'][0] and
value == tweaked_dict['builddependencies'][0][key])
def test_map_toolchain_hierarchies(self):
"""Test mapping between two toolchain hierarchies"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
foss_tc = {'name': 'foss', 'version': '2018a'}
gompi_tc = {'name': 'gompi', 'version': '2018a'}
iimpi_tc = {'name': 'iimpi', 'version': '2016.01'}
# GCCcore is mapped to GCC, iccifort is mapped to GCC, iimpi is mapped to gompi
expected = {
'GCCcore': {'name': 'GCC', 'version': '6.4.0-2.28'},
'iccifort': {'name': 'GCC', 'version': '6.4.0-2.28'},
'iimpi': {'name': 'gompi', 'version': '2018a'},
}
self.assertEqual(map_toolchain_hierarchies(iimpi_tc, foss_tc, self.modtool), expected)
# GCC is mapped to iccifort, gompi is mapped to iimpi
expected = {
'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
'gompi': {'name': 'iimpi', 'version': '2016.01'}
}
self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc, self.modtool), expected)
# Expect an error when there is no possible mapping
error_msg = "No possible mapping from source toolchain spec .*"
self.assertErrorRegex(EasyBuildError, error_msg, map_toolchain_hierarchies,
foss_tc, iimpi_tc, self.modtool)
# Test that we correctly include GCCcore binutils when it is there
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
# Should see a binutils in the mapping (2.26 will get mapped to 2.25)
expected = {
'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
'GCCcore': {'name': 'GCCcore', 'version': '4.9.3'},
'binutils': {'version': '2.25', 'versionsuffix': ''}
}
self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), expected)
def test_map_toolchain_hierarchies(self):
"""Test mapping between two toolchain hierarchies"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
foss_tc = {'name': 'foss', 'version': '2018a'}
gompi_tc = {'name': 'gompi', 'version': '2018a'}
iimpi_tc = {'name': 'iimpi', 'version': '2016.01'}
# GCCcore is mapped to GCC, iccifort is mapped to GCC, iimpi is mapped to gompi
expected = {
'GCCcore': {'name': 'GCC', 'version': '6.4.0-2.28'},
'iccifort': {'name': 'GCC', 'version': '6.4.0-2.28'},
'iimpi': {'name': 'gompi', 'version': '2018a'},
}
self.assertEqual(map_toolchain_hierarchies(iimpi_tc, foss_tc, self.modtool), expected)
# GCC is mapped to iccifort, gompi is mapped to iimpi
expected = {
'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
'gompi': {'name': 'iimpi', 'version': '2016.01'}
}
self.assertEqual(map_toolchain_hierarchies(gompi_tc, iimpi_tc, self.modtool), expected)
# Expect an error when there is no possible mapping
error_msg = "No possible mapping from source toolchain spec .*"
self.assertErrorRegex(EasyBuildError, error_msg, map_toolchain_hierarchies,
foss_tc, iimpi_tc, self.modtool)
# Test that we correctly include GCCcore binutils when it is there
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
# Should see a binutils in the mapping (2.26 will get mapped to 2.25)
expected = {
'GCC': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
'GCCcore': {'name': 'GCCcore', 'version': '4.9.3'},
'binutils': {'version': '2.25', 'versionsuffix': ''}
}
self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), expected)
def test_find_potential_version_mappings(self):
"""Test ability to find potential version mappings of a dependency for a given toolchain mapping"""
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
init_config(
build_options={
'robot_path': [test_easyconfigs],
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {
'name': 'iccifort',
'version': '2016.1.150-GCC-4.9.3-2.25'
}
# The below mapping includes a binutils mapping (2.26 to 2.25)
tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc,
iccifort_binutils_tc,
self.modtool)
ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc',
'hwloc-1.6.2-GCC-4.9.3-2.26.eb')
parsed_ec = process_easyconfig(ec_spec)[0]
gzip_dep = [
dep for dep in parsed_ec['ec']['dependencies']
if dep['name'] == 'gzip'
][0]
self.assertEqual(gzip_dep['full_mod_name'], 'gzip/1.4-GCC-4.9.3-2.26')
potential_versions = find_potential_version_mappings(
gzip_dep, tc_mapping)
self.assertEqual(len(potential_versions), 1)
# Should see version 1.6 of gzip with iccifort toolchain
expected = {
'path':
os.path.join(test_easyconfigs, 'g', 'gzip',
'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'),
'toolchain': {
'name': 'iccifort',
'version': '2016.1.150-GCC-4.9.3-2.25'
},
'version':
'1.6',
}
self.assertEqual(potential_versions[0], expected)
def test_map_easyconfig_to_target_tc_hierarchy(self):
"""Test mapping of easyconfig to target hierarchy"""
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
init_config(
build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {
'name': 'iccifort',
'version': '2016.1.150-GCC-4.9.3-2.25'
}
# The below mapping includes a binutils mapping (2.26 to 2.25)
tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc,
iccifort_binutils_tc,
self.modtool)
ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc',
'hwloc-1.6.2-GCC-4.9.3-2.26.eb')
tweaked_spec = map_easyconfig_to_target_tc_hierarchy(
ec_spec, tc_mapping)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
tweaked_dict = tweaked_ec['ec'].asdict()
# First check the mapped toolchain
key, value = 'toolchain', iccifort_binutils_tc
self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
# Also check that binutils has been mapped
for key, value in {
'name': 'binutils',
'version': '2.25',
'versionsuffix': ''
}.items():
self.assertTrue(
key in tweaked_dict['builddependencies'][0]
and value == tweaked_dict['builddependencies'][0][key])
def test_map_easyconfig_to_target_tc_hierarchy(self):
"""Test mapping of easyconfig to target hierarchy"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
build_options = {
'robot_path': [test_easyconfigs],
'silent': True,
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
get_toolchain_hierarchy.clear()
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
# The below mapping includes a binutils mapping (2.26 to 2.25)
tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool)
ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb')
tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
tweaked_dict = tweaked_ec['ec'].asdict()
# First check the mapped toolchain
key, value = 'toolchain', iccifort_binutils_tc
self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
# Also check that binutils has been mapped
for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items():
self.assertTrue(key in tweaked_dict['builddependencies'][0] and
value == tweaked_dict['builddependencies'][0][key])
# Now test the case where we try to update the dependencies
init_config(build_options=build_options)
get_toolchain_hierarchy.clear()
tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping, update_dep_versions=True)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
tweaked_dict = tweaked_ec['ec'].asdict()
# First check the mapped toolchain
key, value = 'toolchain', iccifort_binutils_tc
self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
# Also check that binutils has been mapped
for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items():
self.assertTrue(
key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key]
)
# Also check that the gzip dependency was upgraded
for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items():
self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key])
# Make sure there are checksums for our next test
self.assertTrue(tweaked_dict['checksums'])
# Test the case where we also update the software version at the same time
init_config(build_options=build_options)
get_toolchain_hierarchy.clear()
new_version = '1.x.3'
tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec,
tc_mapping,
update_build_specs={'version': new_version},
update_dep_versions=True)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
tweaked_dict = tweaked_ec['ec'].asdict()
# First check the mapped toolchain
key, value = 'toolchain', iccifort_binutils_tc
self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
# Also check that binutils has been mapped
for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items():
self.assertTrue(
key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key]
)
# Also check that the gzip dependency was upgraded
for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items():
self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key])
# Finally check that the version was upgraded
key, value = 'version', new_version
self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
# and that the checksum was removed
self.assertFalse(tweaked_dict['checksums'])
# Check that if we update a software version, it also updates the version if the software appears in an
# extension list (like for a PythonBundle)
ec_spec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
# Create the trivial toolchain mapping
toolchain = {'name': 'gompi', 'version': '2018a'}
tc_mapping = map_toolchain_hierarchies(toolchain, toolchain, self.modtool)
# Update the software version
init_config(build_options=build_options)
get_toolchain_hierarchy.clear()
new_version = '1.x.3'
tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec,
tc_mapping,
update_build_specs={'version': new_version},
update_dep_versions=False)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
extensions = tweaked_ec['ec']['exts_list']
# check one extension with the same name exists and that the version has been updated
hit_extension = 0
for extension in extensions:
if isinstance(extension, tuple) and extension[0] == 'toy':
self.assertEqual(extension[1], new_version)
# Make sure checksum has been purged
self.assertFalse('checksums' in extension[2])
hit_extension += 1
self.assertEqual(hit_extension, 1, "Should only have updated one extension")
def test_find_potential_version_mappings(self):
"""Test ability to find potential version mappings of a dependency for a given toolchain mapping"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'robot_path': [test_easyconfigs],
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
# The below mapping includes a binutils mapping (2.26 to 2.25)
tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool)
ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb')
parsed_ec = process_easyconfig(ec_spec)[0]
gzip_dep = [dep for dep in parsed_ec['ec']['dependencies'] if dep['name'] == 'gzip'][0]
self.assertEqual(gzip_dep['full_mod_name'], 'gzip/1.4-GCC-4.9.3-2.26')
potential_versions = find_potential_version_mappings(gzip_dep, tc_mapping)
self.assertEqual(len(potential_versions), 1)
# Should see version 1.6 of gzip with iccifort toolchain
expected = {
'path': os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'),
'toolchain': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
'version': '1.6',
'versionsuffix': '',
}
self.assertEqual(potential_versions[0], expected)
# Test that we can override respecting the versionsuffix
# Create toolchain mapping for OpenBLAS
gcc_4_tc = {'name': 'GCC', 'version': '4.8.2'}
gcc_6_tc = {'name': 'GCC', 'version': '6.4.0-2.28'}
tc_mapping = map_toolchain_hierarchies(gcc_4_tc, gcc_6_tc, self.modtool)
# Create a dep with the necessary params (including versionsuffix)
openblas_dep = {
'toolchain': {'version': '4.8.2', 'name': 'GCC'},
'name': 'OpenBLAS',
'system': False,
'versionsuffix': '-LAPACK-3.4.2',
'version': '0.2.8'
}
self.mock_stderr(True)
potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping)
errtxt = self.get_stderr()
warning_stub = "\nWARNING: There may be newer version(s) of dep 'OpenBLAS' available with a different " \
"versionsuffix to '-LAPACK-3.4.2'"
self.mock_stderr(False)
self.assertTrue(errtxt.startswith(warning_stub))
self.assertEqual(len(potential_versions), 0)
potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping, ignore_versionsuffixes=True)
self.assertEqual(len(potential_versions), 1)
expected = {
'path': os.path.join(test_easyconfigs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb'),
'toolchain': {'version': '6.4.0-2.28', 'name': 'GCC'},
'version': '0.2.20',
'versionsuffix': '',
}
self.assertEqual(potential_versions[0], expected)
