def test_check_capability_mapping(self):
"""Test comparing the functionality of two toolchains"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
get_toolchain_hierarchy.clear()
foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True)
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '2016.01'},
incl_capabilities=True)
# Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2
self.assertEqual(foss_hierarchy[0]['name'], 'GCC')
self.assertEqual(foss_hierarchy[1]['name'], 'golf')
self.assertEqual(foss_hierarchy[2]['name'], 'gompi')
self.assertEqual(foss_hierarchy[3]['name'], 'foss')
self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore')
self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort')
self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi')
# golf <-> iimpi (should return False)
self.assertFalse(check_capability_mapping(foss_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs")
# gompi <-> iimpi
self.assertTrue(check_capability_mapping(foss_hierarchy[2], iimpi_hierarchy[2]))
# GCC <-> iimpi
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[2]))
# GCC <-> iccifort
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[1]))
# GCC <-> GCCcore
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[0]))
def test_check_capability_mapping(self):
"""Test comparing the functionality of two toolchains"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
get_toolchain_hierarchy.clear()
foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True)
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '2016.01'},
incl_capabilities=True)
# Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2
self.assertEqual(foss_hierarchy[0]['name'], 'GCC')
self.assertEqual(foss_hierarchy[1]['name'], 'golf')
self.assertEqual(foss_hierarchy[2]['name'], 'gompi')
self.assertEqual(foss_hierarchy[3]['name'], 'foss')
self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore')
self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort')
self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi')
# golf <-> iimpi (should return False)
self.assertFalse(check_capability_mapping(foss_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs")
# gompi <-> iimpi
self.assertTrue(check_capability_mapping(foss_hierarchy[2], iimpi_hierarchy[2]))
# GCC <-> iimpi
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[2]))
# GCC <-> iccifort
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[1]))
# GCC <-> GCCcore
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[0]))
def test_match_minimum_tc_specs(self):
"""Test matching a toolchain to lowest possible in a hierarchy"""
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
init_config(
build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
foss_hierarchy = get_toolchain_hierarchy(
{
'name': 'foss',
'version': '2018a'
}, incl_capabilities=True)
iimpi_hierarchy = get_toolchain_hierarchy(
{
'name': 'iimpi',
'version': '2016.01'
}, incl_capabilities=True)
# Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2
self.assertEqual(foss_hierarchy[0]['name'], 'GCC')
self.assertEqual(foss_hierarchy[1]['name'], 'golf')
self.assertEqual(foss_hierarchy[2]['name'], 'gompi')
self.assertEqual(foss_hierarchy[3]['name'], 'foss')
self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore')
self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort')
self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi')
# base compiler first (GCCcore which maps to GCC/6.4.0-2.28)
self.assertEqual(
match_minimum_tc_specs(iimpi_hierarchy[0], foss_hierarchy), {
'name': 'GCC',
'version': '6.4.0-2.28'
})
# then iccifort (which also maps to GCC/6.4.0-2.28)
self.assertEqual(
match_minimum_tc_specs(iimpi_hierarchy[1], foss_hierarchy), {
'name': 'GCC',
'version': '6.4.0-2.28'
})
# Then MPI
self.assertEqual(
match_minimum_tc_specs(iimpi_hierarchy[2], foss_hierarchy), {
'name': 'gompi',
'version': '2018a'
})
# Check against own math only subtoolchain for math
self.assertEqual(
match_minimum_tc_specs(foss_hierarchy[1], foss_hierarchy), {
'name': 'golf',
'version': '2018a'
})
# Make sure there's an error when we can't do the mapping
error_msg = "No possible mapping from source toolchain spec .*"
self.assertErrorRegex(EasyBuildError, error_msg,
match_minimum_tc_specs, foss_hierarchy[3],
iimpi_hierarchy)
def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool):
"""
Create a map between toolchain hierarchy of the initial toolchain and that of the target toolchain
:param source_toolchain: initial toolchain of the easyconfig(s)
:param target_toolchain: target toolchain for tweaked easyconfig(s)
:param modtool: module tool used
:return: mapping from source hierarchy to target hierarchy
"""
tc_mapping = {}
source_tc_hierarchy = get_toolchain_hierarchy(source_toolchain,
incl_capabilities=True)
target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain,
incl_capabilities=True)
for toolchain_spec in source_tc_hierarchy:
tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(
toolchain_spec, target_tc_hierarchy)
# Check for presence of binutils in source and target toolchain dependency trees
# (only do this when GCCcore is present in both and GCCcore is not the top of the tree)
gcccore = GCCcore.NAME
source_tc_names = [tc_spec['name'] for tc_spec in source_tc_hierarchy]
target_tc_names = [tc_spec['name'] for tc_spec in target_tc_hierarchy]
if gcccore in source_tc_names and gcccore in target_tc_names and source_tc_hierarchy[
-1]['name'] != gcccore:
binutils = 'binutils'
# Determine the dependency trees
source_dep_tree = get_dep_tree_of_toolchain(source_tc_hierarchy[-1],
modtool)
target_dep_tree = get_dep_tree_of_toolchain(target_tc_hierarchy[-1],
modtool)
# Find the binutils mapping
if binutils in [dep['name'] for dep in source_dep_tree]:
# We need the binutils that was built using GCCcore (we assume that everything is using standard behaviour:
# build binutils with GCCcore and then use that for anything built with GCCcore)
binutils_deps = [
dep for dep in target_dep_tree if dep['name'] == binutils
]
binutils_gcccore_deps = [
dep for dep in binutils_deps
if dep['toolchain']['name'] == gcccore
]
if len(binutils_gcccore_deps) == 1:
tc_mapping[binutils] = {
'version': binutils_gcccore_deps[0]['version'],
'versionsuffix': binutils_gcccore_deps[0]['versionsuffix']
}
else:
raise EasyBuildError(
"Target hierarchy %s should have binutils using GCCcore, can't determine mapping!",
target_tc_hierarchy[-1])
return tc_mapping
def test_match_minimum_tc_specs(self):
"""Test matching a toolchain to lowest possible in a hierarchy"""
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
init_config(
build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
get_toolchain_hierarchy.clear()
goolf_hierarchy = get_toolchain_hierarchy(
{
'name': 'goolf',
'version': '1.4.10'
}, incl_capabilities=True)
iimpi_hierarchy = get_toolchain_hierarchy(
{
'name': 'iimpi',
'version': '5.5.3-GCC-4.8.3'
},
incl_capabilities=True)
# Hierarchies are returned with top-level toolchain last, goolf has 4 elements here, intel has 2
self.assertEqual(goolf_hierarchy[0]['name'], 'GCC')
self.assertEqual(goolf_hierarchy[1]['name'], 'golf')
self.assertEqual(goolf_hierarchy[2]['name'], 'gompi')
self.assertEqual(goolf_hierarchy[3]['name'], 'goolf')
self.assertEqual(iimpi_hierarchy[0]['name'], 'iccifort')
self.assertEqual(iimpi_hierarchy[1]['name'], 'iimpi')
# Compiler first
self.assertEqual(
match_minimum_tc_specs(iimpi_hierarchy[0], goolf_hierarchy), {
'name': 'GCC',
'version': '4.7.2'
})
# Then MPI
self.assertEqual(
match_minimum_tc_specs(iimpi_hierarchy[1], goolf_hierarchy), {
'name': 'gompi',
'version': '1.4.10'
})
# Check against own math only subtoolchain for math
self.assertEqual(
match_minimum_tc_specs(goolf_hierarchy[1], goolf_hierarchy), {
'name': 'golf',
'version': '1.4.10'
})
# Make sure there's an error when we can't do the mapping
error_msg = "No possible mapping from source toolchain spec .*"
self.assertErrorRegex(EasyBuildError, error_msg,
match_minimum_tc_specs, goolf_hierarchy[3],
iimpi_hierarchy)
def test_get_toolchain_hierarchy(self):
"""Test get_toolchain_hierarchy function."""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'})
self.assertEqual(goolf_hierarchy, [
{'name': 'GCC', 'version': '4.7.2'},
{'name': 'gompi', 'version': '1.4.10'},
{'name': 'goolf', 'version': '1.4.10'},
])
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'})
self.assertEqual(iimpi_hierarchy, [
{'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'},
{'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
])
# test also including dummy
init_config(build_options={
'add_dummy_to_minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
get_toolchain_hierarchy.clear()
gompi_hierarchy = get_toolchain_hierarchy({'name': 'gompi', 'version': '1.4.10'})
self.assertEqual(gompi_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCC', 'version': '4.7.2'},
{'name': 'gompi', 'version': '1.4.10'},
])
get_toolchain_hierarchy.clear()
# check whether GCCcore is considered as subtoolchain, even if it's only listed as a dep
gcc_hierarchy = get_toolchain_hierarchy({'name': 'GCC', 'version': '4.9.3-2.25'})
self.assertEqual(gcc_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCCcore', 'version': '4.9.3'},
{'name': 'GCC', 'version': '4.9.3-2.25'},
])
get_toolchain_hierarchy.clear()
iccifort_hierarchy = get_toolchain_hierarchy({'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'})
self.assertEqual(iccifort_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCCcore', 'version': '4.9.3'},
{'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
])
def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool):
"""
Create a map between toolchain hierarchy of the initial toolchain and that of the target toolchain
:param source_toolchain: initial toolchain of the easyconfig(s)
:param target_toolchain: target toolchain for tweaked easyconfig(s)
:param modtool: module tool used
:return: mapping from source hierarchy to target hierarchy
"""
tc_mapping = {}
source_tc_hierarchy = get_toolchain_hierarchy(source_toolchain, incl_capabilities=True)
target_tc_hierarchy = get_toolchain_hierarchy(target_toolchain, incl_capabilities=True)
for toolchain_spec in source_tc_hierarchy:
tc_mapping[toolchain_spec['name']] = match_minimum_tc_specs(toolchain_spec, target_tc_hierarchy)
# Check for presence of binutils in source and target toolchain dependency trees
# (only do this when GCCcore is present in both and GCCcore is not the top of the tree)
gcccore = GCCcore.NAME
source_tc_names = [tc_spec['name'] for tc_spec in source_tc_hierarchy]
target_tc_names = [tc_spec['name'] for tc_spec in target_tc_hierarchy]
if gcccore in source_tc_names and gcccore in target_tc_names and source_tc_hierarchy[-1]['name'] != gcccore:
binutils = 'binutils'
# Determine the dependency trees
source_dep_tree = get_dep_tree_of_toolchain(source_tc_hierarchy[-1], modtool)
target_dep_tree = get_dep_tree_of_toolchain(target_tc_hierarchy[-1], modtool)
# Find the binutils mapping
if binutils in [dep['name'] for dep in source_dep_tree]:
# We need the binutils that was built using GCCcore (we assume that everything is using standard behaviour:
# build binutils with GCCcore and then use that for anything built with GCCcore)
binutils_deps = [dep for dep in target_dep_tree if dep['name'] == binutils]
binutils_gcccore_deps = [dep for dep in binutils_deps if dep['toolchain']['name'] == gcccore]
if len(binutils_gcccore_deps) == 1:
tc_mapping[binutils] = {'version': binutils_gcccore_deps[0]['version'],
'versionsuffix': binutils_gcccore_deps[0]['versionsuffix']}
else:
raise EasyBuildError("Target hierarchy %s should have binutils using GCCcore, can't determine mapping!",
target_tc_hierarchy[-1])
return tc_mapping
def test_match_minimum_tc_specs(self):
"""Test matching a toolchain to lowest possible in a hierarchy"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True)
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '2016.01'},
incl_capabilities=True)
# Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2
self.assertEqual(foss_hierarchy[0]['name'], 'GCC')
self.assertEqual(foss_hierarchy[1]['name'], 'golf')
self.assertEqual(foss_hierarchy[2]['name'], 'gompi')
self.assertEqual(foss_hierarchy[3]['name'], 'foss')
self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore')
self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort')
self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi')
# base compiler first (GCCcore which maps to GCC/6.4.0-2.28)
self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[0], foss_hierarchy),
{'name': 'GCC', 'version': '6.4.0-2.28'})
# then iccifort (which also maps to GCC/6.4.0-2.28)
self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[1], foss_hierarchy),
{'name': 'GCC', 'version': '6.4.0-2.28'})
# Then MPI
self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[2], foss_hierarchy),
{'name': 'gompi', 'version': '2018a'})
# Check against own math only subtoolchain for math
self.assertEqual(match_minimum_tc_specs(foss_hierarchy[1], foss_hierarchy),
{'name': 'golf', 'version': '2018a'})
# Make sure there's an error when we can't do the mapping
error_msg = "No possible mapping from source toolchain spec .*"
self.assertErrorRegex(EasyBuildError, error_msg, match_minimum_tc_specs,
foss_hierarchy[3], iimpi_hierarchy)
def test_get_toolchain_hierarchy(self):
"""Test get_toolchain_hierarchy function."""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'})
self.assertEqual(goolf_hierarchy, [
{'name': 'GCC', 'version': '4.7.2'},
{'name': 'gompi', 'version': '1.4.10'},
{'name': 'goolf', 'version': '1.4.10'},
])
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'})
self.assertEqual(iimpi_hierarchy, [
{'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'},
{'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
])
# test also including dummy
init_config(build_options={
'add_dummy_to_minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
get_toolchain_hierarchy.clear()
gompi_hierarchy = get_toolchain_hierarchy({'name': 'gompi', 'version': '1.4.10'})
self.assertEqual(gompi_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCC', 'version': '4.7.2'},
{'name': 'gompi', 'version': '1.4.10'},
])
get_toolchain_hierarchy.clear()
# check whether GCCcore is considered as subtoolchain, even if it's only listed as a dep
gcc_hierarchy = get_toolchain_hierarchy({'name': 'GCC', 'version': '4.9.3-2.25'})
self.assertEqual(gcc_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCCcore', 'version': '4.9.3'},
{'name': 'GCC', 'version': '4.9.3-2.25'},
])
get_toolchain_hierarchy.clear()
iccifort_hierarchy = get_toolchain_hierarchy({'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'})
self.assertEqual(iccifort_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCCcore', 'version': '4.9.3'},
{'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
])
get_toolchain_hierarchy.clear()
build_options = {
'add_dummy_to_minimal_toolchains': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
}
init_config(build_options=build_options)
craycce_hierarchy = get_toolchain_hierarchy({'name': 'CrayCCE', 'version': '5.1.29'})
self.assertEqual(craycce_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'CrayCCE', 'version': '5.1.29'},
])
def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
"""Tweak list of easyconfigs according to provided build specifications."""
tweaked_ecs_path, tweaked_ecs_deps_path = None, None
if targetdirs is not None:
tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(
['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
raise EasyBuildError(
"Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s",
toolchains)
# Toolchain is unique, let's store it
source_toolchain = easyconfigs[-1]['ec']['toolchain']
modifying_toolchains = False
target_toolchain = {}
src_to_dst_tc_mapping = {}
revert_to_regex = False
if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs:
keys = build_specs.keys()
# Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not
# supported
if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain']
for key in keys):
print_warning(
"Combining --try-toolchain* with other build options is not fully supported: using regex"
)
revert_to_regex = True
if not revert_to_regex:
# we're doing something with the toolchain,
# so build specifications should be applied to whole dependency graph;
# obtain full dependency graph for specified easyconfigs;
# easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first)
modifying_toolchains = True
if 'toolchain_name' in keys:
target_toolchain['name'] = build_specs['toolchain_name']
else:
target_toolchain['name'] = source_toolchain['name']
if 'toolchain_version' in keys:
target_toolchain['version'] = build_specs['toolchain_version']
else:
target_toolchain['version'] = source_toolchain['version']
if build_option('map_toolchains'):
try:
src_to_dst_tc_mapping = map_toolchain_hierarchies(
source_toolchain, target_toolchain, modtool)
except EasyBuildError as err:
# make sure exception was raised by match_minimum_tc_specs because toolchain mapping didn't work
if "No possible mapping from source toolchain" in err.msg:
error_msg = err.msg + '\n'
error_msg += "Toolchain %s is not equivalent to toolchain %s in terms of capabilities. "
error_msg += "(If you know what you are doing, "
error_msg += "you can use --disable-map-toolchains to proceed anyway.)"
raise EasyBuildError(error_msg,
target_toolchain['name'],
source_toolchain['name'])
else:
# simply re-raise the exception if something else went wrong
raise err
else:
msg = "Mapping of (sub)toolchains disabled, so falling back to regex mode, "
msg += "disabling recursion and not changing (sub)toolchains for dependencies"
_log.info(msg)
revert_to_regex = True
modifying_toolchains = False
if not revert_to_regex:
_log.debug(
"Applying build specifications recursively (no software name/version found): %s",
build_specs)
orig_ecs = resolve_dependencies(easyconfigs,
modtool,
retain_all_deps=True)
# Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively)
# We can leave any dependencies they may have as they will only be used if required (or originally listed)
_log.debug("Filtering out toolchain hierarchy for %s",
source_toolchain)
i = 0
while i < len(orig_ecs):
tc_names = [
tc['name']
for tc in get_toolchain_hierarchy(source_toolchain)
]
if orig_ecs[i]['ec']['name'] in tc_names:
# drop elements in toolchain hierarchy
del orig_ecs[i]
else:
i += 1
else:
revert_to_regex = True
if revert_to_regex:
# no recursion if software name/version build specification are included or we are amending something
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug(
"Software name/version found, so not applying build specifications recursively: %s"
% build_specs)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
# Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line
# (and use the prepended path so that they are found first).
# easyconfig files for dependencies are also generated but not included, they will be resolved via --robot
# either from existing easyconfigs or, if that fails, from easyconfigs in the appended path
tc_name = orig_ec['ec']['toolchain']['name']
new_ec_file = None
verification_build_specs = copy.copy(build_specs)
if orig_ec['spec'] in listed_ec_paths:
if modifying_toolchains:
if tc_name in src_to_dst_tc_mapping:
new_ec_file = map_easyconfig_to_target_tc_hierarchy(
orig_ec['spec'], src_to_dst_tc_mapping,
tweaked_ecs_path)
# Need to update the toolchain in the build_specs to match the toolchain mapping
keys = verification_build_specs.keys()
if 'toolchain_name' in keys:
verification_build_specs[
'toolchain_name'] = src_to_dst_tc_mapping[tc_name][
'name']
if 'toolchain_version' in keys:
verification_build_specs[
'toolchain_version'] = src_to_dst_tc_mapping[
tc_name]['version']
if 'toolchain' in keys:
verification_build_specs[
'toolchain'] = src_to_dst_tc_mapping[tc_name]
else:
new_ec_file = tweak_one(orig_ec['spec'],
None,
build_specs,
targetdir=tweaked_ecs_path)
if new_ec_file:
new_ecs = process_easyconfig(
new_ec_file, build_specs=verification_build_specs)
tweaked_easyconfigs.extend(new_ecs)
else:
# Place all tweaked dependency easyconfigs in the directory appended to the robot path
if modifying_toolchains:
if tc_name in src_to_dst_tc_mapping:
new_ec_file = map_easyconfig_to_target_tc_hierarchy(
orig_ec['spec'],
src_to_dst_tc_mapping,
targetdir=tweaked_ecs_deps_path)
else:
new_ec_file = tweak_one(orig_ec['spec'],
None,
build_specs,
targetdir=tweaked_ecs_deps_path)
return tweaked_easyconfigs
def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
"""Tweak list of easyconfigs according to provided build specifications."""
tweaked_ecs_path, tweaked_ecs_deps_path = None, None
if targetdirs is not None:
tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s",
toolchains)
# Toolchain is unique, let's store it
source_toolchain = easyconfigs[-1]['ec']['toolchain']
modifying_toolchains = False
target_toolchain = {}
src_to_dst_tc_mapping = {}
revert_to_regex = False
if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs:
keys = build_specs.keys()
# Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not
# supported
if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain'] for key in keys):
warning_msg = "Combining --try-toolchain* with other build options is not fully supported: using regex"
print_warning(warning_msg, silent=build_option('silent'))
revert_to_regex = True
if not revert_to_regex:
# we're doing something with the toolchain,
# so build specifications should be applied to whole dependency graph;
# obtain full dependency graph for specified easyconfigs;
# easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first)
modifying_toolchains = True
if 'toolchain_name' in keys:
target_toolchain['name'] = build_specs['toolchain_name']
else:
target_toolchain['name'] = source_toolchain['name']
if 'toolchain_version' in keys:
target_toolchain['version'] = build_specs['toolchain_version']
else:
target_toolchain['version'] = source_toolchain['version']
if build_option('map_toolchains'):
try:
src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool)
except EasyBuildError as err:
# make sure exception was raised by match_minimum_tc_specs because toolchain mapping didn't work
if "No possible mapping from source toolchain" in err.msg:
error_msg = err.msg + '\n'
error_msg += "Toolchain %s is not equivalent to toolchain %s in terms of capabilities. "
error_msg += "(If you know what you are doing, "
error_msg += "you can use --disable-map-toolchains to proceed anyway.)"
raise EasyBuildError(error_msg, target_toolchain['name'], source_toolchain['name'])
else:
# simply re-raise the exception if something else went wrong
raise err
else:
msg = "Mapping of (sub)toolchains disabled, so falling back to regex mode, "
msg += "disabling recursion and not changing (sub)toolchains for dependencies"
_log.info(msg)
revert_to_regex = True
modifying_toolchains = False
if not revert_to_regex:
_log.debug("Applying build specifications recursively (no software name/version found): %s", build_specs)
orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True)
# Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively)
# We can leave any dependencies they may have as they will only be used if required (or originally listed)
_log.debug("Filtering out toolchain hierarchy for %s", source_toolchain)
i = 0
while i < len(orig_ecs):
tc_names = [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)]
if orig_ecs[i]['ec']['name'] in tc_names:
# drop elements in toolchain hierarchy
del orig_ecs[i]
else:
i += 1
else:
revert_to_regex = True
if revert_to_regex:
# no recursion if software name/version build specification are included or we are amending something
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
# Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line
# (and use the prepended path so that they are found first).
# easyconfig files for dependencies are also generated but not included, they will be resolved via --robot
# either from existing easyconfigs or, if that fails, from easyconfigs in the appended path
tc_name = orig_ec['ec']['toolchain']['name']
new_ec_file = None
verification_build_specs = copy.copy(build_specs)
if orig_ec['spec'] in listed_ec_paths:
if modifying_toolchains:
if tc_name in src_to_dst_tc_mapping:
new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
tweaked_ecs_path)
# Need to update the toolchain in the build_specs to match the toolchain mapping
keys = verification_build_specs.keys()
if 'toolchain_name' in keys:
verification_build_specs['toolchain_name'] = src_to_dst_tc_mapping[tc_name]['name']
if 'toolchain_version' in keys:
verification_build_specs['toolchain_version'] = src_to_dst_tc_mapping[tc_name]['version']
if 'toolchain' in keys:
verification_build_specs['toolchain'] = src_to_dst_tc_mapping[tc_name]
else:
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path)
if new_ec_file:
new_ecs = process_easyconfig(new_ec_file, build_specs=verification_build_specs)
tweaked_easyconfigs.extend(new_ecs)
else:
# Place all tweaked dependency easyconfigs in the directory appended to the robot path
if modifying_toolchains:
if tc_name in src_to_dst_tc_mapping:
new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
targetdir=tweaked_ecs_deps_path)
else:
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
return tweaked_easyconfigs
