def det_toolchain_element_details(tc, elem):
"""
Determine details of a particular toolchain element, for a given Toolchain instance.
"""
# check for cached version first
tc_dict = tc.as_dict()
key = (tc_dict['name'], tc_dict['version'] + tc_dict['versionsuffix'], elem)
if key in _toolchain_details_cache:
_log.debug("Obtained details for '%s' in toolchain '%s' from cache" % (elem, tc_dict))
return _toolchain_details_cache[key]
# grab version from parsed easyconfig file for toolchain
eb_file = robot_find_easyconfig(tc_dict['name'], det_full_ec_version(tc_dict))
tc_ec = process_easyconfig(eb_file, parse_only=True)
if len(tc_ec) > 1:
_log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict)
_log.debug("Full list of toolchain specifications: %s" % tc_ec)
tc_ec = tc_ec[0]['ec']
tc_deps = tc_ec['dependencies']
tc_elem_details = None
for tc_dep in tc_deps:
if tc_dep['name'] == elem:
tc_elem_details = tc_dep
_log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details))
break
if tc_elem_details is None:
# for compiler-only toolchains, toolchain and compilers are one-and-the-same
if tc_ec['name'] == elem:
tc_elem_details = tc_ec
else:
raise EasyBuildError("No toolchain element '%s' found for toolchain %s: %s", elem, tc.as_dict(), tc_ec)
_toolchain_details_cache[key] = tc_elem_details
_log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict))
return _toolchain_details_cache[key]
def det_toolchain_element_details(tc, elem):
"""
Determine details of a particular toolchain element, for a given Toolchain instance.
"""
# check for cached version first
tc_dict = tc.as_dict()
key = (tc_dict['name'], tc_dict['version'] + tc_dict['versionsuffix'], elem)
if key in _toolchain_details_cache:
_log.debug("Obtained details for '%s' in toolchain '%s' from cache" % (elem, tc_dict))
return _toolchain_details_cache[key]
# grab version from parsed easyconfig file for toolchain
eb_file = robot_find_easyconfig(tc_dict['name'], det_full_ec_version(tc_dict))
tc_ec = process_easyconfig(eb_file, parse_only=True)
if len(tc_ec) > 1:
_log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict)
_log.debug("Full list of toolchain specifications: %s" % tc_ec)
tc_ec = tc_ec[0]['ec']
tc_deps = tc_ec['dependencies']
tc_elem_details = None
for tc_dep in tc_deps:
if tc_dep['name'] == elem:
tc_elem_details = tc_dep
_log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details))
break
if tc_elem_details is None:
# for compiler-only toolchains, toolchain and compilers are one-and-the-same
if tc_ec['name'] == elem:
tc_elem_details = tc_ec
else:
_log.error("No toolchain element '%s' found for toolchain %s: %s" % (elem, tc.as_dict(), tc_ec))
_toolchain_details_cache[key] = tc_elem_details
_log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict))
return _toolchain_details_cache[key]
def robot_find_minimal_easyconfig_for_dependency(dep):
"""
Find an easyconfig with minimal toolchain for a dependency
"""
newdep = copy.deepcopy(dep)
toolchain_hierarchy = get_toolchain_hierarchy(dep['toolchain'])
res = None
# reversed search: start with subtoolchains first, i.e. first (dummy or) compiler-only toolchain, etc.
for toolchain in toolchain_hierarchy:
newdep['toolchain'] = toolchain
eb_file = robot_find_easyconfig(newdep['name'],
det_full_ec_version(newdep))
if eb_file is not None:
if newdep['toolchain'] != dep['toolchain']:
_log.info(
"Minimally resolving dependency %s using toolchain %s with %s",
dep, toolchain, eb_file)
res = (newdep, eb_file)
break
if res is None:
_log.debug("Irresolvable minimal dependency found: %s", dep)
return res
def robot_find_minimal_easyconfig_for_dependency(dep):
"""
Find an easyconfig with minimal toolchain for a dependency
"""
newdep = copy.deepcopy(dep)
toolchain_hierarchy = get_toolchain_hierarchy(dep['toolchain'])
res = None
# reversed search: start with subtoolchains first, i.e. first (dummy or) compiler-only toolchain, etc.
for toolchain in toolchain_hierarchy:
newdep['toolchain'] = toolchain
eb_file = robot_find_easyconfig(newdep['name'], det_full_ec_version(newdep))
if eb_file is not None:
if newdep['toolchain'] != dep['toolchain']:
_log.info("Minimally resolving dependency %s using toolchain %s with %s", dep, toolchain, eb_file)
res = (newdep, eb_file)
break
if res is None:
_log.debug("Irresolvable minimal dependency found: %s", dep)
return res
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('unprocessed before resolving deps: %s' % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
tup = (maxloopcnt, unprocessed, irresolvable)
msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(unprocessed,
avail_modules,
retain_all_deps=retain_all_deps)
more_ecs, unprocessed, avail_modules = res
for ec in more_ecs:
if not ec['full_mod_name'] in [
x['full_mod_name'] for x in ordered_ecs
]:
ordered_ecs.append(ec)
# robot: look for existing dependencies, add them
if robot and unprocessed:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in unprocessed
]
additional = []
for entry in unprocessed:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" %
cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [
spec['ec'].full_mod_name for spec in processed_ecs
]
dep_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
if not dep_mod_name in mods:
tup = (path, dep_mod_name, mods)
_log.error(
"easyconfig file %s does not contain module %s (mods: %s)"
% tup)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [
dep for x in unprocessed for dep in x['dependencies']
]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [
EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" %
', '.join(irresolvable_mods_eb))
irresolvable_mods = [
ActiveMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.error('Irresolvable dependencies encountered: %s' %
', '.join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows:\n%s" %
ordered_ecs)
return ordered_ecs
def get_toolchain_hierarchy(parent_toolchain):
"""
Determine list of subtoolchains for specified parent toolchain.
Result starts with the most minimal subtoolchains first, ends with specified toolchain.
The dummy toolchain is considered the most minimal subtoolchain only if the add_dummy_to_minimal_toolchains
build option is enabled.
@param parent_toolchain: dictionary with name/version of parent toolchain
"""
# obtain list of all possible subtoolchains
_, all_tc_classes = search_toolchain('')
subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes)
current_tc_name, current_tc_version = parent_toolchain['name'], parent_toolchain['version']
subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None
# the parent toolchain is at the top of the hierarchy
toolchain_hierarchy = [parent_toolchain]
while subtoolchain_name:
# grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain
path = robot_find_easyconfig(current_tc_name, current_tc_version)
if path is None:
raise EasyBuildError("Could not find easyconfig for %(name)s toolchain version %(version)s",
current_tc_name, current_tc_version)
# parse the easyconfig
parsed_ec = process_easyconfig(path)[0]
# search the dependencies for the version of the subtoolchain
dep_tcs = [dep_toolchain['toolchain'] for dep_toolchain in parsed_ec['dependencies']
if dep_toolchain['toolchain']['name'] == subtoolchain_name]
unique_dep_tc_versions = set([dep_tc['version'] for dep_tc in dep_tcs])
if len(unique_dep_tc_versions) == 1:
subtoolchain_version = dep_tcs[0]['version']
elif len(unique_dep_tc_versions) == 0:
# only retain GCCcore as subtoolchain if version was found
if subtoolchain_name == GCCcore.NAME:
_log.info("No version found for %s; assuming legacy toolchain and skipping it as subtoolchain.",
subtoolchain_name)
subtoolchain_name = GCCcore.SUBTOOLCHAIN
subtoolchain_version = ''
# dummy toolchain: end of the line
elif subtoolchain_name == DUMMY_TOOLCHAIN_NAME:
subtoolchain_version = ''
else:
raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s",
subtoolchain_name, current_tc_name)
else:
raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s",
subtoolchain_name, current_tc_name, unique_dep_tc_versions)
if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option('add_dummy_to_minimal_toolchains'):
# we're done
break
# add to hierarchy and move to next
current_tc_name, current_tc_version = subtoolchain_name, subtoolchain_version
subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None
toolchain_hierarchy.insert(0, {'name': current_tc_name, 'version': current_tc_version})
_log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy)
return toolchain_hierarchy
def resolve_dependencies(easyconfigs,
modtool,
retain_all_deps=False,
raise_error_missing_ecs=True):
"""
Work through the list of easyconfigs to determine an optimal order
:param easyconfigs: list of easyconfigs
:param modtool: ModulesTool instance to use
:param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
:param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
avail_modules = modtool.available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if m not in being_installed]
_log.debug('easyconfigs before resolving deps: %s', easyconfigs)
totally_missing, missing_easyconfigs = [], []
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError(
"Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)",
maxloopcnt, easyconfigs, missing_easyconfigs)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(easyconfigs,
avail_modules,
modtool,
retain_all_deps=retain_all_deps)
resolved_ecs, easyconfigs, avail_modules = res
ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs]
for ec in resolved_ecs:
# only add easyconfig if it's not included yet (based on module name)
if not ec['full_mod_name'] in ordered_ec_mod_names:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [
d['full_mod_name'] for ec in easyconfigs
for d in ec['dependencies'] if d.get('external_module', False)
]
if missing_external_modules:
raise EasyBuildError(
"Missing modules for dependencies marked as external modules: %s",
', '.join(missing_external_modules))
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in easyconfigs
]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
full_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
# no easyconfig found + no module available => missing dependency
if not modtool.exist([full_mod_name])[0]:
if cand_dep not in totally_missing:
totally_missing.append(cand_dep)
# no easyconfig found for dependency, but module is available
# => add to list of missing easyconfigs
elif cand_dep not in missing_easyconfigs:
_log.debug(
"Irresolvable dependency found (no easyconfig file): %s",
cand_dep)
missing_easyconfigs.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
# add dummy entry for this dependency, so --dry-run for example can still report the dep
additional.append({
'dependencies': [],
'ec': None,
'full_mod_name': full_mod_name,
'spec': None,
})
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
verify_easyconfig_filename(path,
cand_dep,
parsed_ec=processed_ecs)
for ec in processed_ecs:
if ec not in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
missing_deps = [
dep for x in easyconfigs for dep in x['dependencies']
]
if missing_deps:
raise_error_missing_deps(
missing_deps,
extra_msg="enable dependency resolution via --robot?")
if totally_missing:
raise_error_missing_deps(
totally_missing,
extra_msg="no easyconfig file or existing module found")
if missing_easyconfigs:
if raise_error_missing_ecs:
raise_error_missing_deps(
missing_easyconfigs,
extra_msg="no easyconfig file found in robot search path")
else:
_log.warning("No easyconfig files found for: %s",
missing_easyconfigs)
_log.info("Dependency resolution complete, building as follows: %s",
ordered_ecs)
return ordered_ecs
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
"""
robot = build_option("robot_path")
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option("retain_all_deps") or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv("MODULEPATH"))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p["full_mod_name"] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug("unprocessed before resolving deps: %s" % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError(
"Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)",
maxloopcnt,
unprocessed,
irresolvable,
)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps)
more_ecs, unprocessed, avail_modules = res
for ec in more_ecs:
if not ec["full_mod_name"] in [x["full_mod_name"] for x in ordered_ecs]:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [
d["full_mod_name"] for ec in unprocessed for d in ec["dependencies"] if d.get("external_module", False)
]
if missing_external_modules:
raise EasyBuildError(
"Missing modules for one or more dependencies marked as external modules: %s", missing_external_modules
)
# robot: look for existing dependencies, add them
if robot and unprocessed:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [EasyBuildMNS().det_full_module_name(p["ec"]) for p in unprocessed]
additional = []
for entry in unprocessed:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry["dependencies"]
candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep["name"], det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" % cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry["dependencies"].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get("hidden", False)
processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [spec["ec"].full_mod_name for spec in processed_ecs]
dep_mod_name = ActiveMNS().det_full_module_name(cand_dep)
if not dep_mod_name in mods:
raise EasyBuildError(
"easyconfig file %s does not contain module %s (mods: %s)", path, dep_mod_name, mods
)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry["ec"])
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
_log.debug("Unprocessed dependencies: %s", unprocessed)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [dep for x in unprocessed for dep in x["dependencies"]]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ", ".join(irresolvable_mods_eb))
irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable]
raise EasyBuildError("Irresolvable dependencies encountered: %s", ", ".join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs)
return ordered_ecs
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
robot = build_option('robot_path')
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['module'] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('unprocessed before resolving deps: %s' % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
tup = (maxloopcnt, unprocessed, irresolvable)
msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules)
for ec in more_ecs:
if not ec['module'] in [x['module'] for x in ordered_ecs]:
ordered_ecs.append(ec)
# robot: look for existing dependencies, add them
if robot and unprocessed:
being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed]
additional = []
for i, entry in enumerate(unprocessed):
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed]
if len(candidates) > 0:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" % cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
processed_ecs = process_easyconfig(path, validate=not retain_all_deps)
# ensure that selected easyconfig provides required dependency
mods = [det_full_module_name(spec['ec']) for spec in processed_ecs]
dep_mod_name = det_full_module_name(cand_dep)
if not dep_mod_name in mods:
tup = (path, dep_mod_name, mods)
_log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = det_full_module_name(entry['ec'], eb_ns=True)
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [dep for x in unprocessed for dep in x['dependencies']]
break
if irresolvable:
irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable]
_log.error('Irresolvable dependencies encountered: %s' % irresolvable_mod_deps)
_log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs)
return ordered_ecs
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True):
"""
Work through the list of easyconfigs to determine an optimal order
:param easyconfigs: list of easyconfigs
:param modtool: ModulesTool instance to use
:param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
:param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
avail_modules = modtool.available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if m not in being_installed]
_log.debug('easyconfigs before resolving deps: %s', easyconfigs)
totally_missing, missing_easyconfigs = [], []
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError("Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)",
maxloopcnt, easyconfigs, missing_easyconfigs)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=retain_all_deps)
resolved_ecs, easyconfigs, avail_modules = res
ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs]
for ec in resolved_ecs:
# only add easyconfig if it's not included yet (based on module name)
if not ec['full_mod_name'] in ordered_ec_mod_names:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies']
if d.get('external_module', False)]
if missing_external_modules:
raise EasyBuildError("Missing modules for dependencies marked as external modules: %s",
', '.join(missing_external_modules))
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep))
if path is None:
full_mod_name = ActiveMNS().det_full_module_name(cand_dep)
# no easyconfig found + no module available => missing dependency
if not modtool.exist([full_mod_name])[0]:
if cand_dep not in totally_missing:
totally_missing.append(cand_dep)
# no easyconfig found for dependency, but module is available
# => add to list of missing easyconfigs
elif cand_dep not in missing_easyconfigs:
_log.debug("Irresolvable dependency found (no easyconfig file): %s", cand_dep)
missing_easyconfigs.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
# add dummy entry for this dependency, so --dry-run for example can still report the dep
additional.append({
'dependencies': [],
'ec': None,
'full_mod_name': full_mod_name,
'spec': None,
})
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs)
for ec in processed_ecs:
if ec not in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
missing_deps = [dep for x in easyconfigs for dep in x['dependencies']]
if missing_deps:
raise_error_missing_deps(missing_deps, extra_msg="enable dependency resolution via --robot?")
if totally_missing:
raise_error_missing_deps(totally_missing, extra_msg="no easyconfig file or existing module found")
if missing_easyconfigs:
if raise_error_missing_ecs:
raise_error_missing_deps(missing_easyconfigs, extra_msg="no easyconfig file found in robot search path")
else:
_log.warning("No easyconfig files found for: %s", missing_easyconfigs)
_log.info("Dependency resolution complete, building as follows: %s", ordered_ecs)
return ordered_ecs
def get_toolchain_hierarchy(parent_toolchain):
"""
Determine list of subtoolchains for specified parent toolchain.
Result starts with the most minimal subtoolchains first, ends with specified toolchain.
The dummy toolchain is considered the most minimal subtoolchain only if the add_dummy_to_minimal_toolchains
build option is enabled.
@param parent_toolchain: dictionary with name/version of parent toolchain
"""
# obtain list of all possible subtoolchains
_, all_tc_classes = search_toolchain("")
subtoolchains = dict((tc_class.NAME, getattr(tc_class, "SUBTOOLCHAIN", None)) for tc_class in all_tc_classes)
current_tc_name, current_tc_version = parent_toolchain["name"], parent_toolchain["version"]
subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None
# the parent toolchain is at the top of the hierarchy
toolchain_hierarchy = [parent_toolchain]
while subtoolchain_name:
# grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain
path = robot_find_easyconfig(current_tc_name, current_tc_version)
if path is None:
raise EasyBuildError(
"Could not find easyconfig for %(name)s toolchain version %(version)s",
current_tc_name,
current_tc_version,
)
# parse the easyconfig
parsed_ec = process_easyconfig(path)[0]
# search the dependencies for the version of the subtoolchain
dep_tcs = [
dep_toolchain["toolchain"]
for dep_toolchain in parsed_ec["dependencies"]
if dep_toolchain["toolchain"]["name"] == subtoolchain_name
]
unique_dep_tc_versions = set([dep_tc["version"] for dep_tc in dep_tcs])
if len(unique_dep_tc_versions) == 1:
subtoolchain_version = dep_tcs[0]["version"]
elif len(unique_dep_tc_versions) == 0:
if subtoolchain_name == DUMMY_TOOLCHAIN_NAME:
subtoolchain_version = ""
else:
raise EasyBuildError(
"No version found for subtoolchain %s in dependencies of %s", subtoolchain_name, current_tc_name
)
else:
raise EasyBuildError(
"Multiple versions of %s found in dependencies of toolchain %s: %s",
subtoolchain_name,
current_tc_name,
unique_dep_tc_versions,
)
if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option("add_dummy_to_minimal_toolchains"):
# we're done
break
# add to hierarchy and move to next
current_tc_name, current_tc_version = subtoolchain_name, subtoolchain_version
subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None
toolchain_hierarchy.insert(0, {"name": current_tc_name, "version": current_tc_version})
_log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy)
return toolchain_hierarchy
def get_toolchain_hierarchy(parent_toolchain):
"""
Determine list of subtoolchains for specified parent toolchain.
Result starts with the most minimal subtoolchains first, ends with specified toolchain.
The dummy toolchain is considered the most minimal subtoolchain only if the add_dummy_to_minimal_toolchains
build option is enabled.
@param parent_toolchain: dictionary with name/version of parent toolchain
"""
# obtain list of all possible subtoolchains
_, all_tc_classes = search_toolchain('')
subtoolchains = dict(
(tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None))
for tc_class in all_tc_classes)
current_tc_name, current_tc_version = parent_toolchain[
'name'], parent_toolchain['version']
subtoolchain_name, subtoolchain_version = subtoolchains[
current_tc_name], None
# the parent toolchain is at the top of the hierarchy
toolchain_hierarchy = [parent_toolchain]
while subtoolchain_name:
# grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain
path = robot_find_easyconfig(current_tc_name, current_tc_version)
if path is None:
raise EasyBuildError(
"Could not find easyconfig for %(name)s toolchain version %(version)s",
current_tc_name, current_tc_version)
# parse the easyconfig
parsed_ec = process_easyconfig(path)[0]
# search the dependencies for the version of the subtoolchain
dep_tcs = [
dep_toolchain['toolchain']
for dep_toolchain in parsed_ec['dependencies']
if dep_toolchain['toolchain']['name'] == subtoolchain_name
]
unique_dep_tc_versions = set([dep_tc['version'] for dep_tc in dep_tcs])
if len(unique_dep_tc_versions) == 1:
subtoolchain_version = dep_tcs[0]['version']
elif len(unique_dep_tc_versions) == 0:
# only retain GCCcore as subtoolchain if version was found
if subtoolchain_name == GCCcore.NAME:
_log.info(
"No version found for %s; assuming legacy toolchain and skipping it as subtoolchain.",
subtoolchain_name)
subtoolchain_name = GCCcore.SUBTOOLCHAIN
subtoolchain_version = ''
# dummy toolchain: end of the line
elif subtoolchain_name == DUMMY_TOOLCHAIN_NAME:
subtoolchain_version = ''
else:
raise EasyBuildError(
"No version found for subtoolchain %s in dependencies of %s",
subtoolchain_name, current_tc_name)
else:
raise EasyBuildError(
"Multiple versions of %s found in dependencies of toolchain %s: %s",
subtoolchain_name, current_tc_name, unique_dep_tc_versions)
if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option(
'add_dummy_to_minimal_toolchains'):
# we're done
break
# add to hierarchy and move to next
current_tc_name, current_tc_version = subtoolchain_name, subtoolchain_version
subtoolchain_name, subtoolchain_version = subtoolchains[
current_tc_name], None
toolchain_hierarchy.insert(0, {
'name': current_tc_name,
'version': current_tc_version
})
_log.info("Found toolchain hierarchy for toolchain %s: %s",
parent_toolchain, toolchain_hierarchy)
return toolchain_hierarchy
