def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('unprocessed before resolving deps: %s' % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
tup = (maxloopcnt, unprocessed, irresolvable)
msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(unprocessed,
avail_modules,
retain_all_deps=retain_all_deps)
more_ecs, unprocessed, avail_modules = res
for ec in more_ecs:
if not ec['full_mod_name'] in [
x['full_mod_name'] for x in ordered_ecs
]:
ordered_ecs.append(ec)
# robot: look for existing dependencies, add them
if robot and unprocessed:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in unprocessed
]
additional = []
for entry in unprocessed:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" %
cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [
spec['ec'].full_mod_name for spec in processed_ecs
]
dep_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
if not dep_mod_name in mods:
tup = (path, dep_mod_name, mods)
_log.error(
"easyconfig file %s does not contain module %s (mods: %s)"
% tup)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [
dep for x in unprocessed for dep in x['dependencies']
]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [
EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" %
', '.join(irresolvable_mods_eb))
irresolvable_mods = [
ActiveMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.error('Irresolvable dependencies encountered: %s' %
', '.join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows:\n%s" %
ordered_ecs)
return ordered_ecs
def resolve_dependencies(easyconfigs,
modtool,
retain_all_deps=False,
raise_error_missing_ecs=True):
"""
Work through the list of easyconfigs to determine an optimal order
:param easyconfigs: list of easyconfigs
:param modtool: ModulesTool instance to use
:param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
:param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
avail_modules = modtool.available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if m not in being_installed]
_log.debug('easyconfigs before resolving deps: %s', easyconfigs)
totally_missing, missing_easyconfigs = [], []
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError(
"Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)",
maxloopcnt, easyconfigs, missing_easyconfigs)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(easyconfigs,
avail_modules,
modtool,
retain_all_deps=retain_all_deps)
resolved_ecs, easyconfigs, avail_modules = res
ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs]
for ec in resolved_ecs:
# only add easyconfig if it's not included yet (based on module name)
if not ec['full_mod_name'] in ordered_ec_mod_names:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [
d['full_mod_name'] for ec in easyconfigs
for d in ec['dependencies'] if d.get('external_module', False)
]
if missing_external_modules:
raise EasyBuildError(
"Missing modules for dependencies marked as external modules: %s",
', '.join(missing_external_modules))
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in easyconfigs
]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
full_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
# no easyconfig found + no module available => missing dependency
if not modtool.exist([full_mod_name])[0]:
if cand_dep not in totally_missing:
totally_missing.append(cand_dep)
# no easyconfig found for dependency, but module is available
# => add to list of missing easyconfigs
elif cand_dep not in missing_easyconfigs:
_log.debug(
"Irresolvable dependency found (no easyconfig file): %s",
cand_dep)
missing_easyconfigs.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
# add dummy entry for this dependency, so --dry-run for example can still report the dep
additional.append({
'dependencies': [],
'ec': None,
'full_mod_name': full_mod_name,
'spec': None,
})
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
verify_easyconfig_filename(path,
cand_dep,
parsed_ec=processed_ecs)
for ec in processed_ecs:
if ec not in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
missing_deps = [
dep for x in easyconfigs for dep in x['dependencies']
]
if missing_deps:
raise_error_missing_deps(
missing_deps,
extra_msg="enable dependency resolution via --robot?")
if totally_missing:
raise_error_missing_deps(
totally_missing,
extra_msg="no easyconfig file or existing module found")
if missing_easyconfigs:
if raise_error_missing_ecs:
raise_error_missing_deps(
missing_easyconfigs,
extra_msg="no easyconfig file found in robot search path")
else:
_log.warning("No easyconfig files found for: %s",
missing_easyconfigs)
_log.info("Dependency resolution complete, building as follows: %s",
ordered_ecs)
return ordered_ecs
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
"""
robot = build_option("robot_path")
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option("retain_all_deps") or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv("MODULEPATH"))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p["full_mod_name"] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug("unprocessed before resolving deps: %s" % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError(
"Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)",
maxloopcnt,
unprocessed,
irresolvable,
)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps)
more_ecs, unprocessed, avail_modules = res
for ec in more_ecs:
if not ec["full_mod_name"] in [x["full_mod_name"] for x in ordered_ecs]:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [
d["full_mod_name"] for ec in unprocessed for d in ec["dependencies"] if d.get("external_module", False)
]
if missing_external_modules:
raise EasyBuildError(
"Missing modules for one or more dependencies marked as external modules: %s", missing_external_modules
)
# robot: look for existing dependencies, add them
if robot and unprocessed:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [EasyBuildMNS().det_full_module_name(p["ec"]) for p in unprocessed]
additional = []
for entry in unprocessed:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry["dependencies"]
candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep["name"], det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" % cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry["dependencies"].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get("hidden", False)
processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [spec["ec"].full_mod_name for spec in processed_ecs]
dep_mod_name = ActiveMNS().det_full_module_name(cand_dep)
if not dep_mod_name in mods:
raise EasyBuildError(
"easyconfig file %s does not contain module %s (mods: %s)", path, dep_mod_name, mods
)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry["ec"])
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
_log.debug("Unprocessed dependencies: %s", unprocessed)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [dep for x in unprocessed for dep in x["dependencies"]]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ", ".join(irresolvable_mods_eb))
irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable]
raise EasyBuildError("Irresolvable dependencies encountered: %s", ", ".join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs)
return ordered_ecs
def test_find_resolved_modules(self):
"""Test find_resolved_modules function."""
nodeps = {
'name': 'nodeps',
'version': '1.2.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dependencies': [],
'full_mod_name': 'nodeps/1.2.3',
'spec': 'nodeps-1.2.3.eb',
'hidden': False,
}
dep1 = {
'name': 'foo',
'version': '2.3.4',
'toolchain': {'name': 'GCC', 'version': '4.7.2'},
'versionsuffix': '',
'hidden': False,
}
dep2 = {
'name': 'bar',
'version': '3.4.5',
'toolchain': {'name': 'gompi', 'version': '1.4.10'},
'versionsuffix': '-test',
'hidden': False,
}
onedep = {
'name': 'onedep',
'version': '3.14',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
'dependencies': [dep1],
'full_mod_name': 'onedep/3.14-goolf-1.4.10',
'spec': 'onedep-3.14-goolf-1.4.10.eb',
}
threedeps = {
'name': 'threedeps',
'version': '9.8.7',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
'dependencies': [dep1, dep2, nodeps],
'full_mod_name': 'threedeps/9.8.7-goolf-1.4.10',
'spec': 'threedeps-9.8.7-goolf-1.4.10.eb',
}
ecs = [
nodeps,
onedep,
threedeps,
]
mods = ['foo/2.3.4-GCC-4.7.2', 'bar/3.4.5-gompi-1.4.10', 'bar/3.4.5-GCC-4.7.2']
ordered_ecs, new_easyconfigs, new_avail_modules = find_resolved_modules(ecs, mods, self.modtool)
# all dependencies are resolved for easyconfigs included in ordered_ecs
self.assertFalse(any([ec['dependencies'] for ec in ordered_ecs]))
# nodeps/ondep easyconfigs have all dependencies resolved
self.assertEqual(len(ordered_ecs), 2)
self.assertEqual(nodeps, ordered_ecs[0])
onedep['dependencies'] = []
self.assertEqual(onedep, ordered_ecs[1])
# threedeps has available dependencies (foo, nodeps) filtered out
self.assertEqual(len(new_easyconfigs), 1)
self.assertEqual(new_easyconfigs[0]['full_mod_name'], 'threedeps/9.8.7-goolf-1.4.10')
self.assertEqual(len(new_easyconfigs[0]['dependencies']), 1)
self.assertEqual(new_easyconfigs[0]['dependencies'][0]['name'], 'bar')
self.assertTrue(new_avail_modules, mods + ['nodeps/1.2.3', 'onedep/3.14-goolf-1.4.10'])
# also check results with retaining all dependencies enabled
ordered_ecs, new_easyconfigs, new_avail_modules = find_resolved_modules(ecs, [], self.modtool,
retain_all_deps=True)
self.assertEqual(len(ordered_ecs), 2)
self.assertEqual([ec['full_mod_name'] for ec in ordered_ecs], ['nodeps/1.2.3', 'onedep/3.14-goolf-1.4.10'])
self.assertEqual(len(new_easyconfigs), 1)
self.assertEqual(len(new_easyconfigs[0]['dependencies']), 2)
self.assertEqual([dep['name'] for dep in new_easyconfigs[0]['dependencies']], ['foo', 'bar'])
self.assertTrue(new_avail_modules, ['nodeps/1.2.3', 'onedep/3.14-goolf-1.4.10'])
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True):
"""
Work through the list of easyconfigs to determine an optimal order
:param easyconfigs: list of easyconfigs
:param modtool: ModulesTool instance to use
:param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
:param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
avail_modules = modtool.available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if m not in being_installed]
_log.debug('easyconfigs before resolving deps: %s', easyconfigs)
totally_missing, missing_easyconfigs = [], []
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError("Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)",
maxloopcnt, easyconfigs, missing_easyconfigs)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=retain_all_deps)
resolved_ecs, easyconfigs, avail_modules = res
ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs]
for ec in resolved_ecs:
# only add easyconfig if it's not included yet (based on module name)
if not ec['full_mod_name'] in ordered_ec_mod_names:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies']
if d.get('external_module', False)]
if missing_external_modules:
raise EasyBuildError("Missing modules for dependencies marked as external modules: %s",
', '.join(missing_external_modules))
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep))
if path is None:
full_mod_name = ActiveMNS().det_full_module_name(cand_dep)
# no easyconfig found + no module available => missing dependency
if not modtool.exist([full_mod_name])[0]:
if cand_dep not in totally_missing:
totally_missing.append(cand_dep)
# no easyconfig found for dependency, but module is available
# => add to list of missing easyconfigs
elif cand_dep not in missing_easyconfigs:
_log.debug("Irresolvable dependency found (no easyconfig file): %s", cand_dep)
missing_easyconfigs.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
# add dummy entry for this dependency, so --dry-run for example can still report the dep
additional.append({
'dependencies': [],
'ec': None,
'full_mod_name': full_mod_name,
'spec': None,
})
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs)
for ec in processed_ecs:
if ec not in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
missing_deps = [dep for x in easyconfigs for dep in x['dependencies']]
if missing_deps:
raise_error_missing_deps(missing_deps, extra_msg="enable dependency resolution via --robot?")
if totally_missing:
raise_error_missing_deps(totally_missing, extra_msg="no easyconfig file or existing module found")
if missing_easyconfigs:
if raise_error_missing_ecs:
raise_error_missing_deps(missing_easyconfigs, extra_msg="no easyconfig file found in robot search path")
else:
_log.warning("No easyconfig files found for: %s", missing_easyconfigs)
_log.info("Dependency resolution complete, building as follows: %s", ordered_ecs)
return ordered_ecs