def test_find_minimally_resolved_modules(self):
"""Test find_minimally_resolved_modules function."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
init_config(
build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_txt = '\n'.join([
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),", # only available with goolf/1.4.10
"]",
])
write_file(barec, barec_txt)
bar = process_easyconfig(barec)[0]
ecs = [bar]
mods = [
'gompi/1.4.10',
'goolf/1.4.10',
# include modules for dependencies, with subtoolchains rather than full toolchain (except for SQLite)
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
]
ordered_ecs, new_easyconfigs, new_avail_modules = find_minimally_resolved_modules(
ecs, mods, [])
# all dependencies are resolved for easyconfigs included in ordered_ecs
self.assertEqual(len(ordered_ecs), 1)
self.assertEqual(ordered_ecs[0]['dependencies'], [])
# module is added to list of available modules
self.assertTrue(bar['ec'].full_mod_name in new_avail_modules)
# nothing left
self.assertEqual(new_easyconfigs, [])
def test_find_minimally_resolved_modules(self):
"""Test find_minimally_resolved_modules function."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_txt = '\n'.join([
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),", # only available with goolf/1.4.10
"]",
])
write_file(barec, barec_txt)
bar = process_easyconfig(barec)[0]
ecs = [bar]
mods = [
'gompi/1.4.10',
'goolf/1.4.10',
# include modules for dependencies, with subtoolchains rather than full toolchain (except for SQLite)
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
]
ordered_ecs, new_easyconfigs, new_avail_modules = find_minimally_resolved_modules(ecs, mods, [])
# all dependencies are resolved for easyconfigs included in ordered_ecs
self.assertEqual(len(ordered_ecs), 1)
self.assertEqual(ordered_ecs[0]['dependencies'], [])
# module is added to list of available modules
self.assertTrue(bar['ec'].full_mod_name in new_avail_modules)
# nothing left
self.assertEqual(new_easyconfigs, [])
def resolve_dependencies(easyconfigs,
retain_all_deps=False,
minimal_toolchains=False,
use_existing_modules=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param easyconfigs: list of easyconfigs
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
@param minimal_toolchains: boolean for whether to try to resolve dependencies with minimum possible toolchain
@param use_existing_modules: boolean for whether to prioritise the reuse of existing modules (works in
combination with minimal_toolchains)
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
existing_modules = modules_tool().available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = existing_modules[:]
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('easyconfigs before resolving deps: %s' % easyconfigs)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError(
"Maximum loop cnt %s reached, so quitting (easyconfigs: %s, irresolvable: %s)",
maxloopcnt, easyconfigs, irresolvable)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
if minimal_toolchains:
res = find_minimally_resolved_modules(
easyconfigs,
avail_modules,
existing_modules,
retain_all_deps=retain_all_deps,
use_existing_modules=use_existing_modules)
else:
res = find_resolved_modules(easyconfigs,
avail_modules,
retain_all_deps=retain_all_deps)
more_ecs, easyconfigs, avail_modules = res
for ec in more_ecs:
if not ec['full_mod_name'] in [
x['full_mod_name'] for x in ordered_ecs
]:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [
d['full_mod_name'] for ec in easyconfigs
for d in ec['dependencies'] if d.get('external_module', False)
]
if missing_external_modules:
raise EasyBuildError(
"Missing modules for one or more dependencies marked as external modules: %s",
missing_external_modules)
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in easyconfigs
]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" %
cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [
spec['ec'].full_mod_name for spec in processed_ecs
]
dep_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
if not dep_mod_name in mods:
raise EasyBuildError(
"easyconfig file %s does not contain module %s (mods: %s)",
path, dep_mod_name, mods)
for ec in processed_ecs:
if not ec in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [
dep for x in easyconfigs for dep in x['dependencies']
]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [
EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" %
', '.join(irresolvable_mods_eb))
irresolvable_mods = [
ActiveMNS().det_full_module_name(dep) for dep in irresolvable
]
raise EasyBuildError("Irresolvable dependencies encountered: %s",
', '.join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows: %s" %
ordered_ecs)
return ordered_ecs
def resolve_dependencies(easyconfigs, retain_all_deps=False, minimal_toolchains=False, use_existing_modules=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param easyconfigs: list of easyconfigs
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
@param minimal_toolchains: boolean for whether to try to resolve dependencies with minimum possible toolchain
@param use_existing_modules: boolean for whether to prioritise the reuse of existing modules (works in
combination with minimal_toolchains)
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
existing_modules = modules_tool().available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = existing_modules[:]
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('easyconfigs before resolving deps: %s' % easyconfigs)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError("Maximum loop cnt %s reached, so quitting (easyconfigs: %s, irresolvable: %s)",
maxloopcnt, easyconfigs, irresolvable)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
if minimal_toolchains:
res = find_minimally_resolved_modules(easyconfigs, avail_modules, existing_modules,
retain_all_deps=retain_all_deps,
use_existing_modules=use_existing_modules)
else:
res = find_resolved_modules(easyconfigs, avail_modules, retain_all_deps=retain_all_deps)
more_ecs, easyconfigs, avail_modules = res
for ec in more_ecs:
if not ec['full_mod_name'] in [x['full_mod_name'] for x in ordered_ecs]:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies']
if d.get('external_module', False)]
if missing_external_modules:
raise EasyBuildError("Missing modules for one or more dependencies marked as external modules: %s",
missing_external_modules)
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" % cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [spec['ec'].full_mod_name for spec in processed_ecs]
dep_mod_name = ActiveMNS().det_full_module_name(cand_dep)
if not dep_mod_name in mods:
raise EasyBuildError("easyconfig file %s does not contain module %s (mods: %s)",
path, dep_mod_name, mods)
for ec in processed_ecs:
if not ec in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [dep for x in easyconfigs for dep in x['dependencies']]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb))
irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable]
raise EasyBuildError("Irresolvable dependencies encountered: %s", ', '.join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs)
return ordered_ecs