def tweak(easyconfigs, build_specs, targetdir=None):
"""Tweak list of easyconfigs according to provided build specifications."""
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(
['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
_log.error(
"Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s"
% toolchains)
if 'name' in build_specs or 'version' in build_specs:
# no recursion if software name/version build specification are included
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug(
"Software name/version found, so not applying build specifications recursively: %s"
% build_specs)
else:
# build specifications should be applied to the whole dependency graph
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
_log.debug(
"Applying build specifications recursively (no software name/version found): %s"
% build_specs)
orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)
# determine toolchain based on last easyconfigs
toolchain = orig_ecs[-1]['ec']['toolchain']
_log.debug("Filtering using toolchain %s" % toolchain)
# filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies
if toolchain['name'] != DUMMY_TOOLCHAIN_NAME:
while orig_ecs[0]['ec']['toolchain'] != toolchain:
orig_ecs = orig_ecs[1:]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
new_ec_file = tweak_one(orig_ec['spec'],
None,
build_specs,
targetdir=targetdir)
# only return tweaked easyconfigs for easyconfigs which were listed originally
# easyconfig files for dependencies are also generated but not included, and will be resolved via --robot
if orig_ec['spec'] in listed_ec_paths:
new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs)
tweaked_easyconfigs.extend(new_ecs)
return tweaked_easyconfigs
def tweak(easyconfigs, build_specs):
"""Tweak list of easyconfigs according to provided build specifications."""
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(
['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
_log.error(
"Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s"
% toolchains)
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)
# determine toolchain based on last easyconfigs
toolchain = orig_ecs[-1]['ec']['toolchain']
_log.debug("Filtering using toolchain %s" % toolchain)
# filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies
if toolchain['name'] != DUMMY_TOOLCHAIN_NAME:
while orig_ecs[0]['ec']['toolchain'] != toolchain:
orig_ecs = orig_ecs[1:]
# generate tweaked easyconfigs, and continue with those instead
easyconfigs = []
for orig_ec in orig_ecs:
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs)
new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs)
easyconfigs.extend(new_ecs)
return easyconfigs
def tweak(easyconfigs, build_specs):
"""Tweak list of easyconfigs according to provided build specifications."""
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
_log.error("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains)
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)
# determine toolchain based on last easyconfigs
toolchain = orig_ecs[-1]['ec']['toolchain']
_log.debug("Filtering using toolchain %s" % toolchain)
# filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies
if toolchain['name'] != DUMMY_TOOLCHAIN_NAME:
while orig_ecs[0]['ec']['toolchain'] != toolchain:
orig_ecs = orig_ecs[1:]
# generate tweaked easyconfigs, and continue with those instead
easyconfigs = []
for orig_ec in orig_ecs:
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs)
new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs)
easyconfigs.extend(new_ecs)
return easyconfigs
def tweak(easyconfigs, build_specs, targetdir=None):
"""Tweak list of easyconfigs according to provided build specifications."""
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
_log.error("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains)
if 'name' in build_specs or 'version' in build_specs:
# no recursion if software name/version build specification are included
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs)
else:
# build specifications should be applied to the whole dependency graph
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
_log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs)
orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)
# determine toolchain based on last easyconfigs
toolchain = orig_ecs[-1]['ec']['toolchain']
_log.debug("Filtering using toolchain %s" % toolchain)
# filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies
if toolchain['name'] != DUMMY_TOOLCHAIN_NAME:
while orig_ecs[0]['ec']['toolchain'] != toolchain:
orig_ecs = orig_ecs[1:]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=targetdir)
# only return tweaked easyconfigs for easyconfigs which were listed originally
# easyconfig files for dependencies are also generated but not included, and will be resolved via --robot
if orig_ec['spec'] in listed_ec_paths:
new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs)
tweaked_easyconfigs.extend(new_ecs)
return tweaked_easyconfigs
def test_build_easyconfigs_in_parallel(self):
"""Basic test for build_easyconfigs_in_parallel function."""
easyconfig_file = os.path.join(os.path.dirname(__file__),
'easyconfigs',
'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(easyconfig_file, validate=False)
ordered_ecs = resolve_dependencies(easyconfigs)
build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs)
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not self.parsed_easyconfigs:
for spec in specs:
self.parsed_easyconfigs.extend(process_easyconfig(spec))
self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs)
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not self.parsed_easyconfigs:
for spec in specs:
self.parsed_easyconfigs.extend(process_easyconfig(spec))
self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs)
if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not options.force:
easyconfigs = skip_available(easyconfigs, testing=testing)
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs)
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s) and exit
if options.job:
curdir = os.getcwd()
# the options to ignore (help options can't reach here)
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs)
easyconfigs = skip_available(easyconfigs)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, build_specs=build_specs)
cmd = "eb %(spec)s --regtest --sequential -ld"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd}
jobs = build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
print "List of submitted jobs:"
for job in jobs:
print "%s: %s" % (job.name, job.jobid)
print "(%d jobs submitted)" % len(jobs)
# determine leaf nodes in dependency graph, and report them
all_deps = set()
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs',
'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug(
"Skipping easyconfigs from %s that already have a module available..."
% easyconfigs)
easyconfigs = skip_available(easyconfigs)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, build_specs=build_specs)
cmd = "eb %(spec)s --regtest --sequential -ld"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {
'cmd': cmd
}
jobs = build_easyconfigs_in_parallel(command,
resolved,
output_dir=output_dir)
print "List of submitted jobs:"
for job in jobs:
print "%s: %s" % (job.name, job.jobid)
if any([
options.dry_run, options.dry_run_short, options.regtest,
options.search, options.search_short
]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not options.force:
easyconfigs = skip_available(easyconfigs, testing=testing)
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs,
build_specs=build_specs)
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s) and exit
if options.job:
curdir = os.getcwd()
# the options to ignore (help options can't reach here)
def test_resolve_dependencies(self):
""" Test with some basic testcases (also check if he can find dependencies inside the given directory """
easyconfig = {
'spec': '_',
'module': 'name/version',
'dependencies': []
}
build_options = {
'ignore_osdeps': True,
'robot_path': None,
'validate': False,
}
res = resolve_dependencies([deepcopy(easyconfig)], build_options=build_options)
self.assertEqual([easyconfig], res)
easyconfig_dep = {
'ec': {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
},
'spec': '_',
'module': 'foo/1.2.3',
'dependencies': [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
}],
'parsed': True,
}
build_options.update({'robot_path': self.base_easyconfig_dir})
res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options)
# dependency should be found, order should be correct
self.assertEqual(len(res), 2)
self.assertEqual('gzip/1.4', res[0]['module'])
self.assertEqual('foo/1.2.3', res[-1]['module'])
# here we have include a Dependency in the easyconfig list
easyconfig['module'] = 'gzip/1.4'
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
build_options.update({'robot_path': None})
res = resolve_dependencies(ecs, build_options=build_options)
# all dependencies should be resolved
self.assertEqual(0, sum(len(ec['dependencies']) for ec in res))
# this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled
ecs = [deepcopy(easyconfig_dep)]
msg = "Irresolvable dependencies encountered"
self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs, build_options=build_options)
# test if dependencies of an automatically found file are also loaded
easyconfig_dep['dependencies'] = [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'GCC', 'version': '4.6.3'},
'dummy': True,
}]
ecs = [deepcopy(easyconfig_dep)]
build_options.update({'robot_path': self.base_easyconfig_dir})
res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options)
# GCC should be first (required by gzip dependency)
self.assertEqual('GCC/4.6.3', res[0]['module'])
self.assertEqual('foo/1.2.3', res[-1]['module'])
# make sure that only missing stuff is built, and that available modules are not rebuilt
# monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'FFTW/3.3.3-gompi-1.4.10',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies(ecs, build_options=build_options)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 2)
# goolf should be first, the software itself second
self.assertEqual('goolf/1.4.10', res[0]['module'])
self.assertEqual('foo/1.2.3', res[1]['module'])
# force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt
build_options.update({'force': True})
easyconfig['module'] = 'this/is/already/there'
MockModule.avail_modules.append('this/is/already/there')
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
res = resolve_dependencies(ecs, build_options=build_options)
# there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available)
self.assertEqual(len(res), 3)
# goolf should be first, the software itself second
self.assertEqual('this/is/already/there', res[0]['module'])
self.assertEqual('goolf/1.4.10', res[1]['module'])
self.assertEqual('foo/1.2.3', res[2]['module'])
# build that are listed but already have a module available are not retained without force
build_options.update({'force': False})
newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled
res = resolve_dependencies(newecs, build_options=build_options)
self.assertEqual(len(res), 2)
self.assertEqual('goolf/1.4.10', res[0]['module'])
self.assertEqual('foo/1.2.3', res[1]['module'])
# with retain_all_deps enabled, all dependencies ae retained
build_options.update({'retain_all_deps': True})
ecs = [deepcopy(easyconfig_dep)]
newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled
res = resolve_dependencies(newecs, build_options=build_options)
self.assertEqual(len(res), 9)
self.assertEqual('GCC/4.7.2', res[0]['module'])
self.assertEqual('goolf/1.4.10', res[-2]['module'])
self.assertEqual('foo/1.2.3', res[-1]['module'])
build_options.update({'retain_all_deps': False})
# provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'FFTW/3.3.3-gompi-1.4.10',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 4)
# goolf should be first, the software itself second
self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['module'])
self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['module'])
self.assertEqual('goolf/1.4.10', res[2]['module'])
self.assertEqual('foo/1.2.3', res[3]['module'])
def test_resolve_dependencies(self):
""" Test with some basic testcases (also check if he can find dependencies inside the given directory """
easyconfig = {
'spec': '_',
'module': 'name/version',
'dependencies': []
}
build_options = {
'allow_modules_tool_mismatch': True,
'robot_path': None,
'validate': False,
}
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig)])
self.assertEqual([easyconfig], res)
easyconfig_dep = {
'ec': {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
},
'spec':
'_',
'module':
'foo/1.2.3',
'dependencies': [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'dummy': True,
}],
'parsed':
True,
}
build_options.update({'robot_path': self.base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# dependency should be found, order should be correct
self.assertEqual(len(res), 2)
self.assertEqual('gzip/1.4', res[0]['module'])
self.assertEqual('foo/1.2.3', res[-1]['module'])
# here we have include a Dependency in the easyconfig list
easyconfig['module'] = 'gzip/1.4'
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
build_options.update({'robot_path': None})
init_config(build_options=build_options)
res = resolve_dependencies(ecs)
# all dependencies should be resolved
self.assertEqual(0, sum(len(ec['dependencies']) for ec in res))
# this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled
ecs = [deepcopy(easyconfig_dep)]
msg = "Irresolvable dependencies encountered"
self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs)
# test if dependencies of an automatically found file are also loaded
easyconfig_dep['dependencies'] = [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {
'name': 'GCC',
'version': '4.6.3'
},
'dummy': True,
}]
ecs = [deepcopy(easyconfig_dep)]
build_options.update({'robot_path': self.base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# GCC should be first (required by gzip dependency)
self.assertEqual('GCC/4.6.3', res[0]['module'])
self.assertEqual('foo/1.2.3', res[-1]['module'])
# make sure that only missing stuff is built, and that available modules are not rebuilt
# monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'FFTW/3.3.3-gompi-1.4.10',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'dummy': True,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies(ecs)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 2)
# goolf should be first, the software itself second
self.assertEqual('goolf/1.4.10', res[0]['module'])
self.assertEqual('foo/1.2.3', res[1]['module'])
# force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt
build_options.update({'force': True})
init_config(build_options=build_options)
easyconfig['module'] = 'this/is/already/there'
MockModule.avail_modules.append('this/is/already/there')
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
res = resolve_dependencies(ecs)
# there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available)
self.assertEqual(len(res), 3)
# goolf should be first, the software itself second
self.assertEqual('this/is/already/there', res[0]['module'])
self.assertEqual('goolf/1.4.10', res[1]['module'])
self.assertEqual('foo/1.2.3', res[2]['module'])
# build that are listed but already have a module available are not retained without force
build_options.update({'force': False})
init_config(build_options=build_options)
newecs = skip_available(
ecs,
testing=True) # skip available builds since force is not enabled
res = resolve_dependencies(newecs)
self.assertEqual(len(res), 2)
self.assertEqual('goolf/1.4.10', res[0]['module'])
self.assertEqual('foo/1.2.3', res[1]['module'])
# with retain_all_deps enabled, all dependencies ae retained
build_options.update({'retain_all_deps': True})
init_config(build_options=build_options)
ecs = [deepcopy(easyconfig_dep)]
newecs = skip_available(
ecs,
testing=True) # skip available builds since force is not enabled
res = resolve_dependencies(newecs)
self.assertEqual(len(res), 9)
self.assertEqual('GCC/4.7.2', res[0]['module'])
self.assertEqual('goolf/1.4.10', res[-2]['module'])
self.assertEqual('foo/1.2.3', res[-1]['module'])
build_options.update({'retain_all_deps': False})
init_config(build_options=build_options)
# provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'FFTW/3.3.3-gompi-1.4.10',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'dummy': True,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 4)
# goolf should be first, the software itself second
self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
res[0]['module'])
self.assertEqual(
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
res[1]['module'])
self.assertEqual('goolf/1.4.10', res[2]['module'])
self.assertEqual('foo/1.2.3', res[3]['module'])
def test_build_easyconfigs_in_parallel(self):
"""Basic test for build_easyconfigs_in_parallel function."""
easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(easyconfig_file, validate=False)
ordered_ecs = resolve_dependencies(easyconfigs)
build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs)
def test_resolve_dependencies(self):
""" Test with some basic testcases (also check if he can find dependencies inside the given directory """
easyconfig = {"spec": "_", "module": "name/version", "dependencies": []}
build_options = {"ignore_osdeps": True, "robot_path": None, "validate": False}
res = resolve_dependencies([deepcopy(easyconfig)], build_options=build_options)
self.assertEqual([easyconfig], res)
easyconfig_dep = {
"ec": {
"name": "foo",
"version": "1.2.3",
"versionsuffix": "",
"toolchain": {"name": "dummy", "version": "dummy"},
},
"spec": "_",
"module": "foo/1.2.3",
"dependencies": [
{
"name": "gzip",
"version": "1.4",
"versionsuffix": "",
"toolchain": {"name": "dummy", "version": "dummy"},
"dummy": True,
}
],
"parsed": True,
}
build_options.update({"robot_path": self.base_easyconfig_dir})
res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options)
# dependency should be found, order should be correct
self.assertEqual(len(res), 2)
self.assertEqual("gzip/1.4", res[0]["module"])
self.assertEqual("foo/1.2.3", res[-1]["module"])
# here we have include a Dependency in the easyconfig list
easyconfig["module"] = "gzip/1.4"
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
build_options.update({"robot_path": None})
res = resolve_dependencies(ecs, build_options=build_options)
# all dependencies should be resolved
self.assertEqual(0, sum(len(ec["dependencies"]) for ec in res))
# this should not resolve (cannot find gzip-1.4.eb)
ecs = [deepcopy(easyconfig_dep)]
self.assertRaises(EasyBuildError, resolve_dependencies, ecs, build_options=build_options)
# test if dependencies of an automatically found file are also loaded
easyconfig_dep["dependencies"] = [
{
"name": "gzip",
"version": "1.4",
"versionsuffix": "",
"toolchain": {"name": "GCC", "version": "4.6.3"},
"dummy": True,
}
]
ecs = [deepcopy(easyconfig_dep)]
build_options.update({"robot_path": self.base_easyconfig_dir})
res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options)
# GCC should be first (required by gzip dependency)
self.assertEqual("GCC/4.6.3", res[0]["module"])
self.assertEqual("foo/1.2.3", res[-1]["module"])
# make sure that only missing stuff is built, and that available modules are not rebuilt
# monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present
MockModule.avail_modules = [
"GCC/4.7.2",
"OpenMPI/1.6.4-GCC-4.7.2",
"OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2",
"FFTW/3.3.3-gompi-1.4.10",
"ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2",
]
easyconfig_dep["dependencies"] = [
{
"name": "goolf",
"version": "1.4.10",
"versionsuffix": "",
"toolchain": {"name": "dummy", "version": "dummy"},
"dummy": True,
}
]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies(ecs, build_options=build_options)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 2)
# goolf should be first, the software itself second
self.assertEqual("goolf/1.4.10", res[0]["module"])
self.assertEqual("foo/1.2.3", res[1]["module"])
# force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt
build_options.update({"force": True})
easyconfig["module"] = "this/is/already/there"
MockModule.avail_modules.append("this/is/already/there")
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
res = resolve_dependencies(ecs, build_options=build_options)
# there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available)
self.assertEqual(len(res), 3)
# goolf should be first, the software itself second
self.assertEqual("this/is/already/there", res[0]["module"])
self.assertEqual("goolf/1.4.10", res[1]["module"])
self.assertEqual("foo/1.2.3", res[2]["module"])
# build that are listed but already have a module available are not retained without force
build_options.update({"force": False})
newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled
res = resolve_dependencies(newecs, build_options=build_options)
self.assertEqual(len(res), 2)
self.assertEqual("goolf/1.4.10", res[0]["module"])
self.assertEqual("foo/1.2.3", res[1]["module"])
# with retain_all_deps enabled, all dependencies ae retained
build_options.update({"retain_all_deps": True})
ecs = [deepcopy(easyconfig_dep)]
newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled
res = resolve_dependencies(newecs, build_options=build_options)
self.assertEqual(len(res), 9)
self.assertEqual("GCC/4.7.2", res[0]["module"])
self.assertEqual("goolf/1.4.10", res[-2]["module"])
self.assertEqual("foo/1.2.3", res[-1]["module"])
build_options.update({"retain_all_deps": False})
# provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up
MockModule.avail_modules = ["GCC/4.7.2", "OpenMPI/1.6.4-GCC-4.7.2", "gompi/1.4.10", "FFTW/3.3.3-gompi-1.4.10"]
easyconfig_dep["dependencies"] = [
{
"name": "goolf",
"version": "1.4.10",
"versionsuffix": "",
"toolchain": {"name": "dummy", "version": "dummy"},
"dummy": True,
}
]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 4)
# goolf should be first, the software itself second
self.assertEqual("OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2", res[0]["module"])
self.assertEqual("ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2", res[1]["module"])
self.assertEqual("goolf/1.4.10", res[2]["module"])
self.assertEqual("foo/1.2.3", res[3]["module"])
