def test_resolve_dependencies_existing_modules(self):
"""Test order in case modules already being available."""
def mkdepspec(name, version):
"""Create a dep spec with given name/version."""
dep = {
'name': name,
'version': version,
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': False,
'short_mod_name': '%s/%s' % (name, version),
'full_mod_name': '%s/%s' % (name, version),
}
return dep
def mkspec(name, version, deps):
"""Create a spec with given name/version/deps."""
spec = {
'ec': {
'name': name,
'version': version,
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
},
'spec': '_',
'short_mod_name': '%s/%s' % (name, version),
'full_mod_name': '%s/%s' % (name, version),
'dependencies': [],
'parsed': True,
}
for depname, depver in deps:
spec['dependencies'].append(mkdepspec(depname, depver))
return spec
ecs = [
mkspec('three', '3.0', [('twoone', '2.1'), ('one', '1.0')]),
mkspec('four', '4.0', [('three', '3.0'), ('twoone', '2.1')]),
mkspec('twoone', '2.1', [('one', '1.0'), ('two', '2.0')]),
mkspec('two', '2.0', [('one', '1.0')]),
mkspec('one', '1.0', []),
]
expected = ['one/1.0', 'two/2.0', 'twoone/2.1', 'three/3.0', 'four/4.0']
# order is correct if modules are not available yet
res = resolve_dependencies(ecs, self.modtool)
self.assertEqual([x['full_mod_name'] for x in res], expected)
# precreate matching modules
modpath = os.path.join(self.test_prefix, 'modules')
mods = ['four/4.0', 'one/1.0', 'three/3.0', 'two/2.0', 'twooone/2.1']
for mod in mods:
write_file(os.path.join(modpath, mod), '#%Module\n')
self.reset_modulepath([modpath])
# order is correct even if modules are already available
res = resolve_dependencies(ecs, self.modtool)
self.assertEqual([x['full_mod_name'] for x in res], expected)
def tweak(easyconfigs, build_specs, targetdir=None):
"""Tweak list of easyconfigs according to provided build specifications."""
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(["%(name)s/%(version)s" % ec["ec"]["toolchain"] for ec in easyconfigs])
if len(toolchains) > 1:
raise EasyBuildError(
"Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s", toolchains
)
if "name" in build_specs or "version" in build_specs:
# no recursion if software name/version build specification are included
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs)
else:
# build specifications should be applied to the whole dependency graph
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
_log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs)
orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec["spec"] for ec in easyconfigs]
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)
# determine toolchain based on last easyconfigs
toolchain = orig_ecs[-1]["ec"]["toolchain"]
_log.debug("Filtering using toolchain %s" % toolchain)
# filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies
if toolchain["name"] != DUMMY_TOOLCHAIN_NAME:
while orig_ecs[0]["ec"]["toolchain"] != toolchain:
orig_ecs = orig_ecs[1:]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
new_ec_file = tweak_one(orig_ec["spec"], None, build_specs, targetdir=targetdir)
# only return tweaked easyconfigs for easyconfigs which were listed originally
# easyconfig files for dependencies are also generated but not included, and will be resolved via --robot
if orig_ec["spec"] in listed_ec_paths:
new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs)
tweaked_easyconfigs.extend(new_ecs)
return tweaked_easyconfigs
def test_robot_archived_easyconfigs(self):
"""Test whether robot can pick up archived easyconfigs when asked."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-ictce-4.1.13.eb')
gzip_ectxt = read_file(gzip_ec)
test_ec = os.path.join(self.test_prefix, 'test.eb')
tc_spec = "toolchain = {'name': 'ictce', 'version': '3.2.2.u3'}"
regex = re.compile("^toolchain = .*", re.M)
test_ectxt = regex.sub(tc_spec, gzip_ectxt)
write_file(test_ec, test_ectxt)
ecs, _ = parse_easyconfigs([(test_ec, False)])
self.assertErrorRegex(EasyBuildError, "Irresolvable dependencies encountered", resolve_dependencies,
ecs, self.modtool, retain_all_deps=True)
# --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs
init_config(build_options={
'consider_archived_easyconfigs': True,
'robot_path': [test_ecs],
})
res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True)
self.assertEqual([ec['full_mod_name'] for ec in res], ['ictce/3.2.2.u3', 'gzip/1.5-ictce-3.2.2.u3'])
expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb')
self.assertTrue(os.path.samefile(res[0]['spec'], expected))
def test_build_easyconfigs_in_parallel(self):
"""Basic test for build_easyconfigs_in_parallel function."""
easyconfig_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "gzip-1.5-goolf-1.4.10.eb")
easyconfigs = process_easyconfig(easyconfig_file)
ordered_ecs = resolve_dependencies(easyconfigs)
jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False)
self.assertEqual(len(jobs), 8)
def test_build_easyconfigs_in_parallel_slurm(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job."""
# install mocked versions of 'sbatch' and 'scontrol' commands
sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch')
write_file(sbatch, MOCKED_SBATCH)
adjust_permissions(sbatch, stat.S_IXUSR, add=True)
scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol')
write_file(scontrol, MOCKED_SCONTROL)
adjust_permissions(scontrol, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')])
topdir = os.path.dirname(os.path.abspath(__file__))
test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')
foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb')
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
'job_max_walltime': 5,
'force': True,
}
init_config(args=['--job-backend=Slurm'], build_options=build_options)
easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
self.mock_stdout(True)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False)
self.mock_stdout(False)
# jobs are submitted for foss & gzip (listed easyconfigs)
self.assertEqual(len(jobs), 2)
# last job (gzip) has a dependency on second-to-last job (foss)
self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a')
expected = {
'dependency': 'afterok:%s' % jobs[0].jobid,
'hold': True,
'job-name': 'gzip-1.5-foss-2018a',
'nodes': 1,
'ntasks': 3,
'ntasks-per-node': 3,
'output': '%x-%j.out',
'time': 300, # 60*5 (unit is minutes)
'wrap': "echo '%s'" % test_ec,
}
self.assertEqual(jobs[1].job_specs, expected)
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not self.parsed_easyconfigs:
for spec in specs:
self.parsed_easyconfigs.extend(process_easyconfig(spec))
self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs)
def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
"""Tweak list of easyconfigs according to provided build specifications."""
tweaked_ecs_path, tweaked_ecs_deps_path = None, None
if targetdirs is not None:
tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s",
toolchains)
if 'name' in build_specs or 'version' in build_specs:
# no recursion if software name/version build specification are included
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs)
else:
# build specifications should be applied to the whole dependency graph
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
_log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs)
orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
# determine toolchain based on last easyconfigs
if orig_ecs:
toolchain = orig_ecs[-1]['ec']['toolchain']
_log.debug("Filtering using toolchain %s" % toolchain)
# filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies
if toolchain['name'] != DUMMY_TOOLCHAIN_NAME:
while orig_ecs[0]['ec']['toolchain'] != toolchain:
orig_ecs = orig_ecs[1:]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
# Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use the
# prepended path so that they are found first).
# easyconfig files for dependencies are also generated but not included, they will be resolved via --robot
# either from existing easyconfigs or, if that fails, from easyconfigs in the appended path
if orig_ec['spec'] in listed_ec_paths:
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path)
new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs)
tweaked_easyconfigs.extend(new_ecs)
else:
# Place all tweaked dependency easyconfigs in the directory appended to the robot path
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
return tweaked_easyconfigs
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs # noqa (ignore unused import)
except ImportError:
print "GC3Pie not available, skipping test"
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False, recursive=False)
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path)
build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0'))
self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def get_dep_tree_of_toolchain(toolchain_spec, modtool):
"""
Get list of dependencies of a toolchain (as EasyConfig objects)
:param toolchain_spec: toolchain spec to get the dependencies of
:param modtool: module tool used
:return: The dependency tree of the toolchain spec
"""
path = robot_find_easyconfig(toolchain_spec['name'], toolchain_spec['version'])
if path is None:
raise EasyBuildError("Could not find easyconfig for %s toolchain version %s",
toolchain_spec['name'], toolchain_spec['version'])
ec = process_easyconfig(path, validate=False)
return [dep['ec'] for dep in resolve_dependencies(ec, modtool, retain_all_deps=True)]
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not self.parsed_easyconfigs:
for spec in specs:
self.parsed_easyconfigs.extend(process_easyconfig(spec))
# filter out external modules
for ec in self.parsed_easyconfigs:
for dep in ec['dependencies'][:]:
if dep.get('external_module', False):
ec['dependencies'].remove(dep)
self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs, modules_tool(), retain_all_deps=True)
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
build_options = {
'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'valid_module_classes': config.module_classes(),
'validate': False,
}
init_config(args=['--job-backend=PbsPython'], build_options=build_options)
ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs)
jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False)
self.assertEqual(len(jobs), 8)
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" % output_dir)
return True # success
def session_state():
"""Get session state: timestamp, dump of environment, system info."""
def xtest_resolve_dependencies(self):
""" Test with some basic testcases (also check if he can find dependencies inside the given directory """
# replace Modules class with something we have control over
config.modules_tool = mock_module
ectools.modules_tool = mock_module
robot.modules_tool = mock_module
os.environ['module'] = "() { eval `/bin/echo $*`\n}"
base_easyconfig_dir = find_full_path(
os.path.join("test", "framework", "easyconfigs"))
self.assertTrue(base_easyconfig_dir)
easyconfig = {
'spec': '_',
'full_mod_name': 'name/version',
'short_mod_name': 'name/version',
'dependencies': []
}
build_options = {
'allow_modules_tool_mismatch': True,
'robot_path': None,
'validate': False,
}
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig)])
self.assertEqual([easyconfig], res)
easyconfig_dep = {
'ec': {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
},
'spec':
'_',
'short_mod_name':
'foo/1.2.3',
'full_mod_name':
'foo/1.2.3',
'dependencies': [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'dummy': True,
'hidden': False,
}],
'parsed':
True,
}
build_options.update({
'robot': True,
'robot_path': base_easyconfig_dir
})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# dependency should be found, order should be correct
self.assertEqual(len(res), 2)
self.assertEqual('gzip/1.4', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
# hidden dependencies are found too, but only retained if they're not available (or forced to be retained
hidden_dep = {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-deps',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'dummy': True,
'hidden': True,
}
easyconfig_moredeps = deepcopy(easyconfig_dep)
easyconfig_moredeps['dependencies'].append(hidden_dep)
easyconfig_moredeps['hiddendependencies'] = [hidden_dep]
# toy/.0.0-deps is available and thus should be omitted
res = resolve_dependencies([deepcopy(easyconfig_moredeps)])
self.assertEqual(len(res), 2)
full_mod_names = [ec['full_mod_name'] for ec in res]
self.assertFalse('toy/.0.0-deps' in full_mod_names)
res = resolve_dependencies([deepcopy(easyconfig_moredeps)],
retain_all_deps=True)
self.assertEqual(
len(res), 4
) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1)
self.assertEqual('gzip/1.4', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
full_mod_names = [ec['full_mod_name'] for ec in res]
self.assertTrue('toy/.0.0-deps' in full_mod_names)
self.assertTrue('ictce/4.1.13' in full_mod_names)
# here we have included a dependency in the easyconfig list
easyconfig['full_mod_name'] = 'gzip/1.4'
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
build_options.update({'robot_path': None})
init_config(build_options=build_options)
res = resolve_dependencies(ecs)
# all dependencies should be resolved
self.assertEqual(0, sum(len(ec['dependencies']) for ec in res))
# this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled
ecs = [deepcopy(easyconfig_dep)]
msg = "Irresolvable dependencies encountered"
self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs)
# test if dependencies of an automatically found file are also loaded
easyconfig_dep['dependencies'] = [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {
'name': 'GCC',
'version': '4.6.3'
},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
build_options.update({'robot_path': base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# GCC should be first (required by gzip dependency)
self.assertEqual('GCC/4.6.3', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
# make sure that only missing stuff is built, and that available modules are not rebuilt
# monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'FFTW/3.3.3-gompi-1.4.10',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies(ecs)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 2)
# goolf should be first, the software itself second
self.assertEqual('goolf/1.4.10', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[1]['full_mod_name'])
# force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt
build_options.update({'force': True})
init_config(build_options=build_options)
easyconfig['full_mod_name'] = 'this/is/already/there'
MockModule.avail_modules.append('this/is/already/there')
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
res = resolve_dependencies(ecs)
# there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available)
self.assertEqual(len(res), 3)
# goolf should be first, the software itself second
self.assertEqual('this/is/already/there', res[0]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[1]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[2]['full_mod_name'])
# build that are listed but already have a module available are not retained without force
build_options.update({'force': False})
init_config(build_options=build_options)
newecs = skip_available(
ecs) # skip available builds since force is not enabled
res = resolve_dependencies(newecs)
self.assertEqual(len(res), 2)
self.assertEqual('goolf/1.4.10', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[1]['full_mod_name'])
# with retain_all_deps enabled, all dependencies ae retained
build_options.update({'retain_all_deps': True})
init_config(build_options=build_options)
ecs = [deepcopy(easyconfig_dep)]
newecs = skip_available(
ecs) # skip available builds since force is not enabled
res = resolve_dependencies(newecs)
self.assertEqual(len(res), 9)
self.assertEqual('GCC/4.7.2', res[0]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[-2]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
build_options.update({'retain_all_deps': False})
init_config(build_options=build_options)
# provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'FFTW/3.3.3-gompi-1.4.10',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 4)
# goolf should be first, the software itself second
self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
res[0]['full_mod_name'])
self.assertEqual(
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
res[1]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[2]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[3]['full_mod_name'])
config.modules_tool = ORIG_MODULES_TOOL
ectools.modules_tool = ORIG_ECTOOLS_MODULES_TOOL
robot.modules_tool = ORIG_ROBOT_MODULES_TOOL
if ORIG_MODULE_FUNCTION is not None:
os.environ['module'] = ORIG_MODULE_FUNCTION
else:
if 'module' in os.environ:
del os.environ['module']
def xtest_resolve_dependencies(self):
""" Test with some basic testcases (also check if he can find dependencies inside the given directory """
# replace Modules class with something we have control over
config.modules_tool = mock_module
ectools.modules_tool = mock_module
robot.modules_tool = mock_module
os.environ['module'] = "() { eval `/bin/echo $*`\n}"
base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs"))
self.assertTrue(base_easyconfig_dir)
easyconfig = {
'spec': '_',
'full_mod_name': 'name/version',
'short_mod_name': 'name/version',
'dependencies': []
}
build_options = {
'allow_modules_tool_mismatch': True,
'robot_path': None,
'validate': False,
}
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig)])
self.assertEqual([easyconfig], res)
easyconfig_dep = {
'ec': {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
},
'spec': '_',
'short_mod_name': 'foo/1.2.3',
'full_mod_name': 'foo/1.2.3',
'dependencies': [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': False,
}],
'parsed': True,
}
build_options.update({'robot': True, 'robot_path': base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# dependency should be found, order should be correct
self.assertEqual(len(res), 2)
self.assertEqual('gzip/1.4', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
# hidden dependencies are found too, but only retained if they're not available (or forced to be retained
hidden_dep = {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-deps',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': True,
}
easyconfig_moredeps = deepcopy(easyconfig_dep)
easyconfig_moredeps['dependencies'].append(hidden_dep)
easyconfig_moredeps['hiddendependencies'] = [hidden_dep]
# toy/.0.0-deps is available and thus should be omitted
res = resolve_dependencies([deepcopy(easyconfig_moredeps)])
self.assertEqual(len(res), 2)
full_mod_names = [ec['full_mod_name'] for ec in res]
self.assertFalse('toy/.0.0-deps' in full_mod_names)
res = resolve_dependencies([deepcopy(easyconfig_moredeps)], retain_all_deps=True)
self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1)
self.assertEqual('gzip/1.4', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
full_mod_names = [ec['full_mod_name'] for ec in res]
self.assertTrue('toy/.0.0-deps' in full_mod_names)
self.assertTrue('ictce/4.1.13' in full_mod_names)
# here we have included a dependency in the easyconfig list
easyconfig['full_mod_name'] = 'gzip/1.4'
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
build_options.update({'robot_path': None})
init_config(build_options=build_options)
res = resolve_dependencies(ecs)
# all dependencies should be resolved
self.assertEqual(0, sum(len(ec['dependencies']) for ec in res))
# this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled
ecs = [deepcopy(easyconfig_dep)]
msg = "Irresolvable dependencies encountered"
self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs)
# test if dependencies of an automatically found file are also loaded
easyconfig_dep['dependencies'] = [{
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'GCC', 'version': '4.6.3'},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
build_options.update({'robot_path': base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# GCC should be first (required by gzip dependency)
self.assertEqual('GCC/4.6.3', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
# make sure that only missing stuff is built, and that available modules are not rebuilt
# monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'FFTW/3.3.3-gompi-1.4.10',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies(ecs)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 2)
# goolf should be first, the software itself second
self.assertEqual('goolf/1.4.10', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[1]['full_mod_name'])
# force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt
build_options.update({'force': True})
init_config(build_options=build_options)
easyconfig['full_mod_name'] = 'this/is/already/there'
MockModule.avail_modules.append('this/is/already/there')
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
res = resolve_dependencies(ecs)
# there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available)
self.assertEqual(len(res), 3)
# goolf should be first, the software itself second
self.assertEqual('this/is/already/there', res[0]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[1]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[2]['full_mod_name'])
# build that are listed but already have a module available are not retained without force
build_options.update({'force': False})
init_config(build_options=build_options)
newecs = skip_available(ecs) # skip available builds since force is not enabled
res = resolve_dependencies(newecs)
self.assertEqual(len(res), 2)
self.assertEqual('goolf/1.4.10', res[0]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[1]['full_mod_name'])
# with retain_all_deps enabled, all dependencies ae retained
build_options.update({'retain_all_deps': True})
init_config(build_options=build_options)
ecs = [deepcopy(easyconfig_dep)]
newecs = skip_available(ecs) # skip available builds since force is not enabled
res = resolve_dependencies(newecs)
self.assertEqual(len(res), 9)
self.assertEqual('GCC/4.7.2', res[0]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[-2]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[-1]['full_mod_name'])
build_options.update({'retain_all_deps': False})
init_config(build_options=build_options)
# provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up
MockModule.avail_modules = [
'GCC/4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'FFTW/3.3.3-gompi-1.4.10',
]
easyconfig_dep['dependencies'] = [{
'name': 'goolf',
'version': '1.4.10',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'dummy': True,
'hidden': False,
}]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 4)
# goolf should be first, the software itself second
self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['full_mod_name'])
self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['full_mod_name'])
self.assertEqual('goolf/1.4.10', res[2]['full_mod_name'])
self.assertEqual('foo/1.2.3', res[3]['full_mod_name'])
config.modules_tool = ORIG_MODULES_TOOL
ectools.modules_tool = ORIG_ECTOOLS_MODULES_TOOL
robot.modules_tool = ORIG_ROBOT_MODULES_TOOL
if ORIG_MODULE_FUNCTION is not None:
os.environ['module'] = ORIG_MODULE_FUNCTION
else:
if 'module' in os.environ:
del os.environ['module']
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set by option parsers via set_tmpdir
eb_tmpdir = tempfile.gettempdir()
search_query = options.search or options.search_filename or options.search_short
# initialise logging for main
global _log
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
silent=(testing or options.terse or search_query), colorize=options.color)
# disallow running EasyBuild as root
if os.getuid() == 0:
raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, "
"so let's end this here.")
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs': build_specs,
'command_line': eb_cmd_line,
'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
'pr_path': pr_path,
'robot_path': robot_path,
'silent': testing,
'try_to_generate': try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options, cmdline_options=options)
if modtool is None:
modtool = modules_tool(testing=testing)
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
if search_query:
search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
terse=options.terse)
# GitHub options that warrant a silent cleanup & exit
if options.check_github:
check_github()
elif options.install_github_token:
install_github_token(options.github_user, silent=build_option('silent'))
elif options.review_pr:
print review_pr(options.review_pr, colored=use_color(options.color))
elif options.list_installed_software:
detailed = options.list_installed_software == 'detailed'
print list_software(output_format=options.output_format, detailed=detailed, only_installed=True)
elif options.list_software:
print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')
# non-verbose cleanup after handling GitHub integration stuff or printing terse info
early_stop_options = [
options.check_github,
options.install_github_token,
options.list_installed_software,
options.list_software,
options.review_pr,
options.terse,
search_query,
]
if any(early_stop_options):
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list() # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
if options.install_latest_eb_release:
if orig_paths:
raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
"other easyconfigs")
else:
eb_file = find_easybuild_easyconfig()
orig_paths.append(eb_file)
categorized_paths = categorize_files_by_type(orig_paths)
# command line options that do not require any easyconfigs to be specified
no_ec_opts = [options.aggregate_regtest, options.new_pr, options.regtest, options.update_pr, search_query]
# determine paths to easyconfigs
paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
if paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif not any(no_ec_opts):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if options.check_style:
_log.debug("Running style check...")
if cmdline_easyconfigs_style_check([path[0] for path in paths]):
print_msg("All style checks passed!", prefix=False)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
else:
raise EasyBuildError("One or more style checks FAILED!")
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
dry_run_mode = options.dry_run or options.dry_run_short
new_update_pr = options.new_pr or options.update_pr
# skip modules that are already installed unless forced
if not (options.force or options.rebuild or dry_run_mode or options.extended_dry_run or new_update_pr):
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
# resolve dependencies if robot is enabled, except in dry run mode
# one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
if options.robot and (not dry_run_mode or new_update_pr):
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
elif new_update_pr:
ordered_ecs = None
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# creating/updating PRs
if new_update_pr:
if options.new_pr:
new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr,
commit_msg=options.pr_commit_msg)
else:
update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg)
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
elif options.check_conflicts:
if check_conflicts(easyconfigs, modtool):
print_error("One or more conflicts detected!")
sys.exit(1)
else:
print_msg("\nNo conflicts detected!\n", prefix=False)
# dump source script to set up build environment
elif options.dump_env_script:
dump_env_script(easyconfigs)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any(no_ec_opts + [options.check_conflicts, dry_run_mode, options.dump_env_script]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def test_resolve_dependencies_minimal(self):
"""Test resolved dependencies with minimal toolchain."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
self.install_mock_module()
init_config(build_options={
'allow_modules_tool_mismatch': True,
'minimal_toolchains': True,
'use_existing_modules': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
'valid_module_classes': module_classes(),
'validate': False,
})
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_lines = [
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),",
"]",
# toolchain as list line, for easy modification later;
# the use of %(version_major)s here is mainly to check if templates are being handled correctly
# (it doesn't make much sense, but it serves the purpose)
"toolchain = {'name': 'goolf', 'version': '%(version_major)s.4.10'}",
]
write_file(barec, '\n'.join(barec_lines))
bar = process_easyconfig(barec)[0]
# all modules in the dep graph, in order
all_mods_ordered = [
'GCC/4.7.2',
'hwloc/1.6.2-GCC-4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
'FFTW/3.3.3-gompi-1.4.10',
'goolf/1.4.10',
'bar/1.2.3-goolf-1.4.10',
]
# no modules available, so all dependencies are retained
MockModule.avail_modules = []
res = resolve_dependencies([bar], self.modtool)
self.assertEqual(len(res), 10)
self.assertEqual([x['full_mod_name'] for x in res], all_mods_ordered)
MockModule.avail_modules = [
'GCC/4.7.2',
'gompi/1.4.10',
'goolf/1.4.10',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
]
# test resolving dependencies with minimal toolchain (rather than using goolf/1.4.10 for all of them)
# existing modules are *not* taken into account when determining minimal subtoolchain, by default
res = resolve_dependencies([bar], self.modtool)
self.assertEqual(len(res), 1)
self.assertEqual(res[0]['full_mod_name'], bar['ec'].full_mod_name)
# test retaining all dependencies, regardless of whether modules are available or not
res = resolve_dependencies([bar], self.modtool, retain_all_deps=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertEqual(mods, all_mods_ordered)
self.assertTrue('SQLite/3.8.10.2-GCC-4.7.2' in mods)
# test taking into account existing modules
# with an SQLite module with goolf/1.4.10 in place, this toolchain should be used rather than GCC/4.7.2
MockModule.avail_modules = [
'SQLite/3.8.10.2-goolf-1.4.10',
]
# parsed easyconfigs are cached, so clear the cache before reprocessing easyconfigs
ecec._easyconfigs_cache.clear()
bar = process_easyconfig(barec)[0]
res = resolve_dependencies([bar], self.modtool, retain_all_deps=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertTrue('SQLite/3.8.10.2-goolf-1.4.10' in mods)
self.assertFalse('SQLite/3.8.10.2-GCC-4.7.2' in mods)
# Check whether having 2 version of dummy toolchain is ok
# Clear easyconfig and toolchain caches
ecec._easyconfigs_cache.clear()
get_toolchain_hierarchy.clear()
init_config(build_options={
'allow_modules_tool_mismatch': True,
'minimal_toolchains': True,
'add_dummy_to_minimal_toolchains': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
'valid_module_classes': module_classes(),
'validate': False,
})
impi_txt = read_file(os.path.join(test_easyconfigs, 'i', 'impi', 'impi-4.1.3.049.eb'))
self.assertTrue(re.search("^toolchain = {'name': 'dummy', 'version': ''}", impi_txt, re.M))
gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4.eb'))
self.assertTrue(re.search("^toolchain = {'name': 'dummy', 'version': 'dummy'}", gzip_txt, re.M))
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_lines = [
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('impi', '4.1.3.049'),", # has toolchain ('dummy', '')
" ('gzip', '1.4'),", # has toolchain ('dummy', 'dummy')
"]",
# toolchain as list line, for easy modification later
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
]
write_file(barec, '\n'.join(barec_lines))
bar = process_easyconfig(barec)[0]
res = resolve_dependencies([bar], self.modtool, retain_all_deps=True)
self.assertEqual(len(res), 11)
mods = [x['full_mod_name'] for x in res]
self.assertTrue('impi/4.1.3.049' in mods)
self.assertTrue('gzip/1.4' in mods)
def regtest(easyconfig_paths, modtool, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
:param easyconfig_paths: path of easyconfigs to run regtest on
:param modtool: ModulesTool instance to use
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place all log files and the test output as xml
regtest_output_dir = build_option('regtest_output_dir')
testoutput = build_option('testoutput')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif testoutput is not None:
output_dir = os.path.abspath(testoutput)
else:
# default: current dir + easybuild-test-[timestamp]
dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
output_dir = os.path.join(cur_dir, dirname)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
else:
raise EasyBuildError("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError as err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" % output_dir)
return True # success
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs
except ImportError:
print "GC3Pie not available, skipping test"
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir',
'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir),
stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False,
recursive=False)
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
options = init_config(args=['--job-backend=GC3Pie'],
build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
topdir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (
topdir, test_easyblocks_path)
jobs = build_easyconfigs_in_parallel(cmd,
ordered_ecs,
prepare_first=False)
self.assertTrue(
os.path.join(self.test_installpath, 'modules', 'all', 'toy',
'0.0'))
self.assertTrue(
os.path.join(self.test_installpath, 'software', 'toy', '0.0',
'bin', 'toy'))
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError, err:
test_results.append((ecfile, "parsing_easyconfigs", "easyconfig file error: %s" % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option("force"):
_log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs)
easyconfigs = skip_available(easyconfigs)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option("sequential"):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, build_specs=build_specs)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {"cmd": cmd}
jobs = build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
print "List of submitted jobs:"
for job in jobs:
print "%s: %s" % (job.name, job.jobid)
print "(%d jobs submitted)" % len(jobs)
# determine leaf nodes in dependency graph, and report them
all_deps = set()
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
options, orig_paths = eb_go.options, eb_go.args
global _log
(build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings
# load hook implementations (if any)
hooks = load_hooks(options.hooks)
run_hook(START, hooks)
if modtool is None:
modtool = modules_tool(testing=testing)
# check whether any (EasyBuild-generated) modules are loaded already in the current session
modtool.check_loaded_modules()
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
if search_query:
search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
terse=options.terse)
# GitHub options that warrant a silent cleanup & exit
if options.check_github:
check_github()
elif options.install_github_token:
install_github_token(options.github_user, silent=build_option('silent'))
elif options.close_pr:
close_pr(options.close_pr, motivation_msg=options.close_pr_msg)
elif options.list_prs:
print(list_prs(options.list_prs))
elif options.merge_pr:
merge_pr(options.merge_pr)
elif options.review_pr:
print(review_pr(pr=options.review_pr, colored=use_color(options.color)))
elif options.list_installed_software:
detailed = options.list_installed_software == 'detailed'
print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True))
elif options.list_software:
print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed'))
# non-verbose cleanup after handling GitHub integration stuff or printing terse info
early_stop_options = [
options.check_github,
options.install_github_token,
options.list_installed_software,
options.list_software,
options.close_pr,
options.list_prs,
options.merge_pr,
options.review_pr,
options.terse,
search_query,
]
if any(early_stop_options):
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list() # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
if options.install_latest_eb_release:
if orig_paths:
raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
"other easyconfigs")
else:
eb_file = find_easybuild_easyconfig()
orig_paths.append(eb_file)
categorized_paths = categorize_files_by_type(orig_paths)
# command line options that do not require any easyconfigs to be specified
new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr
no_ec_opts = [options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr]
# determine paths to easyconfigs
determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
if determined_paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in determined_paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif any(no_ec_opts):
paths = determined_paths
else:
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums)
# handle --check-contrib & --check-style options
if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style):
clean_exit(logfile, eb_tmpdir, testing)
# verify easyconfig filenames, if desired
if options.verify_easyconfig_filenames:
_log.info("Verifying easyconfig filenames...")
for easyconfig in easyconfigs:
verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec'])
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
if options.containerize:
# if --containerize/-C create a container recipe (and optionally container image), and stop
containerize(easyconfigs)
clean_exit(logfile, eb_tmpdir, testing)
forced = options.force or options.rebuild
dry_run_mode = options.dry_run or options.dry_run_short
# skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums):
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
# resolve dependencies if robot is enabled, except in dry run mode
# one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
if options.robot and (not dry_run_mode or new_update_preview_pr):
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
elif new_update_preview_pr:
ordered_ecs = None
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# creating/updating PRs
if new_update_preview_pr:
if options.new_pr:
new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr,
commit_msg=options.pr_commit_msg)
elif options.preview_pr:
print(review_pr(paths=determined_paths, colored=use_color(options.color)))
else:
update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
elif options.check_conflicts:
if check_conflicts(easyconfigs, modtool):
print_error("One or more conflicts detected!")
sys.exit(1)
else:
print_msg("\nNo conflicts detected!\n", prefix=False)
# dump source script to set up build environment
elif options.dump_env_script:
dump_env_script(easyconfigs)
elif options.inject_checksums:
inject_checksums(ordered_ecs, options.inject_checksums)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
if any(no_ec_opts) or any(stop_options):
clean_exit(logfile, eb_tmpdir, testing)
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
clean_exit(logfile, eb_tmpdir, testing)
# build software, will exit when errors occurs (except when testing)
if not testing or (testing and do_build):
exit_on_failure = not (options.dump_test_report or options.upload_test_report)
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
run_hook(END, hooks)
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
"""Tweak list of easyconfigs according to provided build specifications."""
tweaked_ecs_path, tweaked_ecs_deps_path = None, None
if targetdirs is not None:
tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s",
toolchains)
# Toolchain is unique, let's store it
source_toolchain = easyconfigs[-1]['ec']['toolchain']
modifying_toolchains = False
target_toolchain = {}
src_to_dst_tc_mapping = {}
revert_to_regex = False
if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs:
keys = build_specs.keys()
# Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not
# supported
if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain'] for key in keys):
warning_msg = "Combining --try-toolchain* with other build options is not fully supported: using regex"
print_warning(warning_msg, silent=build_option('silent'))
revert_to_regex = True
if not revert_to_regex:
# we're doing something with the toolchain,
# so build specifications should be applied to whole dependency graph;
# obtain full dependency graph for specified easyconfigs;
# easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first)
modifying_toolchains = True
if 'toolchain_name' in keys:
target_toolchain['name'] = build_specs['toolchain_name']
else:
target_toolchain['name'] = source_toolchain['name']
if 'toolchain_version' in keys:
target_toolchain['version'] = build_specs['toolchain_version']
else:
target_toolchain['version'] = source_toolchain['version']
if build_option('map_toolchains'):
try:
src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool)
except EasyBuildError as err:
# make sure exception was raised by match_minimum_tc_specs because toolchain mapping didn't work
if "No possible mapping from source toolchain" in err.msg:
error_msg = err.msg + '\n'
error_msg += "Toolchain %s is not equivalent to toolchain %s in terms of capabilities. "
error_msg += "(If you know what you are doing, "
error_msg += "you can use --disable-map-toolchains to proceed anyway.)"
raise EasyBuildError(error_msg, target_toolchain['name'], source_toolchain['name'])
else:
# simply re-raise the exception if something else went wrong
raise err
else:
msg = "Mapping of (sub)toolchains disabled, so falling back to regex mode, "
msg += "disabling recursion and not changing (sub)toolchains for dependencies"
_log.info(msg)
revert_to_regex = True
modifying_toolchains = False
if not revert_to_regex:
_log.debug("Applying build specifications recursively (no software name/version found): %s", build_specs)
orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True)
# Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively)
# We can leave any dependencies they may have as they will only be used if required (or originally listed)
_log.debug("Filtering out toolchain hierarchy for %s", source_toolchain)
i = 0
while i < len(orig_ecs):
tc_names = [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)]
if orig_ecs[i]['ec']['name'] in tc_names:
# drop elements in toolchain hierarchy
del orig_ecs[i]
else:
i += 1
else:
revert_to_regex = True
if revert_to_regex:
# no recursion if software name/version build specification are included or we are amending something
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
# Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line
# (and use the prepended path so that they are found first).
# easyconfig files for dependencies are also generated but not included, they will be resolved via --robot
# either from existing easyconfigs or, if that fails, from easyconfigs in the appended path
tc_name = orig_ec['ec']['toolchain']['name']
new_ec_file = None
verification_build_specs = copy.copy(build_specs)
if orig_ec['spec'] in listed_ec_paths:
if modifying_toolchains:
if tc_name in src_to_dst_tc_mapping:
new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
tweaked_ecs_path)
# Need to update the toolchain in the build_specs to match the toolchain mapping
keys = verification_build_specs.keys()
if 'toolchain_name' in keys:
verification_build_specs['toolchain_name'] = src_to_dst_tc_mapping[tc_name]['name']
if 'toolchain_version' in keys:
verification_build_specs['toolchain_version'] = src_to_dst_tc_mapping[tc_name]['version']
if 'toolchain' in keys:
verification_build_specs['toolchain'] = src_to_dst_tc_mapping[tc_name]
else:
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_path)
if new_ec_file:
new_ecs = process_easyconfig(new_ec_file, build_specs=verification_build_specs)
tweaked_easyconfigs.extend(new_ecs)
else:
# Place all tweaked dependency easyconfigs in the directory appended to the robot path
if modifying_toolchains:
if tc_name in src_to_dst_tc_mapping:
new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
targetdir=tweaked_ecs_deps_path)
else:
new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
return tweaked_easyconfigs
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set by option parsers via set_tmpdir
eb_tmpdir = tempfile.gettempdir()
search_query = options.search or options.search_filename or options.search_short
# initialise logging for main
global _log
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
silent=(testing or options.terse or search_query), colorize=options.color)
# disallow running EasyBuild as root (by default)
check_root_usage(allow_use_as_root=options.allow_use_as_root_and_accept_consequences)
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs': build_specs,
'command_line': eb_cmd_line,
'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
'pr_path': pr_path,
'robot_path': robot_path,
'silent': testing,
'try_to_generate': try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options, cmdline_options=options)
# load hook implementations (if any)
hooks = load_hooks(options.hooks)
run_hook(START, hooks)
if modtool is None:
modtool = modules_tool(testing=testing)
# check whether any (EasyBuild-generated) modules are loaded already in the current session
modtool.check_loaded_modules()
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
if search_query:
search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
terse=options.terse)
# GitHub options that warrant a silent cleanup & exit
if options.check_github:
check_github()
elif options.install_github_token:
install_github_token(options.github_user, silent=build_option('silent'))
elif options.merge_pr:
merge_pr(options.merge_pr)
elif options.review_pr:
print review_pr(pr=options.review_pr, colored=use_color(options.color))
elif options.list_installed_software:
detailed = options.list_installed_software == 'detailed'
print list_software(output_format=options.output_format, detailed=detailed, only_installed=True)
elif options.list_software:
print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')
# non-verbose cleanup after handling GitHub integration stuff or printing terse info
early_stop_options = [
options.check_github,
options.install_github_token,
options.list_installed_software,
options.list_software,
options.merge_pr,
options.review_pr,
options.terse,
search_query,
]
if any(early_stop_options):
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list() # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
if options.install_latest_eb_release:
if orig_paths:
raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
"other easyconfigs")
else:
eb_file = find_easybuild_easyconfig()
orig_paths.append(eb_file)
categorized_paths = categorize_files_by_type(orig_paths)
# command line options that do not require any easyconfigs to be specified
new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr
no_ec_opts = [options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr]
# determine paths to easyconfigs
determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
if determined_paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in determined_paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif any(no_ec_opts):
paths = determined_paths
else:
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums)
# handle --check-contrib & --check-style options
if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style):
clean_exit(logfile, eb_tmpdir, testing)
# verify easyconfig filenames, if desired
if options.verify_easyconfig_filenames:
_log.info("Verifying easyconfig filenames...")
for easyconfig in easyconfigs:
verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec'])
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
if options.containerize:
# if --containerize/-C create a container recipe (and optionally container image), and stop
containerize(easyconfigs)
clean_exit(logfile, eb_tmpdir, testing)
forced = options.force or options.rebuild
dry_run_mode = options.dry_run or options.dry_run_short
# skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums):
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
# resolve dependencies if robot is enabled, except in dry run mode
# one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
if options.robot and (not dry_run_mode or new_update_preview_pr):
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
elif new_update_preview_pr:
ordered_ecs = None
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# creating/updating PRs
if new_update_preview_pr:
if options.new_pr:
new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr,
commit_msg=options.pr_commit_msg)
elif options.preview_pr:
print review_pr(paths=determined_paths, colored=use_color(options.color))
else:
update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
elif options.check_conflicts:
if check_conflicts(easyconfigs, modtool):
print_error("One or more conflicts detected!")
sys.exit(1)
else:
print_msg("\nNo conflicts detected!\n", prefix=False)
# dump source script to set up build environment
elif options.dump_env_script:
dump_env_script(easyconfigs)
elif options.inject_checksums:
inject_checksums(ordered_ecs, options.inject_checksums)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
if any(no_ec_opts) or any(stop_options):
clean_exit(logfile, eb_tmpdir, testing)
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
clean_exit(logfile, eb_tmpdir, testing)
# build software, will exit when errors occurs (except when testing)
if not testing or (testing and do_build):
exit_on_failure = not (options.dump_test_report or options.upload_test_report)
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
run_hook(END, hooks)
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
"""
register_lock_cleanup_signal_handlers()
# if $CDPATH is set, unset it, it'll only cause trouble...
# see https://github.com/easybuilders/easybuild-framework/issues/2944
if 'CDPATH' in os.environ:
del os.environ['CDPATH']
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
options, orig_paths = eb_go.options, eb_go.args
global _log
(build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate,
from_pr_list, tweaked_ecs_paths) = cfg_settings
# load hook implementations (if any)
hooks = load_hooks(options.hooks)
run_hook(START, hooks)
if modtool is None:
modtool = modules_tool(testing=testing)
# check whether any (EasyBuild-generated) modules are loaded already in the current session
modtool.check_loaded_modules()
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# if easystack is provided with the command, commands with arguments from it will be executed
if options.easystack:
# TODO add general_options (i.e. robot) to build options
orig_paths, general_options = parse_easystack(options.easystack)
if general_options:
raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.")
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
if search_query:
search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
terse=options.terse)
if options.check_eb_deps:
print_checks(check_easybuild_deps(modtool))
# GitHub options that warrant a silent cleanup & exit
if options.check_github:
check_github()
elif options.install_github_token:
install_github_token(options.github_user, silent=build_option('silent'))
elif options.close_pr:
close_pr(options.close_pr, motivation_msg=options.close_pr_msg)
elif options.list_prs:
print(list_prs(options.list_prs))
elif options.merge_pr:
merge_pr(options.merge_pr)
elif options.review_pr:
print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing,
max_ecs=options.review_pr_max, filter_ecs=options.review_pr_filter))
elif options.add_pr_labels:
add_pr_labels(options.add_pr_labels)
elif options.list_installed_software:
detailed = options.list_installed_software == 'detailed'
print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True))
elif options.list_software:
print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed'))
elif options.create_index:
print_msg("Creating index for %s..." % options.create_index, prefix=False)
index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age)
index = load_index(options.create_index)
print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False)
# non-verbose cleanup after handling GitHub integration stuff or printing terse info
early_stop_options = [
options.add_pr_labels,
options.check_eb_deps,
options.check_github,
options.create_index,
options.install_github_token,
options.list_installed_software,
options.list_software,
options.close_pr,
options.list_prs,
options.merge_pr,
options.review_pr,
options.terse,
search_query,
]
if any(early_stop_options):
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list() # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
if options.skip_test_step:
if options.ignore_test_failure:
raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. "
"Please use only one of them.")
else:
print_warning("Will not run the test step as requested via skip-test-step. "
"Consider using ignore-test-failure instead and verify the results afterwards")
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
if options.install_latest_eb_release:
if orig_paths:
raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
"other easyconfigs")
else:
eb_file = find_easybuild_easyconfig()
orig_paths.append(eb_file)
if options.copy_ec:
# figure out list of files to copy + target location (taking into account --from-pr)
orig_paths, target_path = det_copy_ec_specs(orig_paths, from_pr_list)
categorized_paths = categorize_files_by_type(orig_paths)
# command line options that do not require any easyconfigs to be specified
pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr
pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop
pr_options = pr_options or options.update_branch_github or options.update_pr
no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query]
# determine paths to easyconfigs
determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
# only copy easyconfigs here if we're not using --try-* (that's handled below)
copy_ec = options.copy_ec and not tweaked_ecs_paths
if copy_ec or options.fix_deprecated_easyconfigs or options.show_ec:
if options.copy_ec:
# at this point some paths may still just be filenames rather than absolute paths,
# so try to determine full path for those too via robot search path
paths = locate_files(orig_paths, robot_path)
copy_files(paths, target_path, target_single_file=True, allow_empty=False, verbose=True)
elif options.fix_deprecated_easyconfigs:
fix_deprecated_easyconfigs(determined_paths)
elif options.show_ec:
for path in determined_paths:
print_msg("Contents of %s:" % path)
print_msg(read_file(path), prefix=False)
clean_exit(logfile, eb_tmpdir, testing)
if determined_paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in determined_paths]
elif 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif any(no_ec_opts):
paths = determined_paths
else:
print_error("Please provide one or multiple easyconfig files, or use software build " +
"options to make EasyBuild search for easyconfigs",
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s", paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([x for (x, _) in paths] or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums)
# handle --check-contrib & --check-style options
if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style):
clean_exit(logfile, eb_tmpdir, testing)
# verify easyconfig filenames, if desired
if options.verify_easyconfig_filenames:
_log.info("Verifying easyconfig filenames...")
for easyconfig in easyconfigs:
verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec'])
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
if options.containerize:
# if --containerize/-C create a container recipe (and optionally container image), and stop
containerize(easyconfigs)
clean_exit(logfile, eb_tmpdir, testing)
forced = options.force or options.rebuild
dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules
keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options
keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only
# skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
if not keep_available_modules:
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# keep track for which easyconfigs we should set the corresponding module as default
if options.set_default_module:
for easyconfig in easyconfigs:
easyconfig['ec'].set_default_module = True
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
# resolve dependencies if robot is enabled, except in dry run mode
# one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
if options.robot and (not dry_run_mode or pr_options):
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
elif pr_options:
ordered_ecs = None
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
if options.copy_ec and tweaked_ecs_paths:
all_specs = [spec['spec'] for spec in
resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)]
tweaked_ecs_in_all_ecs = [path for path in all_specs if
any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)]
if tweaked_ecs_in_all_ecs:
# Clean them, then copy them
clean_up_easyconfigs(tweaked_ecs_in_all_ecs)
copy_files(tweaked_ecs_in_all_ecs, target_path, allow_empty=False, verbose=True)
clean_exit(logfile, eb_tmpdir, testing)
# creating/updating PRs
if pr_options:
if options.new_pr:
new_pr(categorized_paths, ordered_ecs)
elif options.new_branch_github:
new_branch_github(categorized_paths, ordered_ecs)
elif options.new_pr_from_branch:
new_pr_from_branch(options.new_pr_from_branch)
elif options.preview_pr:
print(review_pr(paths=determined_paths, colored=use_color(options.color)))
elif options.sync_branch_with_develop:
sync_branch_with_develop(options.sync_branch_with_develop)
elif options.sync_pr_with_develop:
sync_pr_with_develop(options.sync_pr_with_develop)
elif options.update_branch_github:
update_branch(options.update_branch_github, categorized_paths, ordered_ecs)
elif options.update_pr:
update_pr(options.update_pr, categorized_paths, ordered_ecs)
else:
raise EasyBuildError("Unknown PR option!")
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
if options.missing_modules:
txt = missing_deps(easyconfigs, modtool)
else:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
elif options.check_conflicts:
if check_conflicts(easyconfigs, modtool):
print_error("One or more conflicts detected!")
sys.exit(1)
else:
print_msg("\nNo conflicts detected!\n", prefix=False)
# dump source script to set up build environment
elif options.dump_env_script:
dump_env_script(easyconfigs)
elif options.inject_checksums:
with rich_live_cm():
inject_checksums(ordered_ecs, options.inject_checksums)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
if any(no_ec_opts) or any(stop_options):
clean_exit(logfile, eb_tmpdir, testing)
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
clean_exit(logfile, eb_tmpdir, testing)
# build software, will exit when errors occurs (except when testing)
if not testing or (testing and do_build):
exit_on_failure = not (options.dump_test_report or options.upload_test_report)
with rich_live_cm():
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded "
if build_option('ignore_test_failure'):
success_msg += "(with --ignore-test-failure) "
success_msg += "for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
run_hook(END, hooks)
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
@param args: command line arguments to use
@param logfile: log file to use
@param do_build: whether or not to actually perform the build
@param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set by option parsers via set_tmpdir
eb_tmpdir = tempfile.gettempdir()
# initialise logging for main
global _log
_log, logfile = init_logging(logfile,
logtostdout=options.logtostdout,
silent=testing or options.terse)
# disallow running EasyBuild as root
if os.getuid() == 0:
raise EasyBuildError(
"You seem to be running EasyBuild with root privileges which is not wise, "
"so let's end this here.")
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir,
tweaked_ecs=tweaked_ecs,
from_pr=options.from_pr)
auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
robot_path = det_robot_path(options.robot_paths,
tweaked_ecs_path,
pr_path,
auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs':
build_specs,
'command_line':
eb_cmd_line,
'external_modules_metadata':
parse_external_modules_metadata(options.external_modules_metadata),
'pr_path':
pr_path,
'robot_path':
robot_path,
'silent':
testing,
'try_to_generate':
try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options,
cmdline_options=options)
if modtool is None:
modtool = modules_tool(testing=testing)
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug(
"Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
query = options.search or options.search_filename or options.search_short
if query:
search_easyconfigs(query,
short=options.search_short,
filename_only=options.search_filename,
terse=options.terse)
# GitHub integration
cleanup_and_exit = handle_github_options(options, orig_paths)
# non-verbose cleanup and exit after handling GitHub integration stuff or printing terse info
if cleanup_and_exit or options.terse:
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list(
) # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# command line options that do not require any easyconfigs to be specified
no_ec_opts = [
options.aggregate_regtest, options.new_pr, options.review_pr,
options.search, options.search_filename, options.search_short,
options.regtest, options.update_pr
]
# determine paths to easyconfigs
paths = det_easyconfig_paths(orig_paths)
if paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs,
robot_path,
try_to_generate,
testing=testing)
elif not any(no_ec_opts):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths]
or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs,
build_specs,
modtool,
targetdir=tweaked_ecs_path)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
# dump source script to set up build environment
if options.dump_env_script:
dump_env_script(easyconfigs)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any(no_ec_opts +
[options.dry_run, options.dry_run_short, options.dump_env_script]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not (options.force or options.rebuild or options.extended_dry_run):
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [
ec for ec in easyconfigs if ec not in retained_ecs
]:
print_msg("%s is already installed (module found), skipping" %
skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
if options.robot:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(
ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([
ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)
])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt,
len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths),
overall_success, success_msg,
init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
}
init_config(args=['--job-backend=PbsPython'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False)
# only one job submitted since foss/2018a module is already available
self.assertEqual(len(jobs), 1)
regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'")
self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script))
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True)
jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False)
# make sure command is correct, and that --hidden is there when it needs to be
for i, ec in enumerate(ordered_ecs):
if ec['hidden']:
regex = re.compile("eb %s.* --hidden" % ec['spec'])
else:
regex = re.compile("eb %s" % ec['spec'])
self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script))
for job in jobs:
self.assertEqual(job.cores, build_options['job_cores'])
# no deps for GCC/4.6.3 (toolchain) and intel/2018a (test easyconfig with 'fake' deps)
self.assertEqual(len(jobs[0].deps), 0)
self.assertEqual(len(jobs[1].deps), 0)
# only dependency for toy/0.0-deps is intel/2018a (dep marked as external module is filtered out)
self.assertTrue('toy-0.0-deps.eb' in jobs[2].script)
self.assertEqual(len(jobs[2].deps), 1)
self.assertTrue('intel-2018a.eb' in jobs[2].deps[0].script)
# dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
self.assertEqual(len(jobs[3].deps), 2)
regex = re.compile('toy-0.0-deps.eb\s* --hidden')
self.assertTrue(regex.search(jobs[3].deps[0].script))
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
# also test use of --pre-create-installdir
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool)
# installation directory doesn't exist yet before submission
toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
self.assertFalse(os.path.exists(toy_installdir))
jobs = submit_jobs(ordered_ecs, '', testing=False)
self.assertEqual(len(jobs), 1)
# software install dir is created (by default) as part of job submission process (fetch_step is run)
self.assertTrue(os.path.exists(toy_installdir))
remove_dir(toy_installdir)
remove_dir(os.path.dirname(toy_installdir))
self.assertFalse(os.path.exists(toy_installdir))
# installation directory does *not* get created when --pre-create-installdir is used
build_options['pre_create_installdir'] = False
init_config(args=['--job-backend=PbsPython'], build_options=build_options)
jobs = submit_jobs(ordered_ecs, '', testing=False)
self.assertEqual(len(jobs), 1)
self.assertFalse(os.path.exists(toy_installdir))
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(
self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
}
init_config(args=['--job-backend=PbsPython'],
build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip',
'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'",
ordered_ecs,
prepare_first=False)
self.assertEqual(len(jobs), 8)
regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'")
self.assertTrue(
regex.search(jobs[-1].script),
"Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script))
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip',
'gzip-1.4-GCC-4.6.3.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file),
self.modtool,
retain_all_deps=True)
jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False)
# make sure command is correct, and that --hidden is there when it needs to be
for i, ec in enumerate(ordered_ecs):
if ec['hidden']:
regex = re.compile("eb %s.* --hidden" % ec['spec'])
else:
regex = re.compile("eb %s" % ec['spec'])
self.assertTrue(
regex.search(jobs[i].script),
"Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script))
for job in jobs:
self.assertEqual(job.cores, build_options['job_cores'])
# no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps)
self.assertEqual(len(jobs[0].deps), 0)
self.assertEqual(len(jobs[1].deps), 0)
# only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out)
self.assertTrue('toy-0.0-deps.eb' in jobs[2].script)
self.assertEqual(len(jobs[2].deps), 1)
self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script)
# dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
self.assertEqual(len(jobs[3].deps), 2)
regex = re.compile('toy-0.0-deps.eb\s* --hidden')
self.assertTrue(regex.search(jobs[3].deps[0].script))
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs',
'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug(
"Skipping easyconfigs from %s that already have a module available..."
% easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {
'cmd': cmd
}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" %
output_dir)
return True # success
def main(args=None, logfile=None, do_build=None, testing=False):
"""
Main function: parse command line options, and act accordingly.
@param args: command line arguments to use
@param logfile: log file to use
@param do_build: whether or not to actually perform the build
@param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set by option parsers via set_tmpdir
eb_tmpdir = tempfile.gettempdir()
# initialise logging for main
global _log
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=testing or options.last_log)
# disallow running EasyBuild as root
if os.getuid() == 0:
raise EasyBuildError(
"You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here."
)
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section("config")
build_options = {
"build_specs": build_specs,
"command_line": eb_cmd_line,
"pr_path": pr_path,
"robot_path": robot_path,
"silent": testing,
"try_to_generate": try_to_generate,
"valid_stops": [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options, cmdline_options=options)
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or "(none)"
print_msg(last_log, log=_log, prefix=False)
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works
init_session_state.update({"easybuild_configuration": eb_config})
init_session_state.update({"module_list": modlist})
_log.debug("Initial session state: %s" % init_session_state)
# GitHub integration
if options.review_pr or options.new_pr or options.update_pr:
if options.review_pr:
print review_pr(options.review_pr, colored=options.color)
elif options.new_pr:
new_pr(orig_paths, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg)
elif options.update_pr:
update_pr(options.update_pr, orig_paths, commit_msg=options.pr_commit_msg)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# search for easyconfigs, if a query is specified
query = options.search or options.search_short
if query:
search_easyconfigs(query, short=not options.search)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# command line options that do not require any easyconfigs to be specified
no_ec_opts = [
options.aggregate_regtest,
options.new_pr,
options.review_pr,
options.search,
options.search_short,
options.regtest,
options.update_pr,
]
# determine paths to easyconfigs
paths = det_easyconfig_paths(orig_paths)
if paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in paths]
else:
if "name" in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif not any(no_ec_opts):
print_error(
(
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"
),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing,
)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
txt = dry_run(easyconfigs, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any(no_ec_opts + [options.dry_run, options.dry_run_short]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not (options.force or options.rebuild or options.extended_dry_run):
retained_ecs = skip_available(easyconfigs)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec["full_mod_name"])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
if options.robot:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(
easyconfigs,
minimal_toolchains=build_option("minimal_toolchains"),
use_existing_modules=build_option("use_existing_modules"),
)
else:
ordered_ecs = easyconfigs
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get("success", False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if "original_spec" in ec and os.path.isfile(ec["spec"]):
os.remove(ec["spec"])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def test_resolve_dependencies_minimal(self):
"""Test resolved dependencies with minimal toolchain."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
install_mock_module()
init_config(build_options={
'allow_modules_tool_mismatch': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
'valid_module_classes': module_classes(),
'validate': False,
})
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_lines = [
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),",
"]",
# toolchain as list line, for easy modification later
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
]
write_file(barec, '\n'.join(barec_lines))
bar = process_easyconfig(barec)[0]
# all modules in the dep graph, in order
all_mods_ordered = [
'GCC/4.7.2',
'hwloc/1.6.2-GCC-4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
'FFTW/3.3.3-gompi-1.4.10',
'goolf/1.4.10',
'bar/1.2.3-goolf-1.4.10',
]
# no modules available, so all dependencies are retained
MockModule.avail_modules = []
res = resolve_dependencies([bar], minimal_toolchains=True)
self.assertEqual(len(res), 10)
self.assertEqual([x['full_mod_name'] for x in res], all_mods_ordered)
# cleanup
shutil.rmtree(os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
MockModule.avail_modules = [
'GCC/4.7.2',
'gompi/1.4.10',
'goolf/1.4.10',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
]
# test resolving dependencies with minimal toolchain (rather than using goolf/1.4.10 for all of them)
# existing modules are *not* taken into account when determining minimal subtoolchain, by default
res = resolve_dependencies([bar], minimal_toolchains=True)
self.assertEqual(len(res), 1)
self.assertEqual(res[0]['full_mod_name'], bar['ec'].full_mod_name)
# cleanup
shutil.rmtree(os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
# test retaining all dependencies, regardless of whether modules are available or not
res = resolve_dependencies([bar], minimal_toolchains=True, retain_all_deps=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertEqual(mods, all_mods_ordered)
self.assertTrue('SQLite/3.8.10.2-GCC-4.7.2' in mods)
# cleanup
shutil.rmtree(os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
# test taking into account existing modules
# with an SQLite module with goolf/1.4.10 in place, this toolchain should be used rather than GCC/4.7.2
MockModule.avail_modules = [
'SQLite/3.8.10.2-goolf-1.4.10',
]
res = resolve_dependencies([bar], minimal_toolchains=True, retain_all_deps=True, use_existing_modules=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertTrue('SQLite/3.8.10.2-goolf-1.4.10' in mods)
self.assertFalse('SQLite/3.8.10.2-GCC-4.7.2' in mods)
def test_resolve_dependencies_minimal(self):
"""Test resolved dependencies with minimal toolchain."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
install_mock_module()
init_config(
build_options={
'allow_modules_tool_mismatch': True,
'external_modules_metadata': ConfigObj(),
'robot_path': test_easyconfigs,
'valid_module_classes': module_classes(),
'validate': False,
})
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_lines = [
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),",
"]",
# toolchain as list line, for easy modification later
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
]
write_file(barec, '\n'.join(barec_lines))
bar = process_easyconfig(barec)[0]
# all modules in the dep graph, in order
all_mods_ordered = [
'GCC/4.7.2',
'hwloc/1.6.2-GCC-4.7.2',
'OpenMPI/1.6.4-GCC-4.7.2',
'gompi/1.4.10',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
'FFTW/3.3.3-gompi-1.4.10',
'goolf/1.4.10',
'bar/1.2.3-goolf-1.4.10',
]
# no modules available, so all dependencies are retained
MockModule.avail_modules = []
res = resolve_dependencies([bar], minimal_toolchains=True)
self.assertEqual(len(res), 10)
self.assertEqual([x['full_mod_name'] for x in res], all_mods_ordered)
# cleanup
shutil.rmtree(
os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
MockModule.avail_modules = [
'GCC/4.7.2',
'gompi/1.4.10',
'goolf/1.4.10',
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
]
# test resolving dependencies with minimal toolchain (rather than using goolf/1.4.10 for all of them)
# existing modules are *not* taken into account when determining minimal subtoolchain, by default
res = resolve_dependencies([bar], minimal_toolchains=True)
self.assertEqual(len(res), 1)
self.assertEqual(res[0]['full_mod_name'], bar['ec'].full_mod_name)
# cleanup
shutil.rmtree(
os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
# test retaining all dependencies, regardless of whether modules are available or not
res = resolve_dependencies([bar],
minimal_toolchains=True,
retain_all_deps=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertEqual(mods, all_mods_ordered)
self.assertTrue('SQLite/3.8.10.2-GCC-4.7.2' in mods)
# cleanup
shutil.rmtree(
os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs'))
# test taking into account existing modules
# with an SQLite module with goolf/1.4.10 in place, this toolchain should be used rather than GCC/4.7.2
MockModule.avail_modules = [
'SQLite/3.8.10.2-goolf-1.4.10',
]
res = resolve_dependencies([bar],
minimal_toolchains=True,
retain_all_deps=True,
use_existing_modules=True)
self.assertEqual(len(res), 10)
mods = [x['full_mod_name'] for x in res]
self.assertTrue('SQLite/3.8.10.2-goolf-1.4.10' in mods)
self.assertFalse('SQLite/3.8.10.2-GCC-4.7.2' in mods)
def test_build_easyconfigs_in_parallel(self):
"""Basic test for build_easyconfigs_in_parallel function."""
easyconfig_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "gzip-1.5-goolf-1.4.10.eb")
easyconfigs = process_easyconfig(easyconfig_file, validate=False)
ordered_ecs = resolve_dependencies(easyconfigs)
build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs)
def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
"""Tweak list of easyconfigs according to provided build specifications."""
tweaked_ecs_path, tweaked_ecs_deps_path = None, None
if targetdirs is not None:
tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(
['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
raise EasyBuildError(
"Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s",
toolchains)
# Toolchain is unique, let's store it
source_toolchain = easyconfigs[-1]['ec']['toolchain']
modifying_toolchains = False
target_toolchain = {}
src_to_dst_tc_mapping = {}
revert_to_regex = False
if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs:
keys = build_specs.keys()
# Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not
# supported
if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain']
for key in keys):
warning_msg = "Combining --try-toolchain* with other build options is not fully supported: using regex"
print_warning(warning_msg, silent=build_option('silent'))
revert_to_regex = True
if not revert_to_regex:
# we're doing something with the toolchain,
# so build specifications should be applied to whole dependency graph;
# obtain full dependency graph for specified easyconfigs;
# easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first)
modifying_toolchains = True
if 'toolchain_name' in keys:
target_toolchain['name'] = build_specs['toolchain_name']
else:
target_toolchain['name'] = source_toolchain['name']
if 'toolchain_version' in keys:
target_toolchain['version'] = build_specs['toolchain_version']
else:
target_toolchain['version'] = source_toolchain['version']
if build_option('map_toolchains'):
try:
src_to_dst_tc_mapping = map_toolchain_hierarchies(
source_toolchain, target_toolchain, modtool)
except EasyBuildError as err:
# make sure exception was raised by match_minimum_tc_specs because toolchain mapping didn't work
if "No possible mapping from source toolchain" in err.msg:
error_msg = err.msg + '\n'
error_msg += "Toolchain %s is not equivalent to toolchain %s in terms of capabilities. "
error_msg += "(If you know what you are doing, "
error_msg += "you can use --disable-map-toolchains to proceed anyway.)"
raise EasyBuildError(error_msg,
target_toolchain['name'],
source_toolchain['name'])
else:
# simply re-raise the exception if something else went wrong
raise err
else:
msg = "Mapping of (sub)toolchains disabled, so falling back to regex mode, "
msg += "disabling recursion and not changing (sub)toolchains for dependencies"
_log.info(msg)
revert_to_regex = True
modifying_toolchains = False
if not revert_to_regex:
_log.debug(
"Applying build specifications recursively (no software name/version found): %s",
build_specs)
orig_ecs = resolve_dependencies(easyconfigs,
modtool,
retain_all_deps=True)
# Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively)
# We can leave any dependencies they may have as they will only be used if required (or originally listed)
_log.debug("Filtering out toolchain hierarchy for %s",
source_toolchain)
i = 0
while i < len(orig_ecs):
tc_names = [
tc['name']
for tc in get_toolchain_hierarchy(source_toolchain)
]
if orig_ecs[i]['ec']['name'] in tc_names:
# drop elements in toolchain hierarchy
del orig_ecs[i]
else:
i += 1
else:
revert_to_regex = True
if revert_to_regex:
# no recursion if software name/version build specification are included or we are amending something
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug(
"Software name/version found, so not applying build specifications recursively: %s"
% build_specs)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
# Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line
# (and use the prepended path so that they are found first).
# easyconfig files for dependencies are also generated but not included, they will be resolved via --robot
# either from existing easyconfigs or, if that fails, from easyconfigs in the appended path
tc_name = orig_ec['ec']['toolchain']['name']
new_ec_file = None
verification_build_specs = copy.copy(build_specs)
if orig_ec['spec'] in listed_ec_paths:
if modifying_toolchains:
if tc_name in src_to_dst_tc_mapping:
new_ec_file = map_easyconfig_to_target_tc_hierarchy(
orig_ec['spec'], src_to_dst_tc_mapping,
tweaked_ecs_path)
# Need to update the toolchain in the build_specs to match the toolchain mapping
keys = verification_build_specs.keys()
if 'toolchain_name' in keys:
verification_build_specs[
'toolchain_name'] = src_to_dst_tc_mapping[tc_name][
'name']
if 'toolchain_version' in keys:
verification_build_specs[
'toolchain_version'] = src_to_dst_tc_mapping[
tc_name]['version']
if 'toolchain' in keys:
verification_build_specs[
'toolchain'] = src_to_dst_tc_mapping[tc_name]
else:
new_ec_file = tweak_one(orig_ec['spec'],
None,
build_specs,
targetdir=tweaked_ecs_path)
if new_ec_file:
new_ecs = process_easyconfig(
new_ec_file, build_specs=verification_build_specs)
tweaked_easyconfigs.extend(new_ecs)
else:
# Place all tweaked dependency easyconfigs in the directory appended to the robot path
if modifying_toolchains:
if tc_name in src_to_dst_tc_mapping:
new_ec_file = map_easyconfig_to_target_tc_hierarchy(
orig_ec['spec'],
src_to_dst_tc_mapping,
targetdir=tweaked_ecs_deps_path)
else:
new_ec_file = tweak_one(orig_ec['spec'],
None,
build_specs,
targetdir=tweaked_ecs_deps_path)
return tweaked_easyconfigs
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
}
init_config(args=['--job-backend=PbsPython'], build_options=build_options)
ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False)
self.assertEqual(len(jobs), 8)
regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'")
self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script))
ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), retain_all_deps=True)
jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False)
# make sure command is correct, and that --hidden is there when it needs to be
for i, ec in enumerate(ordered_ecs):
if ec['hidden']:
regex = re.compile("eb %s.* --hidden" % ec['spec'])
else:
regex = re.compile("eb %s" % ec['spec'])
self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script))
for job in jobs:
self.assertEqual(job.cores, build_options['job_cores'])
# no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps)
self.assertEqual(len(jobs[0].deps), 0)
self.assertEqual(len(jobs[1].deps), 0)
# only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out)
self.assertTrue('toy-0.0-deps.eb' in jobs[2].script)
self.assertEqual(len(jobs[2].deps), 1)
self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script)
# dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
self.assertEqual(len(jobs[3].deps), 2)
regex = re.compile('toy-0.0-deps.eb\s* --hidden')
self.assertTrue(regex.search(jobs[3].deps[0].script))
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
def test_resolve_dependencies(self):
""" Test with some basic testcases (also check if he can find dependencies inside the given directory """
easyconfig = {
"spec": "_",
"full_mod_name": "name/version",
"short_mod_name": "name/version",
"dependencies": [],
}
build_options = {"allow_modules_tool_mismatch": True, "robot_path": None, "validate": False}
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig)])
self.assertEqual([easyconfig], res)
easyconfig_dep = {
"ec": {
"name": "foo",
"version": "1.2.3",
"versionsuffix": "",
"toolchain": {"name": "dummy", "version": "dummy"},
},
"spec": "_",
"short_mod_name": "foo/1.2.3",
"full_mod_name": "foo/1.2.3",
"dependencies": [
{
"name": "gzip",
"version": "1.4",
"versionsuffix": "",
"toolchain": {"name": "dummy", "version": "dummy"},
"dummy": True,
"hidden": False,
}
],
"parsed": True,
}
build_options.update({"robot": True, "robot_path": self.base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# dependency should be found, order should be correct
self.assertEqual(len(res), 2)
self.assertEqual("gzip/1.4", res[0]["full_mod_name"])
self.assertEqual("foo/1.2.3", res[-1]["full_mod_name"])
# hidden dependencies are found too, but only retained if they're not available (or forced to be retained
hidden_dep = {
"name": "toy",
"version": "0.0",
"versionsuffix": "-deps",
"toolchain": {"name": "dummy", "version": "dummy"},
"dummy": True,
"hidden": True,
}
easyconfig_moredeps = deepcopy(easyconfig_dep)
easyconfig_moredeps["dependencies"].append(hidden_dep)
easyconfig_moredeps["hiddendependencies"] = [hidden_dep]
# toy/.0.0-deps is available and thus should be omitted
res = resolve_dependencies([deepcopy(easyconfig_moredeps)])
self.assertEqual(len(res), 2)
full_mod_names = [ec["full_mod_name"] for ec in res]
self.assertFalse("toy/.0.0-deps" in full_mod_names)
res = resolve_dependencies([deepcopy(easyconfig_moredeps)], retain_all_deps=True)
self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1)
self.assertEqual("gzip/1.4", res[0]["full_mod_name"])
self.assertEqual("foo/1.2.3", res[-1]["full_mod_name"])
full_mod_names = [ec["full_mod_name"] for ec in res]
self.assertTrue("toy/.0.0-deps" in full_mod_names)
self.assertTrue("ictce/4.1.13" in full_mod_names)
# here we have included a dependency in the easyconfig list
easyconfig["full_mod_name"] = "gzip/1.4"
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
build_options.update({"robot_path": None})
init_config(build_options=build_options)
res = resolve_dependencies(ecs)
# all dependencies should be resolved
self.assertEqual(0, sum(len(ec["dependencies"]) for ec in res))
# this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled
ecs = [deepcopy(easyconfig_dep)]
msg = "Irresolvable dependencies encountered"
self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs)
# test if dependencies of an automatically found file are also loaded
easyconfig_dep["dependencies"] = [
{
"name": "gzip",
"version": "1.4",
"versionsuffix": "",
"toolchain": {"name": "GCC", "version": "4.6.3"},
"dummy": True,
"hidden": False,
}
]
ecs = [deepcopy(easyconfig_dep)]
build_options.update({"robot_path": self.base_easyconfig_dir})
init_config(build_options=build_options)
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# GCC should be first (required by gzip dependency)
self.assertEqual("GCC/4.6.3", res[0]["full_mod_name"])
self.assertEqual("foo/1.2.3", res[-1]["full_mod_name"])
# make sure that only missing stuff is built, and that available modules are not rebuilt
# monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present
MockModule.avail_modules = [
"GCC/4.7.2",
"OpenMPI/1.6.4-GCC-4.7.2",
"OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2",
"FFTW/3.3.3-gompi-1.4.10",
"ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2",
]
easyconfig_dep["dependencies"] = [
{
"name": "goolf",
"version": "1.4.10",
"versionsuffix": "",
"toolchain": {"name": "dummy", "version": "dummy"},
"dummy": True,
"hidden": False,
}
]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies(ecs)
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 2)
# goolf should be first, the software itself second
self.assertEqual("goolf/1.4.10", res[0]["full_mod_name"])
self.assertEqual("foo/1.2.3", res[1]["full_mod_name"])
# force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt
build_options.update({"force": True})
init_config(build_options=build_options)
easyconfig["full_mod_name"] = "this/is/already/there"
MockModule.avail_modules.append("this/is/already/there")
ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
res = resolve_dependencies(ecs)
# there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available)
self.assertEqual(len(res), 3)
# goolf should be first, the software itself second
self.assertEqual("this/is/already/there", res[0]["full_mod_name"])
self.assertEqual("goolf/1.4.10", res[1]["full_mod_name"])
self.assertEqual("foo/1.2.3", res[2]["full_mod_name"])
# build that are listed but already have a module available are not retained without force
build_options.update({"force": False})
init_config(build_options=build_options)
newecs = skip_available(ecs) # skip available builds since force is not enabled
res = resolve_dependencies(newecs)
self.assertEqual(len(res), 2)
self.assertEqual("goolf/1.4.10", res[0]["full_mod_name"])
self.assertEqual("foo/1.2.3", res[1]["full_mod_name"])
# with retain_all_deps enabled, all dependencies ae retained
build_options.update({"retain_all_deps": True})
init_config(build_options=build_options)
ecs = [deepcopy(easyconfig_dep)]
newecs = skip_available(ecs) # skip available builds since force is not enabled
res = resolve_dependencies(newecs)
self.assertEqual(len(res), 9)
self.assertEqual("GCC/4.7.2", res[0]["full_mod_name"])
self.assertEqual("goolf/1.4.10", res[-2]["full_mod_name"])
self.assertEqual("foo/1.2.3", res[-1]["full_mod_name"])
build_options.update({"retain_all_deps": False})
init_config(build_options=build_options)
# provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up
MockModule.avail_modules = ["GCC/4.7.2", "OpenMPI/1.6.4-GCC-4.7.2", "gompi/1.4.10", "FFTW/3.3.3-gompi-1.4.10"]
easyconfig_dep["dependencies"] = [
{
"name": "goolf",
"version": "1.4.10",
"versionsuffix": "",
"toolchain": {"name": "dummy", "version": "dummy"},
"dummy": True,
"hidden": False,
}
]
ecs = [deepcopy(easyconfig_dep)]
res = resolve_dependencies([deepcopy(easyconfig_dep)])
# there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep
self.assertEqual(len(res), 4)
# goolf should be first, the software itself second
self.assertEqual("OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2", res[0]["full_mod_name"])
self.assertEqual("ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2", res[1]["full_mod_name"])
self.assertEqual("goolf/1.4.10", res[2]["full_mod_name"])
self.assertEqual("foo/1.2.3", res[3]["full_mod_name"])
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs)
easyconfigs = skip_available(easyconfigs)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" % output_dir)
return True # success
def session_state():
"""Get session state: timestamp, dump of environment, system info."""
def regtest(easyconfig_paths, modtool, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
:param easyconfig_paths: path of easyconfigs to run regtest on
:param modtool: ModulesTool instance to use
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(
aggregate_regtest,
"%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" %
(aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place all log files and the test output as xml
regtest_output_dir = build_option('regtest_output_dir')
testoutput = build_option('testoutput')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif testoutput is not None:
output_dir = os.path.abspath(testoutput)
else:
# default: current dir + easybuild-test-[timestamp]
dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
output_dir = os.path.join(cur_dir, dirname)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(
path, ignore_dirs=build_option('ignore_dirs'))
else:
raise EasyBuildError("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(
process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError as err:
test_results.append((ecfile, 'parsing_easyconfigs',
'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug(
"Skipping easyconfigs from %s that already have a module available..."
% easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {
'cmd': cmd
}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" %
output_dir)
return True # success
def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
"""Tweak list of easyconfigs according to provided build specifications."""
tweaked_ecs_path, tweaked_ecs_deps_path = None, None
if targetdirs is not None:
tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
# make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
toolchains = nub(
['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
if len(toolchains) > 1:
raise EasyBuildError(
"Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s",
toolchains)
if 'name' in build_specs or 'version' in build_specs:
# no recursion if software name/version build specification are included
# in that case, do not construct full dependency graph
orig_ecs = easyconfigs
_log.debug(
"Software name/version found, so not applying build specifications recursively: %s"
% build_specs)
else:
# build specifications should be applied to the whole dependency graph
# obtain full dependency graph for specified easyconfigs
# easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
_log.debug(
"Applying build specifications recursively (no software name/version found): %s"
% build_specs)
orig_ecs = resolve_dependencies(easyconfigs,
modtool,
retain_all_deps=True)
# keep track of originally listed easyconfigs (via their path)
listed_ec_paths = [ec['spec'] for ec in easyconfigs]
# determine toolchain based on last easyconfigs
if orig_ecs:
toolchain = orig_ecs[-1]['ec']['toolchain']
_log.debug("Filtering using toolchain %s" % toolchain)
# filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies
if toolchain['name'] != DUMMY_TOOLCHAIN_NAME:
while orig_ecs[0]['ec']['toolchain'] != toolchain:
orig_ecs = orig_ecs[1:]
# generate tweaked easyconfigs, and continue with those instead
tweaked_easyconfigs = []
for orig_ec in orig_ecs:
# Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use the
# prepended path so that they are found first).
# easyconfig files for dependencies are also generated but not included, they will be resolved via --robot
# either from existing easyconfigs or, if that fails, from easyconfigs in the appended path
if orig_ec['spec'] in listed_ec_paths:
new_ec_file = tweak_one(orig_ec['spec'],
None,
build_specs,
targetdir=tweaked_ecs_path)
new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs)
tweaked_easyconfigs.extend(new_ecs)
else:
# Place all tweaked dependency easyconfigs in the directory appended to the robot path
new_ec_file = tweak_one(orig_ec['spec'],
None,
build_specs,
targetdir=tweaked_ecs_deps_path)
return tweaked_easyconfigs
