def refresh_dependencies(initial_dependencies, altered_dep):
"""
Refresh derived arguments in a dependency
@param initial_dependencies: initial dependency list
@param altered_dep: the dependency to be refreshed
"""
# indicate whether the toolchain is a 'dummy' toolchain
altered_dep['dummy'] = altered_dep['toolchain'][
'name'] == DUMMY_TOOLCHAIN_NAME
# update short/full module names
altered_dep['short_mod_name'] = ActiveMNS().det_short_module_name(
altered_dep)
altered_dep['full_mod_name'] = ActiveMNS().det_full_module_name(
altered_dep)
# replace the dependency in the list of dependencies
new_dependencies = []
for dep in initial_dependencies:
if dep['name'] == altered_dep['name']:
new_dependencies.append(altered_dep)
else:
new_dependencies.append(dep)
return new_dependencies
def map_easyconfig_to_target_tc_hierarchy(ec_spec,
toolchain_mapping,
targetdir=None):
"""
Take an easyconfig spec, parse it, map it to a target toolchain and dump it out
:param ec_spec: Location of original easyconfig file
:param toolchain_mapping: Mapping between source toolchain and target toolchain
:param targetdir: Directory to dump the modified easyconfig file in
:return: Location of the modified easyconfig file
"""
# Fully parse the original easyconfig
parsed_ec = process_easyconfig(ec_spec, validate=False)[0]
# Replace the toolchain if the mapping exists
tc_name = parsed_ec['ec']['toolchain']['name']
if tc_name in toolchain_mapping:
new_toolchain = toolchain_mapping[tc_name]
_log.debug("Replacing parent toolchain %s with %s",
parsed_ec['ec']['toolchain'], new_toolchain)
parsed_ec['ec']['toolchain'] = new_toolchain
# Replace the toolchains of all the dependencies
for key in DEPENDENCY_PARAMETERS:
# loop over a *copy* of dependency dicts (with resolved templates);
# to update the original dep dict, we need to index with idx into self._config[key][0]...
for idx, dep in enumerate(parsed_ec['ec'][key]):
# reference to original dep dict, this is the one we should be updating
orig_dep = parsed_ec['ec']._config[key][0][idx]
# skip dependencies that are marked as external modules
if dep['external_module']:
continue
dep_tc_name = dep['toolchain']['name']
if dep_tc_name in toolchain_mapping:
orig_dep['toolchain'] = toolchain_mapping[dep_tc_name]
# Replace the binutils version (if necessary)
if 'binutils' in toolchain_mapping and (
dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME):
orig_dep.update(toolchain_mapping['binutils'])
# set module names
orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(
dep)
orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(
dep)
# Determine the name of the modified easyconfig and dump it to target_dir
ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'],
det_full_ec_version(parsed_ec['ec']))
tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(),
ec_filename)
parsed_ec['ec'].dump(tweaked_spec, always_overwrite=False, backup=True)
_log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s",
tweaked_spec)
return tweaked_spec
def test_ec(ecfile, short_modname, mod_subdir, modpath_exts,
init_modpaths):
"""Test whether active module naming scheme returns expected values."""
ec = EasyConfig(os.path.join(ecs_dir, ecfile))
self.assertEqual(ActiveMNS().det_full_module_name(ec),
os.path.join(mod_subdir, short_modname))
self.assertEqual(ActiveMNS().det_short_module_name(ec),
short_modname)
self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir)
self.assertEqual(ActiveMNS().det_modpath_extensions(ec),
modpath_exts)
self.assertEqual(ActiveMNS().det_init_modulepaths(ec),
init_modpaths)
def find_resolved_modules(unprocessed, avail_modules):
"""
Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
"""
ordered_ecs = []
new_avail_modules = avail_modules[:]
new_unprocessed = []
for ec in unprocessed:
new_ec = ec.copy()
deps = []
for dep in new_ec['dependencies']:
if not ActiveMNS().det_full_module_name(dep) in new_avail_modules:
deps.append(dep)
new_ec['dependencies'] = deps
if len(new_ec['dependencies']) == 0:
_log.debug("Adding easyconfig %s to final list" % new_ec['spec'])
ordered_ecs.append(new_ec)
new_avail_modules.append(ec['full_mod_name'])
else:
new_unprocessed.append(new_ec)
return ordered_ecs, new_unprocessed, new_avail_modules
def test_is_short_modname_for(self):
"""Test is_short_modname_for method of module naming schemes."""
test_cases = [
('GCC/4.7.2', 'GCC', True),
('gzip/1.6-gompi-1.4.10', 'gzip', True),
('OpenMPI/1.6.4-GCC-4.7.2-no-OFED', 'OpenMPI', True),
('BLACS/1.1-gompi-1.1.0-no-OFED', 'BLACS', True),
('ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1',
'ScaLAPACK', True),
('netCDF-C++/4.2-goolf-1.4.10', 'netCDF-C++', True),
('gcc/4.7.2', 'GCC', False),
('ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1',
'BLACS', False),
('apps/blacs/1.1', 'BLACS', False),
('lib/math/BLACS-stable/1.1', 'BLACS', False),
]
for modname, softname, res in test_cases:
if res:
errormsg = "%s is recognised as a module for '%s'" % (modname,
softname)
else:
errormsg = "%s is NOT recognised as a module for '%s'" % (
modname, softname)
self.assertEqual(
ActiveMNS().is_short_modname_for(modname, softname), res,
errormsg)
def find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=False):
"""
Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
@param easyconfigs: list of parsed easyconfigs
@param avail_modules: list of available modules
@param retain_all_deps: retain all dependencies, regardless of whether modules are available for them or not
"""
ordered_ecs = []
new_easyconfigs = []
# copy, we don't want to modify the origin list of available modules
avail_modules = avail_modules[:]
_log.debug("Finding resolved modules for %s (available modules: %s)", easyconfigs, avail_modules)
ec_mod_names = [ec['full_mod_name'] for ec in easyconfigs]
for easyconfig in easyconfigs:
new_ec = easyconfig.copy()
deps = []
for dep in new_ec['dependencies']:
dep_mod_name = dep.get('full_mod_name', ActiveMNS().det_full_module_name(dep))
# treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run),
# since no corresponding easyconfig can be found for them
if retain_all_deps and dep.get('external_module', False):
_log.debug("Treating dependency marked as external dependency as resolved: %s", dep_mod_name)
elif retain_all_deps and dep_mod_name not in avail_modules:
# if all dependencies should be retained, include dep unless it has been already
_log.debug("Retaining new dep %s in 'retain all deps' mode", dep_mod_name)
deps.append(dep)
# retain dep if it is (still) in the list of easyconfigs
elif dep_mod_name in ec_mod_names:
_log.debug("Dep %s is (still) in list of easyconfigs, retaining it", dep_mod_name)
deps.append(dep)
# retain dep if corresponding module is not available yet;
# fallback to checking with modtool.exist is required,
# for hidden modules and external modules where module name may be partial
elif dep_mod_name not in avail_modules and not modtool.exist([dep_mod_name], skip_avail=True)[0]:
# no module available (yet) => retain dependency as one to be resolved
_log.debug("No module available for dep %s, retaining it", dep)
deps.append(dep)
# update list of dependencies with only those unresolved
new_ec['dependencies'] = deps
# if all dependencies have been resolved, add module for this easyconfig in the list of available modules
if not new_ec['dependencies']:
_log.debug("Adding easyconfig %s to final list" % new_ec['spec'])
ordered_ecs.append(new_ec)
mod_name = easyconfig['full_mod_name']
avail_modules.append(mod_name)
# remove module name from list, so dependencies can be marked as resolved
ec_mod_names.remove(mod_name)
else:
new_easyconfigs.append(new_ec)
return ordered_ecs, new_easyconfigs, avail_modules
def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', prepare_first=True):
"""
Build easyconfigs in parallel by submitting jobs to a batch-queuing system.
Return list of jobs submitted.
Argument `easyconfigs` is a list of easyconfigs which can be
built: e.g. they have no unresolved dependencies. This function
will build them in parallel by submitting jobs.
:param build_command: build command to use
:param easyconfigs: list of easyconfig files
:param output_dir: output directory
:param prepare_first: prepare by runnning fetch step first for each easyconfig
"""
_log.info("going to build these easyconfigs in parallel: %s", easyconfigs)
active_job_backend = job_backend()
if active_job_backend is None:
raise EasyBuildError("Can not use --job if no job backend is available.")
try:
active_job_backend.init()
except RuntimeError as err:
raise EasyBuildError("connection to server failed (%s: %s), can't submit jobs.", err.__class__.__name__, err)
# dependencies have already been resolved,
# so one can linearly walk over the list and use previous job id's
jobs = []
# keep track of which job builds which module
module_to_job = {}
for easyconfig in easyconfigs:
# this is very important, otherwise we might have race conditions
# e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
# running this step here, prevents this
if prepare_first:
prepare_easyconfig(easyconfig)
# the new job will only depend on already submitted jobs
_log.info("creating job for ec: %s" % easyconfig['ec'])
new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir)
# filter out dependencies marked as external modules
deps = [d for d in easyconfig['ec'].all_dependencies if not d.get('external_module', False)]
dep_mod_names = map(ActiveMNS().det_full_module_name, deps)
job_deps = [module_to_job[dep] for dep in dep_mod_names if dep in module_to_job]
# actually (try to) submit job
active_job_backend.queue(new_job, job_deps)
_log.info("job %s for module %s has been submitted", new_job, new_job.module)
# update dictionary
module_to_job[new_job.module] = new_job
jobs.append(new_job)
active_job_backend.complete()
return jobs
def find_resolved_modules(unprocessed, avail_modules, retain_all_deps=False):
"""
Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
"""
ordered_ecs = []
new_avail_modules = avail_modules[:]
new_unprocessed = []
modtool = modules_tool()
for ec in unprocessed:
new_ec = ec.copy()
deps = []
for dep in new_ec['dependencies']:
full_mod_name = ActiveMNS().det_full_module_name(dep)
dep_resolved = full_mod_name in new_avail_modules
if not retain_all_deps:
# hidden modules need special care, since they may not be included in list of available modules
dep_resolved |= dep['hidden'] and modtool.exist(
[full_mod_name])[0]
if not dep_resolved:
deps.append(dep)
new_ec['dependencies'] = deps
if len(new_ec['dependencies']) == 0:
_log.debug("Adding easyconfig %s to final list" % new_ec['spec'])
ordered_ecs.append(new_ec)
new_avail_modules.append(ec['full_mod_name'])
else:
new_unprocessed.append(new_ec)
return ordered_ecs, new_unprocessed, new_avail_modules
def prepare(self):
"""
Creates the absolute filename for the module.
"""
mod_path_suffix = build_option('suffix_modules_path')
# module file goes in general moduleclass category
self.filename = os.path.join(self.module_path, mod_path_suffix,
self.app.full_mod_name)
# make symlink in moduleclass category
mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.app.cfg)
self.class_mod_files = [
os.path.join(self.module_path, p, self.app.full_mod_name)
for p in mod_symlink_paths
]
# create directories and links
for path in [
os.path.dirname(x)
for x in [self.filename] + self.class_mod_files
]:
mkdir(path, parents=True)
# remove module file if it's there (it'll be recreated), see Application.make_module
if os.path.exists(self.filename):
os.remove(self.filename)
return os.path.join(self.module_path, mod_path_suffix)
def mk_node_name(spec):
if spec.get('external_module', False):
node_name = "%s (EXT)" % spec['full_mod_name']
elif omit_versions:
node_name = spec['name']
else:
node_name = ActiveMNS().det_full_module_name(spec)
return node_name
def __init__(self, easyconfigs):
self.container_base = build_option('container_base')
self.container_build_image = build_option('container_build_image')
self.container_path = container_path()
self.easyconfigs = easyconfigs
self.image_format = build_option('container_image_format')
self.img_name = build_option('container_image_name')
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.mns = ActiveMNS()
self.tmpdir = build_option('container_tmpdir')
def find_resolved_modules(easyconfigs, avail_modules, retain_all_deps=False):
"""
Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
@param easyconfigs: list of parsed easyconfigs
@param avail_modules: list of available modules
@param retain_all_deps: retain all dependencies, regardless of whether modules are available for them or not
"""
ordered_ecs = []
new_easyconfigs = []
modtool = modules_tool()
# copy, we don't want to modify the origin list of available modules
avail_modules = avail_modules[:]
_log.debug("Finding resolved modules for %s (available modules: %s)",
easyconfigs, avail_modules)
for easyconfig in easyconfigs:
new_ec = easyconfig.copy()
deps = []
for dep in new_ec['dependencies']:
full_mod_name = dep.get('full_mod_name',
ActiveMNS().det_full_module_name(dep))
# treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run),
# since no corresponding easyconfig can be found for them
if retain_all_deps and dep.get('external_module', False):
_log.debug(
"Treating dependency marked as external dependency as resolved: %s",
dep)
elif retain_all_deps and full_mod_name not in avail_modules:
# if all dependencies should be retained, include dep unless it has been already
_log.debug("Retaining new dep %s in 'retain all deps' mode",
dep)
deps.append(dep)
elif not module_is_available(full_mod_name, modtool, avail_modules,
dep['hidden']):
# no module available (yet) => retain dependency as one to be resolved
_log.debug("No module available for dep %s, retaining it", dep)
deps.append(dep)
# update list of dependencies with only those unresolved
new_ec['dependencies'] = deps
# if all dependencies have been resolved, add module for this easyconfig in the list of available modules
if not new_ec['dependencies']:
_log.debug("Adding easyconfig %s to final list" % new_ec['spec'])
ordered_ecs.append(new_ec)
avail_modules.append(easyconfig['full_mod_name'])
else:
new_easyconfigs.append(new_ec)
return ordered_ecs, new_easyconfigs, avail_modules
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in [f for f in ec_files if not 'broken' in os.path.basename(f)]:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(ec2mod_map[ec_fn], ActiveMNS().det_full_module_name(ecs[0]['ec']))
def raise_error_missing_deps(missing_deps, extra_msg=None):
"""Raise error to report missing dependencies."""
_log.warning("Missing dependencies (details): %s", missing_deps)
mod_names_eb = ', '.join(EasyBuildMNS().det_full_module_name(dep) for dep in missing_deps)
_log.warning("Missing dependencies (EasyBuild module names): %s", mod_names_eb)
mod_names = ', '.join(ActiveMNS().det_full_module_name(dep) for dep in missing_deps)
error_msg = "Missing dependencies: %s" % mod_names
if extra_msg:
error_msg += ' (%s)' % extra_msg
raise EasyBuildError(error_msg)
def find_resolved_modules(unprocessed, avail_modules, retain_all_deps=False):
"""
Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
"""
ordered_ecs = []
new_avail_modules = avail_modules[:]
new_unprocessed = []
modtool = modules_tool()
for ec in unprocessed:
new_ec = ec.copy()
deps = []
for dep in new_ec['dependencies']:
full_mod_name = dep.get('full_mod_name', None)
if full_mod_name is None:
full_mod_name = ActiveMNS().det_full_module_name(dep)
dep_resolved = full_mod_name in new_avail_modules
if not retain_all_deps:
# hidden modules need special care, since they may not be included in list of available modules
dep_resolved |= dep['hidden'] and modtool.exist(
[full_mod_name])[0]
if not dep_resolved:
# treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run),
# since no corresponding easyconfig can be found for them
if retain_all_deps and dep.get('external_module', False):
_log.debug(
"Treating dependency marked as external dependency as resolved: %s",
dep)
else:
# no module available (yet) => retain dependency as one to be resolved
deps.append(dep)
new_ec['dependencies'] = deps
if len(new_ec['dependencies']) == 0:
_log.debug("Adding easyconfig %s to final list" % new_ec['spec'])
ordered_ecs.append(new_ec)
new_avail_modules.append(ec['full_mod_name'])
else:
new_unprocessed.append(new_ec)
return ordered_ecs, new_unprocessed, new_avail_modules
def test_independence(self):
"""Test independency of toolchain instances."""
# tweaking --optarch is required for Cray toolchains (craypre-<optarch> module must be available)
init_config(build_options={'optarch': 'test'})
tc_cflags = {
'CrayCCE': "-craype-verbose -O2",
'CrayGNU': "-craype-verbose -O2",
'CrayIntel':
"-craype-verbose -O2 -ftz -fp-speculation=safe -fp-model source",
'GCC': "-O2 -test",
'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source",
}
toolchains = [
('CrayCCE', '2015.06-XC'),
('CrayGNU', '2015.06-XC'),
('CrayIntel', '2015.06-XC'),
('GCC', '4.7.2'),
('iccifort', '2011.13.367'),
]
# purposely obtain toolchains several times in a row, value for $CFLAGS should not change
for _ in range(3):
for tcname, tcversion in toolchains:
tc = get_toolchain({
'name': tcname,
'version': tcversion
}, {},
mns=ActiveMNS(),
modtool=self.modtool)
tc.set_options({})
tc.prepare()
expected_cflags = tc_cflags[tcname]
msg = "Expected $CFLAGS found for toolchain %s: %s" % (
tcname, expected_cflags)
self.assertEqual(str(tc.variables['CFLAGS']), expected_cflags,
msg)
self.assertEqual(os.environ['CFLAGS'], expected_cflags, msg)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 0 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = "'%s<0.11.4'" % VSC_INSTALL
if sourcepath:
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# determine PyPI URLs for individual packages
pkg_urls = []
for pkg in EASYBUILD_PACKAGES:
# format of pkg entries in templates: "'<pkg_filename>',"
pkg_filename = templates[pkg][1:-2]
# the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
# which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)
# determine download URL via PyPI's 'simple' API
pkg_simple = None
try:
pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
except (urllib2.URLError, urllib2.HTTPError) as err:
# failing to figure out the package download URl may be OK when source tarballs are provided
if sourcepath:
info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg)
else:
raise err
if pkg_simple:
pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename)
res = pkg_url_part_regex.search(pkg_simple)
if res:
pkg_url_part = res.group(1)
else:
error_msg = "Failed to determine PyPI package URL for %s using pattern '%s': %s\n"
error(error_msg % (pkg, pkg_url_part_regex.pattern, pkg_simple))
pkg_url = 'https://pypi.python.org/' + pkg_url_part
pkg_urls.append(pkg_url)
templates.update({
'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
handle.write(ebfile_txt)
handle.close()
debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure EasyBuild can find EasyBuild-*.eb easyconfig file when it needs to;
# (for example when HierarchicalMNS is used as module naming scheme,
# see https://github.com/easybuilders/easybuild-framework/issues/2393)
eb_args.append('--robot-paths=%s:' % tmpdir)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
if print_debug:
os.environ['EASYBUILD_DEBUG'] = '1'
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def __init__(self, *args, **kwargs):
"""Extra initialization: determine system MPI version, prefix and any associated envvars."""
super(SystemMPI, self).__init__(*args, **kwargs)
mpi_name = self.cfg['name'].lower()
# Determine MPI wrapper path (real path, with resolved symlinks) to ensure it exists
if mpi_name == 'impi':
# For impi the version information is only found in *some* of the wrappers it ships, in particular it is
# not in mpicc
mpi_c_wrapper = 'mpiicc'
path_to_mpi_c_wrapper = which(mpi_c_wrapper)
if not path_to_mpi_c_wrapper:
mpi_c_wrapper = 'mpigcc'
path_to_mpi_c_wrapper = which(mpi_c_wrapper)
if not path_to_mpi_c_wrapper:
raise EasyBuildError(
"Could not find suitable MPI wrapper to extract version for impi"
)
else:
mpi_c_wrapper = 'mpicc'
path_to_mpi_c_wrapper = which(mpi_c_wrapper)
if path_to_mpi_c_wrapper:
path_to_mpi_c_wrapper = resolve_path(path_to_mpi_c_wrapper)
self.log.info(
"Found path to MPI implementation '%s' %s compiler (with symlinks resolved): %s",
mpi_name, mpi_c_wrapper, path_to_mpi_c_wrapper)
else:
raise EasyBuildError("%s not found in $PATH", mpi_c_wrapper)
# Determine MPI version, installation prefix and underlying compiler
if mpi_name in ('openmpi', 'spectrummpi'):
# Spectrum MPI is based on Open MPI so is also covered by this logic
output_of_ompi_info, _ = run_cmd("ompi_info", simple=False)
# Extract the version of the MPI implementation
if mpi_name == 'spectrummpi':
mpi_version_string = 'Spectrum MPI'
else:
mpi_version_string = 'Open MPI'
self.mpi_version = self.extract_ompi_setting(
mpi_version_string, output_of_ompi_info)
# Extract the installation prefix
self.mpi_prefix = self.extract_ompi_setting(
"Prefix", output_of_ompi_info)
# Extract any OpenMPI environment variables in the current environment and ensure they are added to the
# final module
self.mpi_env_vars = dict((key, value)
for key, value in os.environ.iteritems()
if key.startswith("OMPI_"))
# Extract the C compiler used underneath the MPI implementation, check for the definition of OMPI_MPICC
self.mpi_c_compiler = self.extract_ompi_setting(
"C compiler", output_of_ompi_info)
elif mpi_name == 'impi':
# Extract the version of IntelMPI
# The prefix in the the mpiicc (or mpigcc) script can be used to extract the explicit version
contents_of_mpixcc = read_file(path_to_mpi_c_wrapper)
prefix_regex = re.compile(
r'(?<=compilers_and_libraries_)(.*)(?=/linux/mpi)', re.M)
self.mpi_version = None
res = prefix_regex.search(contents_of_mpixcc)
if res:
self.mpi_version = res.group(1)
else:
# old iimpi version
prefix_regex = re.compile(r'^prefix=(.*)$', re.M)
res = prefix_regex.search(contents_of_mpixcc)
if res:
self.mpi_version = res.group(1).split('/')[-1]
if self.mpi_version is None:
raise EasyBuildError("No version found for system Intel MPI")
else:
self.log.info("Found Intel MPI version %s for system MPI" %
self.mpi_version)
# Extract the installation prefix, if I_MPI_ROOT is defined, let's use that
i_mpi_root = os.environ.get('I_MPI_ROOT')
if i_mpi_root:
self.mpi_prefix = i_mpi_root
else:
# Else just go up three directories from where mpiicc is found
# (it's 3 because bin64 is a symlink to intel64/bin and we are assuming 64 bit)
self.mpi_prefix = os.path.dirname(
os.path.dirname(os.path.dirname(path_to_mpi_c_wrapper)))
# Extract any IntelMPI environment variables in the current environment and ensure they are added to the
# final module
self.mpi_env_vars = {}
for key, value in os.environ.iteritems():
i_mpi_key = key.startswith('I_MPI_') or key.startswith(
'MPICH_')
mpi_profile_key = key.startswith('MPI') and key.endswith(
'PROFILE')
if i_mpi_key or mpi_profile_key:
self.mpi_env_vars[key] = value
# Extract the C compiler used underneath Intel MPI
compile_info, exit_code = run_cmd("%s -compile-info" %
mpi_c_wrapper,
simple=False)
if exit_code == 0:
self.mpi_c_compiler = compile_info.split(' ', 1)[0]
else:
raise EasyBuildError(
"Could not determine C compiler underneath Intel MPI, '%s -compiler-info' "
"returned %s", mpi_c_wrapper, compile_info)
else:
raise EasyBuildError("Unrecognised system MPI implementation %s",
mpi_name)
# Ensure install path of system MPI actually exists
if not os.path.exists(self.mpi_prefix):
raise EasyBuildError(
"Path derived for system MPI (%s) does not exist: %s!",
mpi_name, self.mpi_prefix)
self.log.debug(
"Derived version/install prefix for system MPI %s: %s, %s",
mpi_name, self.mpi_version, self.mpi_prefix)
# For the version of the underlying C compiler need to explicitly extract (to be certain)
self.c_compiler_version = extract_compiler_version(self.mpi_c_compiler)
self.log.debug(
"Derived compiler/version for C compiler underneath system MPI %s: %s, %s",
mpi_name, self.mpi_c_compiler, self.c_compiler_version)
# If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it
if self.cfg['version'] == 'system':
self.log.info(
"Found specified version '%s', going with derived MPI version '%s'",
self.cfg['version'], self.mpi_version)
elif self.cfg['version'] == self.mpi_version:
self.log.info("Specified MPI version %s matches found version" %
self.mpi_version)
else:
raise EasyBuildError(
"Specified version (%s) does not match version reported by MPI (%s)",
self.cfg['version'], self.mpi_version)
# fix installdir and module names (may differ because of changes to version)
mns = ActiveMNS()
self.cfg.full_mod_name = mns.det_full_module_name(self.cfg)
self.cfg.short_mod_name = mns.det_short_module_name(self.cfg)
self.cfg.mod_subdir = mns.det_module_subdir(self.cfg)
# keep track of original values, for restoring later
self.orig_version = self.cfg['version']
self.orig_installdir = self.installdir
def test_hierarchical_mns(self):
"""Test hierarchical module naming scheme."""
moduleclasses = [
'base', 'compiler', 'mpi', 'numlib', 'system', 'toolchain'
]
ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
'valid_module_classes': moduleclasses,
}
def test_ec(ecfile, short_modname, mod_subdir, modpath_exts,
init_modpaths):
"""Test whether active module naming scheme returns expected values."""
ec = EasyConfig(os.path.join(ecs_dir, ecfile))
self.assertEqual(ActiveMNS().det_full_module_name(ec),
os.path.join(mod_subdir, short_modname))
self.assertEqual(ActiveMNS().det_short_module_name(ec),
short_modname)
self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir)
self.assertEqual(ActiveMNS().det_modpath_extensions(ec),
modpath_exts)
self.assertEqual(ActiveMNS().det_init_modulepaths(ec),
init_modpaths)
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
# format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions, init_modpaths)
iccver = '2013.5.192-GCC-4.8.3'
impi_ec = 'impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb'
imkl_ec = 'imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb'
test_ecs = {
'GCC-4.7.2.eb':
('GCC/4.7.2', 'Core', ['Compiler/GCC/4.7.2'], ['Core']),
'OpenMPI-1.6.4-GCC-4.7.2.eb':
('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2',
['MPI/GCC/4.7.2/OpenMPI/1.6.4'], ['Core']),
'gzip-1.5-goolf-1.4.10.eb':
('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4', [], ['Core']),
'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core', [], ['Core']),
'icc-2013.5.192-GCC-4.8.3.eb':
('icc/%s' % iccver, 'Core', ['Compiler/intel/%s' % iccver],
['Core']),
'ifort-2013.3.163.eb': ('ifort/2013.3.163', 'Core',
['Compiler/intel/2013.3.163'], ['Core']),
'CUDA-5.5.22-GCC-4.8.2.eb': ('CUDA/5.5.22', 'Compiler/GCC/4.8.2',
['Compiler/GCC-CUDA/4.8.2-5.5.22'
], ['Core']),
impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s' % iccver,
['MPI/intel/%s/impi/4.1.3.049' % iccver], ['Core']),
imkl_ec: ('imkl/11.1.2.144',
'MPI/intel/%s/impi/4.1.3.049' % iccver, [], ['Core']),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile, *mns_vals)
# impi with dummy toolchain, which doesn't make sense in a hierarchical context
ec = EasyConfig(os.path.join(ecs_dir, 'impi-4.1.3.049.eb'))
self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib',
ActiveMNS().det_modpath_extensions, ec)
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
init_config(build_options=build_options)
# format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions)
test_ecs = {
'GCC-4.7.2.eb':
('GCC/4.7.2', 'Core/compiler',
['Compiler/GCC/4.7.2/%s' % c for c in moduleclasses]),
'OpenMPI-1.6.4-GCC-4.7.2.eb':
('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2/mpi',
['MPI/GCC/4.7.2/OpenMPI/1.6.4/%s' % c for c in moduleclasses]),
'gzip-1.5-goolf-1.4.10.eb':
('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4/tools', []),
'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core/toolchain', []),
'icc-2013.5.192-GCC-4.8.3.eb':
('icc/%s' % iccver, 'Core/compiler',
['Compiler/intel/%s/%s' % (iccver, c) for c in moduleclasses]),
'ifort-2013.3.163.eb':
('ifort/2013.3.163', 'Core/compiler',
['Compiler/intel/2013.3.163/%s' % c for c in moduleclasses]),
'CUDA-5.5.22-GCC-4.8.2.eb':
('CUDA/5.5.22', 'Compiler/GCC/4.8.2/system',
['Compiler/GCC-CUDA/4.8.2-5.5.22/%s' % c for c in moduleclasses]),
impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s/mpi' % iccver, [
'MPI/intel/%s/impi/4.1.3.049/%s' % (iccver, c)
for c in moduleclasses
]),
imkl_ec: ('imkl/11.1.2.144',
'MPI/intel/%s/impi/4.1.3.049/numlib' % iccver, []),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile,
*mns_vals,
init_modpaths=['Core/%s' % c for c in moduleclasses])
# impi with dummy toolchain, which doesn't make sense in a hierarchical context
ec = EasyConfig(os.path.join(ecs_dir, 'impi-4.1.3.049.eb'))
self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib',
ActiveMNS().det_modpath_extensions, ec)
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
test_ecs = {
'GCC-4.7.2.eb': ('GCC/4.7.2', '', [], []),
'OpenMPI-1.6.4-GCC-4.7.2.eb':
('OpenMPI/1.6.4-GCC-4.7.2', '', [], []),
'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5-goolf-1.4.10', '', [], []),
'goolf-1.4.10.eb': ('goolf/1.4.10', '', [], []),
'impi-4.1.3.049.eb': ('impi/4.1.3.049', '', [], []),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile, *mns_vals)
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [
os.path.join(subdir, fil)
for (subdir, _, files) in os.walk(ecs_dir) for fil in files
]
ec_files = [fil for fil in ec_files if not "v2.0" in fil
] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(
ec2mod_map[ec_fn],
ActiveMNS().det_full_module_name(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar',
ActiveMNS().det_full_module_name(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
mns_mods = [
'broken_module_naming_scheme', 'test_module_naming_scheme',
'test_module_naming_scheme_more'
]
for test_mns_mod in mns_mods:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
__import__(mns_path, globals(), locals(), [''])
init_config(build_options=build_options)
# verify that key errors in module naming scheme are reported properly
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = 'BrokenModuleNamingScheme'
init_config(build_options=build_options)
err_pattern = 'nosucheasyconfigparameteravailable'
self.assertErrorRegex(
EasyBuildError, err_pattern, EasyConfig,
os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(ec.toolchain.det_short_module_name(), 'goolf/1.4.10')
# test module naming scheme using all available easyconfig parameters
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeMore'
init_config(build_options=build_options)
# note: these checksums will change if another easyconfig parameter is added
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/9e9ab5a1e978f0843b5aedb63ac4f14c51efb859',
'gzip-1.4.eb': 'gzip/8805ec3152d2a4a08b6c06d740c23abe1a4d059f',
'gzip-1.4-GCC-4.6.3.eb':
'gzip/863557cc81811f8c3f4426a4b45aa269fa54130b',
'gzip-1.5-goolf-1.4.10.eb':
'gzip/b63c2b8cc518905473ccda023100b2d3cff52d55',
'gzip-1.5-ictce-4.1.13.eb':
'gzip/3d49f0e112708a95f79ed38b91b506366c0299ab',
'toy-0.0.eb': 'toy/44a206d9e8c14130cc9f79e061468303c6e91b53',
'toy-0.0-multiple.eb':
'toy/44a206d9e8c14130cc9f79e061468303c6e91b53',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
ec2mod_map[
'gzip-1.5-goolf-1.4.10.eb'] = 'gzip/.b63c2b8cc518905473ccda023100b2d3cff52d55'
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'hidden': False,
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {
'name': 'goolf',
'version': '1.4.10'
},
'hidden': True,
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'hidden': False,
}),
]:
# determine full module name
self.assertEqual(ActiveMNS().det_full_module_name(dep_spec),
ec2mod_map[dep_ec])
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'),
hidden=True)
self.assertEqual(ec.full_mod_name,
ec2mod_map['gzip-1.5-goolf-1.4.10.eb'])
self.assertEqual(ec.toolchain.det_short_module_name(),
'goolf/b7515d0efd346970f55e7aa8522e239a70007021')
# restore default module naming scheme, and retest
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def get_toolchain(self, name, version=None):
"""Get a toolchain object instance to test with."""
tc_class, _ = search_toolchain(name)
self.assertEqual(tc_class.NAME, name)
tc = tc_class(version=version, mns=ActiveMNS(), modtool=self.modtool)
return tc
def _to_key(dep):
"""Determine key for specified dependency."""
return ActiveMNS().det_full_module_name(dep)
def mk_dep_mod_name(spec):
return tuple(ActiveMNS().det_full_module_name(spec).split(os.path.sep))
def resolve_dependencies(easyconfigs,
modtool,
retain_all_deps=False,
raise_error_missing_ecs=True):
"""
Work through the list of easyconfigs to determine an optimal order
:param easyconfigs: list of easyconfigs
:param modtool: ModulesTool instance to use
:param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
:param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
avail_modules = modtool.available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if m not in being_installed]
_log.debug('easyconfigs before resolving deps: %s', easyconfigs)
totally_missing, missing_easyconfigs = [], []
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError(
"Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)",
maxloopcnt, easyconfigs, missing_easyconfigs)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(easyconfigs,
avail_modules,
modtool,
retain_all_deps=retain_all_deps)
resolved_ecs, easyconfigs, avail_modules = res
ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs]
for ec in resolved_ecs:
# only add easyconfig if it's not included yet (based on module name)
if not ec['full_mod_name'] in ordered_ec_mod_names:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [
d['full_mod_name'] for ec in easyconfigs
for d in ec['dependencies'] if d.get('external_module', False)
]
if missing_external_modules:
raise EasyBuildError(
"Missing modules for dependencies marked as external modules: %s",
', '.join(missing_external_modules))
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in easyconfigs
]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
full_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
# no easyconfig found + no module available => missing dependency
if not modtool.exist([full_mod_name])[0]:
if cand_dep not in totally_missing:
totally_missing.append(cand_dep)
# no easyconfig found for dependency, but module is available
# => add to list of missing easyconfigs
elif cand_dep not in missing_easyconfigs:
_log.debug(
"Irresolvable dependency found (no easyconfig file): %s",
cand_dep)
missing_easyconfigs.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
# add dummy entry for this dependency, so --dry-run for example can still report the dep
additional.append({
'dependencies': [],
'ec': None,
'full_mod_name': full_mod_name,
'spec': None,
})
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
verify_easyconfig_filename(path,
cand_dep,
parsed_ec=processed_ecs)
for ec in processed_ecs:
if ec not in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
missing_deps = [
dep for x in easyconfigs for dep in x['dependencies']
]
if missing_deps:
raise_error_missing_deps(
missing_deps,
extra_msg="enable dependency resolution via --robot?")
if totally_missing:
raise_error_missing_deps(
totally_missing,
extra_msg="no easyconfig file or existing module found")
if missing_easyconfigs:
if raise_error_missing_ecs:
raise_error_missing_deps(
missing_easyconfigs,
extra_msg="no easyconfig file found in robot search path")
else:
_log.warning("No easyconfig files found for: %s",
missing_easyconfigs)
_log.info("Dependency resolution complete, building as follows: %s",
ordered_ecs)
return ordered_ecs
def mk_node_name(spec):
if omit_versions:
return spec['name']
else:
return ActiveMNS().det_full_module_name(spec)
def __init__(self, *args, **kwargs):
"""Extra initialization: determine system compiler version and prefix."""
super(SystemCompiler, self).__init__(*args, **kwargs)
# Determine compiler path (real path, with resolved symlinks)
compiler_name = self.cfg['name'].lower()
path_to_compiler = which(compiler_name)
if path_to_compiler:
path_to_compiler = os.path.realpath(path_to_compiler)
self.log.info("Found path to compiler '%s' (with symlinks resolved): %s", compiler_name, path_to_compiler)
else:
raise EasyBuildError("%s not found in $PATH", compiler_name)
# Determine compiler version and installation prefix
if compiler_name == 'gcc':
out, _ = run_cmd("gcc --version", simple=False)
self.extract_compiler_version(out)
# strip off 'bin/gcc'
self.compiler_prefix = os.path.dirname(os.path.dirname(path_to_compiler))
elif compiler_name in ['icc', 'ifort']:
out, _ = run_cmd("%s -V" % compiler_name, simple=False)
self.extract_compiler_version(out)
intelvars_fn = path_to_compiler + 'vars.sh'
if os.path.isfile(intelvars_fn):
self.log.debug("Trying to determine compiler install prefix from %s", intelvars_fn)
intelvars_txt = read_file(intelvars_fn)
prod_dir_regex = re.compile(r'^PROD_DIR=(.*)$', re.M)
res = prod_dir_regex.search(intelvars_txt)
if res:
self.compiler_prefix = res.group(1)
else:
raise EasyBuildError("Failed to determine %s installation prefix from %s",
compiler_name, intelvars_fn)
else:
# strip off 'bin/intel*/icc'
self.compiler_prefix = os.path.dirname(os.path.dirname(os.path.dirname(path_to_compiler)))
else:
raise EasyBuildError("Unknown system compiler %s" % self.cfg['name'])
self.log.debug("Derived version/install prefix for system compiler %s: %s, %s",
compiler_name, self.compiler_version, self.compiler_prefix)
# If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it
if self.cfg['version'] == 'system':
self.log.info("Found specified version '%s', going with derived compiler version '%s'",
self.cfg['version'], self.compiler_version)
elif self.cfg['version'] != self.compiler_version:
raise EasyBuildError("Specified version (%s) does not match version reported by compiler (%s)" %
(self.cfg['version'], self.compiler_version))
# fix installdir and module names (may differ because of changes to version)
mns = ActiveMNS()
self.cfg.full_mod_name = mns.det_full_module_name(self.cfg)
self.cfg.short_mod_name = mns.det_short_module_name(self.cfg)
self.cfg.mod_subdir = mns.det_module_subdir(self.cfg)
# keep track of original values, for restoring later
self.orig_version = self.cfg['version']
self.orig_installdir = self.installdir
def __init__(self, *args, **kwargs):
"""Extra initialization: determine system compiler version and prefix."""
super(SystemCompiler, self).__init__(*args, **kwargs)
# Determine compiler path (real path, with resolved symlinks)
compiler_name = self.cfg['name'].lower()
path_to_compiler = which(compiler_name)
if path_to_compiler:
path_to_compiler = os.path.realpath(path_to_compiler)
self.log.info(
"Found path to compiler '%s' (with symlinks resolved): %s",
compiler_name, path_to_compiler)
else:
raise EasyBuildError("%s not found in $PATH", compiler_name)
# Determine compiler version and installation prefix
if compiler_name == 'gcc':
out, _ = run_cmd("gcc --version", simple=False)
self.extract_compiler_version(out)
# strip off 'bin/gcc'
self.compiler_prefix = os.path.dirname(
os.path.dirname(path_to_compiler))
elif compiler_name in ['icc', 'ifort']:
out, _ = run_cmd("%s -V" % compiler_name, simple=False)
self.extract_compiler_version(out)
intelvars_fn = path_to_compiler + 'vars.sh'
if os.path.isfile(intelvars_fn):
self.log.debug(
"Trying to determine compiler install prefix from %s",
intelvars_fn)
intelvars_txt = read_file(intelvars_fn)
prod_dir_regex = re.compile(r'^PROD_DIR=(.*)$', re.M)
res = prod_dir_regex.search(intelvars_txt)
if res:
self.compiler_prefix = res.group(1)
else:
raise EasyBuildError(
"Failed to determine %s installation prefix from %s",
compiler_name, intelvars_fn)
else:
# strip off 'bin/intel*/icc'
self.compiler_prefix = os.path.dirname(
os.path.dirname(os.path.dirname(path_to_compiler)))
else:
raise EasyBuildError("Unknown system compiler %s" %
self.cfg['name'])
self.log.debug(
"Derived version/install prefix for system compiler %s: %s, %s",
compiler_name, self.compiler_version, self.compiler_prefix)
# If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it
if self.cfg['version'] == 'system':
self.log.info(
"Found specified version '%s', going with derived compiler version '%s'",
self.cfg['version'], self.compiler_version)
elif self.cfg['version'] != self.compiler_version:
raise EasyBuildError(
"Specified version (%s) does not match version reported by compiler (%s)"
% (self.cfg['version'], self.compiler_version))
# fix installdir and module names (may differ because of changes to version)
mns = ActiveMNS()
self.cfg.full_mod_name = mns.det_full_module_name(self.cfg)
self.cfg.short_mod_name = mns.det_short_module_name(self.cfg)
self.cfg.mod_subdir = mns.det_module_subdir(self.cfg)
# keep track of original values, for restoring later
self.orig_version = self.cfg['version']
self.orig_installdir = self.installdir
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('unprocessed before resolving deps: %s' % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
tup = (maxloopcnt, unprocessed, irresolvable)
msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(unprocessed,
avail_modules,
retain_all_deps=retain_all_deps)
more_ecs, unprocessed, avail_modules = res
for ec in more_ecs:
if not ec['full_mod_name'] in [
x['full_mod_name'] for x in ordered_ecs
]:
ordered_ecs.append(ec)
# robot: look for existing dependencies, add them
if robot and unprocessed:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in unprocessed
]
additional = []
for entry in unprocessed:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" %
cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [
spec['ec'].full_mod_name for spec in processed_ecs
]
dep_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
if not dep_mod_name in mods:
tup = (path, dep_mod_name, mods)
_log.error(
"easyconfig file %s does not contain module %s (mods: %s)"
% tup)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [
dep for x in unprocessed for dep in x['dependencies']
]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [
EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" %
', '.join(irresolvable_mods_eb))
irresolvable_mods = [
ActiveMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.error('Irresolvable dependencies encountered: %s' %
', '.join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows:\n%s" %
ordered_ecs)
return ordered_ecs
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [
os.path.join(subdir, fil)
for (subdir, _, files) in os.walk(ecs_dir) for fil in files
]
# TODO FIXME: drop this once 2.0/.yeb support works
ec_files = [
fil for fil in ec_files if not ('v2.0/' in fil or 'yeb/' in fil)
]
build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in [
f for f in ec_files if not 'broken' in os.path.basename(f)
]:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(
ec2mod_map[ec_fn],
ActiveMNS().det_full_module_name(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar',
ActiveMNS().det_full_module_name(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
mns_mods = [
'broken_module_naming_scheme', 'test_module_naming_scheme',
'test_module_naming_scheme_more'
]
for test_mns_mod in mns_mods:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
__import__(mns_path, globals(), locals(), [''])
init_config(build_options=build_options)
# verify that key errors in module naming scheme are reported properly
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = 'BrokenModuleNamingScheme'
init_config(build_options=build_options)
err_pattern = 'nosucheasyconfigparameteravailable'
self.assertErrorRegex(
EasyBuildError, err_pattern, EasyConfig,
os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(ec.toolchain.det_short_module_name(), 'goolf/1.4.10')
# test module naming scheme using all available easyconfig parameters
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeMore'
init_config(build_options=build_options)
# note: these checksums will change if another easyconfig parameter is added
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/9e9ab5a1e978f0843b5aedb63ac4f14c51efb859',
'gzip-1.4.eb': 'gzip/53d5c13e85cb6945bd43a58d1c8d4a4c02f3462d',
'gzip-1.4-GCC-4.6.3.eb':
'gzip/585eba598f33c64ef01c6fa47af0fc37f3751311',
'gzip-1.5-goolf-1.4.10.eb':
'gzip/fceb41e04c26b540b7276c4246d1ecdd1e8251c9',
'gzip-1.5-ictce-4.1.13.eb':
'gzip/ae16b3a0a330d4323987b360c0d024f244ac4498',
'toy-0.0.eb': 'toy/44a206d9e8c14130cc9f79e061468303c6e91b53',
'toy-0.0-multiple.eb':
'toy/44a206d9e8c14130cc9f79e061468303c6e91b53',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
ec2mod_map[
'gzip-1.5-goolf-1.4.10.eb'] = 'gzip/.fceb41e04c26b540b7276c4246d1ecdd1e8251c9'
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'hidden': False,
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {
'name': 'goolf',
'version': '1.4.10'
},
'hidden': True,
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'hidden': False,
}),
]:
# determine full module name
self.assertEqual(ActiveMNS().det_full_module_name(dep_spec),
ec2mod_map[dep_ec])
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'),
hidden=True)
self.assertEqual(ec.full_mod_name,
ec2mod_map['gzip-1.5-goolf-1.4.10.eb'])
self.assertEqual(ec.toolchain.det_short_module_name(),
'goolf/a86eb41d8f9c1d6f2d3d61cdb8f420cc2a21cada')
# restore default module naming scheme, and retest
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def make_module_step(self, fake=False):
"""Install .modulerc file."""
modfile_path = self.module_generator.get_module_filepath(fake=fake)
modulerc = os.path.join(os.path.dirname(modfile_path),
self.module_generator.DOT_MODULERC)
deps = self.cfg['dependencies']
if len(deps) != 1:
raise EasyBuildError(
"There should be exactly one dependency specified, found %d",
len(deps))
# names should match
if self.name != deps[0]['name']:
raise EasyBuildError(
"Name does not match dependency name: %s vs %s", self.name,
deps[0]['name'])
# ensure version to alias to is a prefix of the version of the dependency
if not deps[0]['version'].startswith(
self.version) and not self.version == "default":
raise EasyBuildError(
"Version is not 'default' and not a prefix of dependency version: %s vs %s",
self.version, deps[0]['version'])
alias_modname = deps[0]['short_mod_name']
self.log.info("Adding module version alias for %s to %s",
alias_modname, modulerc)
# add symlink to wrapped module file when generating .modulerc in temporary directory (done during sanity check)
# this is strictly required for Lmod 6.x, for which .modulerc and wrapped module file must be in same location
if fake:
wrapped_mod_path = self.modules_tool.modulefile_path(alias_modname)
wrapped_mod_filename = os.path.basename(wrapped_mod_path)
target = os.path.join(os.path.dirname(modulerc),
wrapped_mod_filename)
mkdir(os.path.dirname(target), parents=True)
symlink(wrapped_mod_path, target)
module_version_specs = {
'modname': alias_modname,
'sym_version': self.version,
'version': deps[0]['version'],
}
self.module_generator.modulerc(module_version=module_version_specs,
filepath=modulerc)
if not fake:
print_msg("updated .modulerc file at %s" % modulerc, log=self.log)
# symlink .modulerc in other locations (unless they're already linked)
mod_symlink_dirs = ActiveMNS().det_module_symlink_paths(self.cfg)
mod_subdir = os.path.dirname(ActiveMNS().det_full_module_name(
self.cfg))
mod_install_path = install_path('mod')
modulerc_filename = os.path.basename(modulerc)
for mod_symlink_dir in mod_symlink_dirs:
modulerc_symlink = os.path.join(mod_install_path,
mod_symlink_dir, mod_subdir,
modulerc_filename)
if os.path.islink(modulerc_symlink):
if resolve_path(modulerc_symlink) == resolve_path(
modulerc):
print_msg("symlink %s to %s already exists",
modulerc_symlink, modulerc)
else:
raise EasyBuildError(
"%s exists but is not a symlink to %s",
modulerc_symlink, modulerc)
else:
# Make sure folder exists
mkdir(os.path.dirname(modulerc_symlink), parents=True)
symlink(modulerc, modulerc_symlink)
print_msg("created symlink %s to .modulerc file at %s",
modulerc_symlink,
modulerc,
log=self.log)
modpath = self.module_generator.get_modules_path(fake=fake)
self.invalidate_module_caches(modpath)
return modpath
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info(
"+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n"
% install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 1 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = 'vsc-install'
if sourcepath:
vsc_install_tarball_paths = glob.glob(
os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (
sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
templates.update({
'source_urls':
'\n'.join([
"'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg)
for pkg in EASYBUILD_PACKAGES
]),
'sources':
"%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s"
% templates,
'pythonpath':
distribute_egg_dir,
})
handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
handle.close()
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [
x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0
]
os.environ['PATH'] = os.pathsep.join(
[os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'),
build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error(
"EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug"
% eb_mod_path)