def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname,
silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(
tcdep['name'])
self._simulated_load_dependency_module(
tcdep['name'], tcdep['version'],
{'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod,
silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version,
{'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(
os.path.join(install_path('mod'), mod_path_suffix,
modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def _simulated_load_dependency_module(self, name, version, metadata, verbose=False):
"""
Set environment variables picked up by utility functions for dependencies specified as external modules.
:param name: software name
:param version: software version
:param metadata: dictionary with software metadata ('prefix' for software installation prefix)
"""
self.log.debug("Defining $EB* environment variables for software named %s", name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
# the prefix can be specified in a number of ways
# * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath>
# * filepath (assumed if environment variable is not defined)
parts = prefix.split(os.path.sep)
env_var = parts[0]
if env_var in os.environ:
prefix = os.environ[env_var]
rel_path = os.path.sep.join(parts[1:])
if rel_path:
prefix = os.path.join(prefix, rel_path, '')
self.log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s",
name, env_var, rel_path, prefix)
else:
self.log.debug("Using specified path as prefix for software named %s: %s", name, prefix)
setvar(get_software_root_env_var_name(name), prefix, verbose=verbose)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def _prepare_dependency_external_module(self, dep):
"""Set environment variables picked up by utility functions for dependencies specified as external modules."""
mod_name = dep['full_mod_name']
metadata = dep['external_module_metadata']
self.log.debug("Defining $EB* environment variables for external module %s", mod_name)
names = metadata.get('name', [])
versions = metadata.get('version', [None]*len(names))
self.log.debug("Metadata for external module %s: %s", mod_name, metadata)
for name, version in zip(names, versions):
self.log.debug("Defining $EB* environment variables for external module %s under name %s", mod_name, name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
if prefix in os.environ:
val = os.environ[prefix]
self.log.debug("Using value of $%s as prefix for external module %s: %s", prefix, mod_name, val)
else:
val = prefix
self.log.debug("Using specified prefix for external module %s: %s", mod_name, val)
setvar(get_software_root_env_var_name(name), val)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name), version)
def _simulated_load_dependency_module(self, name, version, metadata, verbose=False):
"""
Set environment variables picked up by utility functions for dependencies specified as external modules.
@param name: software name
@param version: software version
@param metadata: dictionary with software metadata ('prefix' for software installation prefix)
"""
self.log.debug("Defining $EB* environment variables for software named %s", name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
# the prefix can be specified in a number of ways
# * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath>
# * filepath (assumed if environment variable is not defined)
parts = prefix.split(os.path.sep)
env_var = parts[0]
if env_var in os.environ:
prefix = os.environ[env_var]
rel_path = os.path.sep.join(parts[1:])
if rel_path:
prefix = os.path.join(prefix, rel_path, '')
self.log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s",
name, env_var, rel_path, prefix)
else:
self.log.debug("Using specified path as prefix for software named %s: %s", name, prefix)
setvar(get_software_root_env_var_name(name), prefix, verbose=verbose)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def env_vars_external_module(name, version, metadata):
"""
Determine $EBROOT* and/or $EBVERSION* environment variables that can be set for external module,
based on the provided name, version and metadata.
"""
env_vars = {}
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
# the prefix can be specified in a number of ways
# * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath>
# * filepath (assumed if environment variable is not defined)
parts = prefix.split(os.path.sep)
env_var = parts[0]
if env_var in os.environ:
prefix = os.environ[env_var]
rel_path = os.path.sep.join(parts[1:])
if rel_path:
prefix = os.path.join(prefix, rel_path, '')
_log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s",
name, env_var, rel_path, prefix)
else:
_log.debug("Using specified path as prefix for software named %s: %s", name, prefix)
env_vars[get_software_root_env_var_name(name)] = prefix
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
env_vars[get_software_version_env_var_name(name)] = version
return env_vars
def _simulated_load_dependency_module(self, name, version, metadata, verbose=False):
"""
Set environment variables picked up by utility functions for dependencies specified as external modules.
@param name: software name
@param version: software version
@param metadata: dictionary with software metadata ('prefix' for software installation prefix)
"""
self.log.debug("Defining $EB* environment variables for software named %s", name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
if prefix in os.environ:
val = os.environ[prefix]
self.log.debug("Using value of $%s as prefix for software named %s: %s", prefix, name, val)
else:
val = prefix
self.log.debug("Using specified prefix for software named %s: %s", name, val)
setvar(get_software_root_env_var_name(name), val, verbose=verbose)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def _load_dependencies_modules(self, silent=False):
"""Load modules for dependencies, and handle special cases like external modules."""
dep_mods = [dep['short_mod_name'] for dep in self.dependencies]
if self.dry_run:
dry_run_msg("\nLoading modules for dependencies...\n", silent=silent)
mods_exist = self.modules_tool.exist(dep_mods)
# load available modules for dependencies, simulate load for others
for dep, dep_mod_exists in zip(self.dependencies, mods_exist):
mod_name = dep['short_mod_name']
if dep_mod_exists:
self.modules_tool.load([mod_name])
dry_run_msg("module load %s" % mod_name, silent=silent)
else:
dry_run_msg("module load %s [SIMULATED]" % mod_name, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
if not dep['external_module']:
deproot = '$%s' % get_software_root_env_var_name(dep['name'])
self._simulated_load_dependency_module(dep['name'], dep['version'], {'prefix': deproot})
else:
# load modules for all dependencies
self.log.debug("Loading modules for dependencies: %s", dep_mods)
self.modules_tool.load(dep_mods)
if self.dependencies:
build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']]
if build_dep_mods:
trace_msg("loading modules for build dependencies:")
for dep_mod in build_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no build dependencies specified)")
run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']]
if run_dep_mods:
trace_msg("loading modules for (runtime) dependencies:")
for dep_mod in run_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no (runtime) dependencies specified)")
# append dependency modules to list of modules
self.modules.extend(dep_mods)
# define $EBROOT* and $EBVERSION* for external modules, if metadata is available
for dep in [d for d in self.dependencies if d['external_module']]:
mod_name = dep['full_mod_name']
metadata = dep['external_module_metadata']
self.log.debug("Metadata for external module %s: %s", mod_name, metadata)
names = metadata.get('name', [])
versions = metadata.get('version', [None] * len(names))
self.log.debug("Defining $EB* environment variables for external module %s using names %s, versions %s",
mod_name, names, versions)
for name, version in zip(names, versions):
self._simulated_load_dependency_module(name, version, metadata, verbose=True)
def _load_dependencies_modules(self, silent=False):
"""Load modules for dependencies, and handle special cases like external modules."""
dep_mods = [dep['short_mod_name'] for dep in self.dependencies]
if self.dry_run:
dry_run_msg("\nLoading modules for dependencies...\n", silent=silent)
mods_exist = self.modules_tool.exist(dep_mods)
# load available modules for dependencies, simulate load for others
for dep, dep_mod_exists in zip(self.dependencies, mods_exist):
mod_name = dep['short_mod_name']
if dep_mod_exists:
self.modules_tool.load([mod_name])
dry_run_msg("module load %s" % mod_name, silent=silent)
else:
dry_run_msg("module load %s [SIMULATED]" % mod_name, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
if not dep['external_module']:
deproot = '$%s' % get_software_root_env_var_name(dep['name'])
self._simulated_load_dependency_module(dep['name'], dep['version'], {'prefix': deproot})
else:
# load modules for all dependencies
self.log.debug("Loading modules for dependencies: %s", dep_mods)
self.modules_tool.load(dep_mods)
if self.dependencies:
build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']]
if build_dep_mods:
trace_msg("loading modules for build dependencies:")
for dep_mod in build_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no build dependencies specified)")
run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']]
if run_dep_mods:
trace_msg("loading modules for (runtime) dependencies:")
for dep_mod in run_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no (runtime) dependencies specified)")
# append dependency modules to list of modules
self.modules.extend(dep_mods)
# define $EBROOT* and $EBVERSION* for external modules, if metadata is available
for dep in [d for d in self.dependencies if d['external_module']]:
mod_name = dep['full_mod_name']
metadata = dep['external_module_metadata']
self.log.debug("Metadata for external module %s: %s", mod_name, metadata)
names = metadata.get('name', [])
versions = metadata.get('version', [None] * len(names))
self.log.debug("Defining $EB* environment variables for external module %s using names %s, versions %s",
mod_name, names, versions)
for name, version in zip(names, versions):
self._simulated_load_dependency_module(name, version, metadata, verbose=True)
def handle_allowed_system_deps(self):
"""Handle allowed system dependencies."""
for (name, version) in self['allow_system_deps']:
env.setvar(get_software_root_env_var_name(name),
name) # root is set to name, not an actual path
env.setvar(
get_software_version_env_var_name(name), version
) # version is expected to be something that makes sense
def check_readiness_step(self):
"""Make sure EasyBuild can be installed with a loaded EasyBuild module."""
env_var_name = get_software_root_env_var_name(self.name)
if env_var_name in os.environ:
os.environ.pop(env_var_name)
self.log.debug("$%s is unset so EasyBuild can be installed with a loaded EasyBuild module" % env_var_name)
else:
self.log.debug("Not unsetting $%s since it's not set" % env_var_name)
super(EB_EasyBuildMeta, self).check_readiness_step()
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(tcdep['name'])
self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version, {'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def _simulated_load_dependency_module(self,
name,
version,
metadata,
verbose=False):
"""
Set environment variables picked up by utility functions for dependencies specified as external modules.
@param name: software name
@param version: software version
@param metadata: dictionary with software metadata ('prefix' for software installation prefix)
"""
self.log.debug(
"Defining $EB* environment variables for software named %s", name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
if prefix in os.environ:
val = os.environ[prefix]
self.log.debug(
"Using value of $%s as prefix for software named %s: %s",
prefix, name, val)
else:
val = prefix
self.log.debug(
"Using specified prefix for software named %s: %s", name,
val)
setvar(get_software_root_env_var_name(name), val, verbose=verbose)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name),
version,
verbose=verbose)
def template_module_only_test(self,
easyblock,
name='foo',
version='1.3.2',
extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
tmpdir = tempfile.mkdtemp()
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
toolchain = None
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# easyblocks deriving from IntelBase require a license file to be found for --module-only
bases = list(app_class.__bases__)
for base in copy.copy(bases):
bases.extend(base.__bases__)
if app_class == IntelBase or IntelBase in bases:
os.environ['INTEL_LICENSE_FILE'] = os.path.join(
tmpdir, 'intel.lic')
write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license')
elif app_class == EB_IMOD:
# $JAVA_HOME must be set for IMOD
os.environ['JAVA_HOME'] = tmpdir
elif app_class == PythonBundle:
# $EBROOTPYTHON must be set for PythonBundle easyblock
os.environ[
'EBROOTPYTHON'] = '/fake/install/prefix/Python/2.7.14-foss-2018a'
elif app_class == EB_OpenFOAM:
# proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER
write_file(
os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([
'#%Module',
'setenv EBROOTGCC %s' % tmpdir,
'setenv EBVERSIONGCC 4.9.3',
]))
write_file(
os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'),
'\n'.join([
'#%Module',
'setenv EBROOTOPENMPI %s' % tmpdir,
'setenv EBVERSIONOPENMPI 1.10.2',
]))
write_file(
os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([
'#%Module',
'module load GCC/4.9.3-2.25',
'module load OpenMPI/1.10.2-GCC-4.9.3-2.25',
]))
os.environ['MODULEPATH'] = tmpdir
toolchain = {'name': 'gompi', 'version': '2016a'}
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname,
name=name,
version=version,
extratxt=extra_txt,
toolchain=toolchain)
# take into account that for some easyblock, particular dependencies are hard required early on
# (in prepare_step for exampel);
# we just set the corresponding $EBROOT* environment variables here to fool it...
req_deps = {
# QScintilla easyblock requires that either PyQt or PyQt5 are available as dependency
# (PyQt is easier, since PyQt5 is only supported for sufficiently recent QScintilla versions)
'qscintilla.py': [('PyQt', '4.12')],
# MotionCor2 and Gctf easyblock requires CUDA as dependency
'motioncor2.py': [('CUDA', '10.1.105')],
'gctf.py': [('CUDA', '10.1.105')],
}
easyblock_fn = os.path.basename(easyblock)
for (dep_name, dep_version) in req_deps.get(easyblock_fn, []):
dep_root_envvar = get_software_root_env_var_name(dep_name)
os.environ[dep_root_envvar] = '/value/should/not/matter'
dep_version_envvar = get_software_version_env_var_name(dep_name)
os.environ[dep_version_envvar] = dep_version
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
change_dir(orig_workdir)
if os.path.basename(easyblock) == 'modulerc.py':
# .modulerc must be cleaned up to avoid causing trouble (e.g. "Duplicate version symbol" errors)
modulerc = os.path.join(TMPDIR, 'modules', 'all', name,
'.modulerc')
if os.path.exists(modulerc):
remove_file(modulerc)
modulerc += '.lua'
if os.path.exists(modulerc):
remove_file(modulerc)
else:
modfile = os.path.join(TMPDIR, 'modules', 'all', name, version)
luamodfile = '%s.lua' % modfile
self.assertTrue(
os.path.exists(modfile) or os.path.exists(luamodfile),
"Module file %s or %s was generated" % (modfile, luamodfile))
if os.path.exists(modfile):
modtxt = read_file(modfile)
else:
modtxt = read_file(luamodfile)
none_regex = re.compile('None')
self.assertFalse(none_regex.search(modtxt),
"None not found in module file: %s" % modtxt)
# cleanup
app.close_log()
remove_file(app.logfile)
remove_dir(tmpdir)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def handle_allowed_system_deps(self):
"""Handle allowed system dependencies."""
for (name, version) in self['allow_system_deps']:
env.setvar(get_software_root_env_var_name(name), name) # root is set to name, not an actual path
env.setvar(get_software_version_env_var_name(name), version) # version is expected to be something that makes sense
