def _simulated_load_dependency_module(self, name, version, metadata, verbose=False):
"""
Set environment variables picked up by utility functions for dependencies specified as external modules.
@param name: software name
@param version: software version
@param metadata: dictionary with software metadata ('prefix' for software installation prefix)
"""
self.log.debug("Defining $EB* environment variables for software named %s", name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
# the prefix can be specified in a number of ways
# * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath>
# * filepath (assumed if environment variable is not defined)
parts = prefix.split(os.path.sep)
env_var = parts[0]
if env_var in os.environ:
prefix = os.environ[env_var]
rel_path = os.path.sep.join(parts[1:])
if rel_path:
prefix = os.path.join(prefix, rel_path, '')
self.log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s",
name, env_var, rel_path, prefix)
else:
self.log.debug("Using specified path as prefix for software named %s: %s", name, prefix)
setvar(get_software_root_env_var_name(name), prefix, verbose=verbose)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def _simulated_load_dependency_module(self, name, version, metadata, verbose=False):
"""
Set environment variables picked up by utility functions for dependencies specified as external modules.
:param name: software name
:param version: software version
:param metadata: dictionary with software metadata ('prefix' for software installation prefix)
"""
self.log.debug("Defining $EB* environment variables for software named %s", name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
# the prefix can be specified in a number of ways
# * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath>
# * filepath (assumed if environment variable is not defined)
parts = prefix.split(os.path.sep)
env_var = parts[0]
if env_var in os.environ:
prefix = os.environ[env_var]
rel_path = os.path.sep.join(parts[1:])
if rel_path:
prefix = os.path.join(prefix, rel_path, '')
self.log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s",
name, env_var, rel_path, prefix)
else:
self.log.debug("Using specified path as prefix for software named %s: %s", name, prefix)
setvar(get_software_root_env_var_name(name), prefix, verbose=verbose)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def _prepare_dependency_external_module(self, dep):
"""Set environment variables picked up by utility functions for dependencies specified as external modules."""
mod_name = dep['full_mod_name']
metadata = dep['external_module_metadata']
self.log.debug("Defining $EB* environment variables for external module %s", mod_name)
names = metadata.get('name', [])
versions = metadata.get('version', [None]*len(names))
self.log.debug("Metadata for external module %s: %s", mod_name, metadata)
for name, version in zip(names, versions):
self.log.debug("Defining $EB* environment variables for external module %s under name %s", mod_name, name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
if prefix in os.environ:
val = os.environ[prefix]
self.log.debug("Using value of $%s as prefix for external module %s: %s", prefix, mod_name, val)
else:
val = prefix
self.log.debug("Using specified prefix for external module %s: %s", mod_name, val)
setvar(get_software_root_env_var_name(name), val)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name), version)
def env_vars_external_module(name, version, metadata):
"""
Determine $EBROOT* and/or $EBVERSION* environment variables that can be set for external module,
based on the provided name, version and metadata.
"""
env_vars = {}
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
# the prefix can be specified in a number of ways
# * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath>
# * filepath (assumed if environment variable is not defined)
parts = prefix.split(os.path.sep)
env_var = parts[0]
if env_var in os.environ:
prefix = os.environ[env_var]
rel_path = os.path.sep.join(parts[1:])
if rel_path:
prefix = os.path.join(prefix, rel_path, '')
_log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s",
name, env_var, rel_path, prefix)
else:
_log.debug("Using specified path as prefix for software named %s: %s", name, prefix)
env_vars[get_software_root_env_var_name(name)] = prefix
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
env_vars[get_software_version_env_var_name(name)] = version
return env_vars
def _simulated_load_dependency_module(self, name, version, metadata, verbose=False):
"""
Set environment variables picked up by utility functions for dependencies specified as external modules.
@param name: software name
@param version: software version
@param metadata: dictionary with software metadata ('prefix' for software installation prefix)
"""
self.log.debug("Defining $EB* environment variables for software named %s", name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
if prefix in os.environ:
val = os.environ[prefix]
self.log.debug("Using value of $%s as prefix for software named %s: %s", prefix, name, val)
else:
val = prefix
self.log.debug("Using specified prefix for software named %s: %s", name, val)
setvar(get_software_root_env_var_name(name), val, verbose=verbose)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def handle_allowed_system_deps(self):
"""Handle allowed system dependencies."""
for (name, version) in self['allow_system_deps']:
env.setvar(get_software_root_env_var_name(name),
name) # root is set to name, not an actual path
env.setvar(
get_software_version_env_var_name(name), version
) # version is expected to be something that makes sense
def _simulated_load_dependency_module(self,
name,
version,
metadata,
verbose=False):
"""
Set environment variables picked up by utility functions for dependencies specified as external modules.
@param name: software name
@param version: software version
@param metadata: dictionary with software metadata ('prefix' for software installation prefix)
"""
self.log.debug(
"Defining $EB* environment variables for software named %s", name)
# define $EBROOT env var for install prefix, picked up by get_software_root
prefix = metadata.get('prefix')
if prefix is not None:
if prefix in os.environ:
val = os.environ[prefix]
self.log.debug(
"Using value of $%s as prefix for software named %s: %s",
prefix, name, val)
else:
val = prefix
self.log.debug(
"Using specified prefix for software named %s: %s", name,
val)
setvar(get_software_root_env_var_name(name), val, verbose=verbose)
# define $EBVERSION env var for software version, picked up by get_software_version
if version is not None:
setvar(get_software_version_env_var_name(name),
version,
verbose=verbose)
def template_module_only_test(self,
easyblock,
name='foo',
version='1.3.2',
extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
tmpdir = tempfile.mkdtemp()
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
toolchain = None
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# easyblocks deriving from IntelBase require a license file to be found for --module-only
bases = list(app_class.__bases__)
for base in copy.copy(bases):
bases.extend(base.__bases__)
if app_class == IntelBase or IntelBase in bases:
os.environ['INTEL_LICENSE_FILE'] = os.path.join(
tmpdir, 'intel.lic')
write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license')
elif app_class == EB_IMOD:
# $JAVA_HOME must be set for IMOD
os.environ['JAVA_HOME'] = tmpdir
elif app_class == PythonBundle:
# $EBROOTPYTHON must be set for PythonBundle easyblock
os.environ[
'EBROOTPYTHON'] = '/fake/install/prefix/Python/2.7.14-foss-2018a'
elif app_class == EB_OpenFOAM:
# proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER
write_file(
os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([
'#%Module',
'setenv EBROOTGCC %s' % tmpdir,
'setenv EBVERSIONGCC 4.9.3',
]))
write_file(
os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'),
'\n'.join([
'#%Module',
'setenv EBROOTOPENMPI %s' % tmpdir,
'setenv EBVERSIONOPENMPI 1.10.2',
]))
write_file(
os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([
'#%Module',
'module load GCC/4.9.3-2.25',
'module load OpenMPI/1.10.2-GCC-4.9.3-2.25',
]))
os.environ['MODULEPATH'] = tmpdir
toolchain = {'name': 'gompi', 'version': '2016a'}
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname,
name=name,
version=version,
extratxt=extra_txt,
toolchain=toolchain)
# take into account that for some easyblock, particular dependencies are hard required early on
# (in prepare_step for exampel);
# we just set the corresponding $EBROOT* environment variables here to fool it...
req_deps = {
# QScintilla easyblock requires that either PyQt or PyQt5 are available as dependency
# (PyQt is easier, since PyQt5 is only supported for sufficiently recent QScintilla versions)
'qscintilla.py': [('PyQt', '4.12')],
# MotionCor2 and Gctf easyblock requires CUDA as dependency
'motioncor2.py': [('CUDA', '10.1.105')],
'gctf.py': [('CUDA', '10.1.105')],
}
easyblock_fn = os.path.basename(easyblock)
for (dep_name, dep_version) in req_deps.get(easyblock_fn, []):
dep_root_envvar = get_software_root_env_var_name(dep_name)
os.environ[dep_root_envvar] = '/value/should/not/matter'
dep_version_envvar = get_software_version_env_var_name(dep_name)
os.environ[dep_version_envvar] = dep_version
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
change_dir(orig_workdir)
if os.path.basename(easyblock) == 'modulerc.py':
# .modulerc must be cleaned up to avoid causing trouble (e.g. "Duplicate version symbol" errors)
modulerc = os.path.join(TMPDIR, 'modules', 'all', name,
'.modulerc')
if os.path.exists(modulerc):
remove_file(modulerc)
modulerc += '.lua'
if os.path.exists(modulerc):
remove_file(modulerc)
else:
modfile = os.path.join(TMPDIR, 'modules', 'all', name, version)
luamodfile = '%s.lua' % modfile
self.assertTrue(
os.path.exists(modfile) or os.path.exists(luamodfile),
"Module file %s or %s was generated" % (modfile, luamodfile))
if os.path.exists(modfile):
modtxt = read_file(modfile)
else:
modtxt = read_file(luamodfile)
none_regex = re.compile('None')
self.assertFalse(none_regex.search(modtxt),
"None not found in module file: %s" % modtxt)
# cleanup
app.close_log()
remove_file(app.logfile)
remove_dir(tmpdir)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def handle_allowed_system_deps(self):
"""Handle allowed system dependencies."""
for (name, version) in self['allow_system_deps']:
env.setvar(get_software_root_env_var_name(name), name) # root is set to name, not an actual path
env.setvar(get_software_version_env_var_name(name), version) # version is expected to be something that makes sense
