def test_get_easyblock_class(self):
"""Test get_easyblock_class function."""
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.easyblocks.generic.toolchain import Toolchain
from easybuild.easyblocks.toy import EB_toy
for easyblock, easyblock_class in [
('ConfigureMake', ConfigureMake),
('easybuild.easyblocks.generic.configuremake.ConfigureMake', ConfigureMake),
('Toolchain', Toolchain),
('EB_toy', EB_toy),
]:
self.assertEqual(get_easyblock_class(easyblock), easyblock_class)
self.assertEqual(get_easyblock_class(None, name='gzip'), ConfigureMake)
self.assertEqual(get_easyblock_class(None, name='toy'), EB_toy)
def process_easyconfig_file(ec_file):
"""Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested)."""
ectxt = read_file(ec_file)
name, easyblock = fetch_parameters_from_easyconfig(ectxt,
['name', 'easyblock'])
derived_easyblock_class = get_easyblock_class(
easyblock, name=name, error_on_missing_easyblock=False)
fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class)
if ectxt != fixed_ectxt:
if go.options.backup:
try:
backup_ec_file = '%s.bk' % ec_file
i = 1
while os.path.exists(backup_ec_file):
backup_ec_file = '%s.bk%d' % (ec_file, i)
i += 1
os.rename(ec_file, backup_ec_file)
log.info("Backed up %s to %s" % (ec_file, backup_ec_file))
except OSError, err:
raise EasyBuildError(
"Failed to backup %s before rewriting it: %s", ec_file,
err)
write_file(ec_file, fixed_ectxt)
log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt)
log.info("%s: fixed" % ec_file)
def avail_easyconfig_params(self):
"""
Print the available easyconfig parameters, for the given easyblock.
"""
app = get_easyblock_class(self.options.easyblock)
extra = app.extra_options()
mapping = convert_to_help(extra, has_default=False)
if len(extra) > 0:
ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__
extra_names = [x[0] for x in extra]
else:
ebb_msg = ''
extra_names = []
txt = ["Available easyconfig parameters%s" % ebb_msg]
params = [(k, v) for (k, v) in mapping.items() if k.upper() not in ['HIDDEN']]
for key, values in params:
txt.append("%s" % key.upper())
txt.append('-' * len(key))
for name, value in values:
tabs = "\t" * (3 - (len(name) + 1) / 8)
if name in extra_names:
starred = '(*)'
else:
starred = ''
txt.append("%s%s:%s%s" % (name, starred, tabs, value))
txt.append('')
return "\n".join(txt)
def pre_configure_hook(self, *args, **kwargs):
"Modify configopts (here is more efficient than parse_hook since only called once)"
orig_enable_templating = self.cfg.enable_templating
self.cfg.enable_templating = False
modify_all_opts(self.cfg, opts_changes, opts_to_skip=PARSE_OPTS)
ec = self.cfg
# additional changes for CMakeMake EasyBlocks
CMakeMake_configopts_changes = (
' -DZLIB_ROOT=$NIXUSER_PROFILE ' +
' -DOPENGL_INCLUDE_DIR=$NIXUSER_PROFILE/include -DOPENGL_gl_LIBRARY=$NIXUSER_PROFILE/lib/libGL.so '
+ ' -DOPENGL_glu_LIBRARY=$NIXUSER_PROFILE/lib/libGLU.so ' +
' -DJPEG_INCLUDE_DIR=$NIXUSER_PROFILE/include -DJPEG_LIBRARY=$NIXUSER_PROFILE/lib/libjpeg.so '
+
' -DPNG_PNG_INCLUDE_DIR=$NIXUSER_PROFILE/include -DPNG_LIBRARY=$NIXUSER_PROFILE/lib/libpng.so '
+ ' -DPYTHON_EXECUTABLE=$EBROOTPYTHON/bin/python ' +
' -DCURL_LIBRARY=$NIXUSER_PROFILE/lib/libcurl.so -DCURL_INCLUDE_DIR=$NIXUSER_PROFILE/include '
+ ' -DCMAKE_SYSTEM_PREFIX_PATH=$NIXUSER_PROFILE ' +
' -DCMAKE_SKIP_INSTALL_RPATH=ON ')
if ec.easyblock is None or isinstance(ec.easyblock, str):
c = get_easyblock_class(ec.easyblock, name=ec.name)
elif isinstance(ec.easyblock, type):
c = ec.easyblock
if c == CMakeMake or issubclass(c, CMakeMake):
# skip for those
if (ec['name'], ec['version']) in [('ROOT', '5.34.36'),
('mariadb', '10.4.11')]:
pass
else:
update_opts(ec, CMakeMake_configopts_changes, 'configopts',
PREPEND)
self.cfg.enable_templating = orig_enable_templating
def check_sha256_checksums(ecs, whitelist=None):
"""
Check whether all provided (parsed) easyconfigs have SHA256 checksums for sources & patches.
:param whitelist: list of regex patterns on easyconfig filenames; check is skipped for matching easyconfigs
:return: list of strings describing checksum issues (missing checksums, wrong checksum type, etc.)
"""
checksum_issues = []
if whitelist is None:
whitelist = []
for ec in ecs:
# skip whitelisted software
ec_fn = os.path.basename(ec.path)
if any(re.match(regex, ec_fn) for regex in whitelist):
_log.info(
"Skipping SHA256 checksum check for %s because of whitelist (%s)",
ec.path, whitelist)
continue
eb_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
checksum_issues.extend(eb_class(ec).check_checksums())
return checksum_issues
def pre_configure_hook(self, *args, **kwargs):
"Modify configopts (here is more efficient than parse_hook since only called once)"
orig_enable_templating = self.cfg.enable_templating
self.cfg.enable_templating = False
modify_all_opts(self.cfg,
opts_changes,
opts_to_skip=PARSE_OPTS +
['exts_list', 'postinstallcmds', 'modluafooter'])
# additional changes for CMakeMake EasyBlocks
ec = self.cfg
if ec.easyblock is None or isinstance(ec.easyblock, str):
c = get_easyblock_class(ec.easyblock, name=ec.name)
elif isinstance(ec.easyblock, type):
c = ec.easyblock
if c == CMakeMake or issubclass(c, CMakeMake):
# skip for those
if (ec['name'], ec['version']) in [('ROOT', '5.34.36'),
('mariadb', '10.4.11')]:
pass
else:
update_opts(ec, ' -DCMAKE_SKIP_INSTALL_RPATH=ON ', 'configopts',
PREPEND)
# disable XHOST
update_opts(ec, ' -DENABLE_XHOST=OFF ', 'configopts', PREPEND)
# use verbose makefile to get the command lines that are executed
update_opts(ec, ' -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON ', 'configopts',
PREPEND)
# additional changes for MesonNinja EasyBlocks
if (c == MesonNinja or issubclass(c, MesonNinja)) and c != CMakeNinja:
update_opts(ec, False, 'fail_on_missing_ninja_meson_dep', REPLACE)
self.cfg.enable_templating = orig_enable_templating
def process_easyconfig_file(ec_file):
"""Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested)."""
ectxt = read_file(ec_file)
name, easyblock = fetch_parameters_from_easyconfig(ectxt, ['name', 'easyblock'])
derived_easyblock_class = get_easyblock_class(easyblock, name=name, default_fallback=False)
fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class)
if ectxt != fixed_ectxt:
if go.options.backup:
try:
backup_ec_file = '%s.bk' % ec_file
i = 1
while os.path.exists(backup_ec_file):
backup_ec_file = '%s.bk%d' % (ec_file, i)
i += 1
os.rename(ec_file, backup_ec_file)
log.info("Backed up %s to %s" % (ec_file, backup_ec_file))
except OSError, err:
raise EasyBuildError("Failed to backup %s before rewriting it: %s", ec_file, err)
write_file(ec_file, fixed_ectxt)
log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt)
log.info("%s: fixed" % ec_file)
def dump_env_script(easyconfigs):
"""
Dump source scripts that set up build environment for specified easyconfigs.
:param easyconfigs: list of easyconfigs to generate scripts for
"""
ecs_and_script_paths = []
for easyconfig in easyconfigs:
script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0]
ecs_and_script_paths.append((easyconfig['ec'], script_path))
# don't just overwrite existing scripts
existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)]
if existing_scripts:
if build_option('force'):
_log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts))
else:
raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s",
' '.join(existing_scripts))
orig_env = copy.deepcopy(os.environ)
for ec, script_path in ecs_and_script_paths:
# obtain EasyBlock instance
app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
app = app_class(ec)
# mimic dry run, and keep quiet
app.dry_run = app.silent = app.toolchain.dry_run = True
# prepare build environment (in dry run mode)
app.check_readiness_step()
app.prepare_step(start_dir=False)
# compose script
ecfile = os.path.basename(ec.path)
script_lines = [
"#!/bin/bash",
"# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile),
"# usage: source %s" % os.path.basename(script_path),
]
script_lines.extend(['', "# toolchain & dependency modules"])
if app.toolchain.modules:
script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules])
else:
script_lines.append("# (no modules loaded)")
script_lines.extend(['', "# build environment"])
if app.toolchain.vars:
env_vars = sorted(app.toolchain.vars.items())
script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars])
else:
script_lines.append("# (no build environment defined)")
write_file(script_path, '\n'.join(script_lines))
print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False)
restore_env(orig_env)
def dump_env_script(easyconfigs):
"""
Dump source scripts that set up build environment for specified easyconfigs.
:param easyconfigs: list of easyconfigs to generate scripts for
"""
ecs_and_script_paths = []
for easyconfig in easyconfigs:
script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0]
ecs_and_script_paths.append((easyconfig['ec'], script_path))
# don't just overwrite existing scripts
existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)]
if existing_scripts:
if build_option('force'):
_log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts))
else:
raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s",
' '.join(existing_scripts))
orig_env = copy.deepcopy(os.environ)
for ec, script_path in ecs_and_script_paths:
# obtain EasyBlock instance
app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
app = app_class(ec)
# mimic dry run, and keep quiet
app.dry_run = app.silent = app.toolchain.dry_run = True
# prepare build environment (in dry run mode)
app.check_readiness_step()
app.prepare_step(start_dir=False)
# compose script
ecfile = os.path.basename(ec.path)
script_lines = [
"#!/bin/bash",
"# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile),
"# usage: source %s" % os.path.basename(script_path),
]
script_lines.extend(['', "# toolchain & dependency modules"])
if app.toolchain.modules:
script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules])
else:
script_lines.append("# (no modules loaded)")
script_lines.extend(['', "# build environment"])
if app.toolchain.vars:
env_vars = sorted(app.toolchain.vars.items())
script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars])
else:
script_lines.append("# (no build environment defined)")
write_file(script_path, '\n'.join(script_lines))
print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False)
restore_env(orig_env)
def test_get_easyblock_class(self):
"""Test get_easyblock_class function."""
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.easyblocks.generic.toolchain import Toolchain
from easybuild.easyblocks.toy import EB_toy
for easyblock, easyblock_class in [
('ConfigureMake', ConfigureMake),
('easybuild.easyblocks.generic.configuremake.ConfigureMake', ConfigureMake),
('Toolchain', Toolchain),
('EB_toy', EB_toy),
]:
self.assertEqual(get_easyblock_class(easyblock), easyblock_class)
self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None)
self.assertEqual(get_easyblock_class(None, name='toy'), EB_toy)
self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY')
self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None)
def test_get_easyblock_class(self):
"""Test get_easyblock_class function."""
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.easyblocks.generic.toolchain import Toolchain
from easybuild.easyblocks.toy import EB_toy
for easyblock, easyblock_class in [
('ConfigureMake', ConfigureMake),
('easybuild.easyblocks.generic.configuremake.ConfigureMake', ConfigureMake),
('Toolchain', Toolchain),
('EB_toy', EB_toy),
]:
self.assertEqual(get_easyblock_class(easyblock), easyblock_class)
self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None)
self.assertEqual(get_easyblock_class(None, name='toy'), EB_toy)
self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY')
self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None)
def template_module_only_test(self,
easyblock,
name='foo',
version='1.3.2',
extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, name=name, version=version, extratxt=extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
os.chdir(orig_workdir)
modfile = os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.3.2')
luamodfile = '%s.lua' % modfile
self.assertTrue(
os.path.exists(modfile) or os.path.exists(luamodfile),
"Module file %s or %s was generated" % (modfile, luamodfile))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def install_step(self):
"""Install components, if specified."""
comp_cnt = len(self.cfg['components'])
for idx, cfg in enumerate(self.comp_cfgs):
easyblock = cfg.get('easyblock') or self.cfg['default_easyblock']
if easyblock is None:
raise EasyBuildError(
"No easyblock specified for component %s v%s", cfg['name'],
cfg['version'])
elif easyblock == 'Bundle':
raise EasyBuildError(
"The '%s' easyblock can not be used to install components in a bundle",
easyblock)
print_msg("installing bundle component %s v%s (%d/%d)..." %
(cfg['name'], cfg['version'], idx + 1, comp_cnt))
self.log.info("Installing component %s v%s using easyblock %s",
cfg['name'], cfg['version'], easyblock)
comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg)
# correct build/install dirs
comp.builddir = self.builddir
comp.install_subdir, comp.installdir = self.install_subdir, self.installdir
# figure out correct start directory
comp.guess_start_dir()
# need to run fetch_patches to ensure per-component patches are applied
comp.fetch_patches()
# location of first unpacked source is used to determine where to apply patch(es)
comp.src = [{'finalpath': comp.cfg['start_dir']}]
# run relevant steps
for step_name in ['patch', 'configure', 'build', 'install']:
if step_name in cfg['skipsteps']:
comp.log.info("Skipping '%s' step for component %s v%s",
step_name, cfg['name'], cfg['version'])
else:
comp.run_step(
step_name,
[lambda x: getattr(x, '%s_step' % step_name)])
# update environment to ensure stuff provided by former components can be picked up by latter components
# once the installation is finalised, this is handled by the generated module
reqs = comp.make_module_req_guess()
for envvar in reqs:
curr_val = os.getenv(envvar, '')
curr_paths = curr_val.split(os.pathsep)
for subdir in reqs[envvar]:
path = os.path.join(self.installdir, subdir)
if path not in curr_paths:
if curr_val:
new_val = '%s:%s' % (path, curr_val)
else:
new_val = path
env.setvar(envvar, new_val)
def template_init_test(self, easyblock):
"""Test whether all easyconfigs can be initialized."""
def check_extra_options_format(extra_options):
"""Make sure extra_options value is of correct format."""
# EasyBuild v1.x
self.assertTrue(isinstance(extra_options, list))
for extra_option in extra_options:
self.assertTrue(isinstance(extra_option, tuple))
self.assertEqual(len(extra_option), 2)
self.assertTrue(isinstance(extra_option[0], basestring))
self.assertTrue(isinstance(extra_option[1], list))
self.assertEqual(len(extra_option[1]), 3)
# EasyBuild v2.0 (breaks backward compatibility compared to v1.x)
#self.assertTrue(isinstance(extra_options, dict))
#for key in extra_options:
# self.assertTrue(isinstance(extra_options[key], list))
# self.assertTrue(len(extra_options[key]), 3)
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# obtain easyblock class name using regex
f = open(easyblock, "r")
txt = f.read()
f.close()
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
extra_options = app_class.extra_options()
check_extra_options_format(extra_options)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_txt = ''
for (key, val) in extra_options:
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_init_test(self, easyblock):
"""Test whether all easyconfigs can be initialized."""
def check_extra_options_format(extra_options):
"""Make sure extra_options value is of correct format."""
# EasyBuild v1.x
self.assertTrue(isinstance(extra_options, list))
for extra_option in extra_options:
self.assertTrue(isinstance(extra_option, tuple))
self.assertEqual(len(extra_option), 2)
self.assertTrue(isinstance(extra_option[0], basestring))
self.assertTrue(isinstance(extra_option[1], list))
self.assertEqual(len(extra_option[1]), 3)
# EasyBuild v2.0 (breaks backward compatibility compared to v1.x)
#self.assertTrue(isinstance(extra_options, dict))
#for key in extra_options:
# self.assertTrue(isinstance(extra_options[key], list))
# self.assertTrue(len(extra_options[key]), 3)
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# obtain easyblock class name using regex
f = open(easyblock, "r")
txt = f.read()
f.close()
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
extra_options = app_class.extra_options()
check_extra_options_format(extra_options)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_txt = ''
for (key, val) in extra_options:
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def test_make_module_pythonpackage(self):
"""Test make_module_step of PythonPackage easyblock."""
app_class = get_easyblock_class('PythonPackage')
self.writeEC('PythonPackage', name='testpypkg', version='3.14')
app = app_class(EasyConfig(self.eb_file))
# install dir should not be there yet
self.assertFalse(os.path.exists(app.installdir))
# create install dir and populate it with subdirs/files
mkdir(app.installdir, parents=True)
# $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH
write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!')
write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar')
write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo')
pyver = '.'.join(map(str, sys.version_info[:2]))
write_file(os.path.join(app.installdir, 'lib', 'python%s' % pyver, 'site-packages', 'foo.egg'), 'foo egg')
write_file(os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'), 'libfoo: foo')
# create module file
app.make_module_step()
self.assertTrue(TMPDIR in app.installdir)
self.assertTrue(TMPDIR in app.installdir_mod)
modtxt = None
for cand_mod_filename in ['3.14', '3.14.lua']:
full_modpath = os.path.join(app.installdir_mod, 'testpypkg', cand_mod_filename)
if os.path.exists(full_modpath):
modtxt = read_file(full_modpath)
break
self.assertFalse(modtxt is None)
regexs = [
(r'^prepend.path.*\WCPATH\W.*include"?\W*$', True),
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WPATH\W.*bin"?\W*$', True),
(r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$', True),
(r'^prepend.path.*\WPYTHONPATH\W.*lib/python2.[0-9]/site-packages"?\W*$', True),
# lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False),
]
for (pattern, found) in regexs:
regex = re.compile(pattern, re.M)
if found:
assert_msg = "Pattern '%s' found in: %s" % (regex.pattern, modtxt)
else:
assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern, modtxt)
self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def test_make_module_pythonpackage(self):
"""Test make_module_step of PythonPackage easyblock."""
app_class = get_easyblock_class('PythonPackage')
self.writeEC('PythonPackage', name='testpypkg', version='3.14')
app = app_class(EasyConfig(self.eb_file))
# install dir should not be there yet
self.assertFalse(os.path.exists(app.installdir))
# create install dir and populate it with subdirs/files
mkdir(app.installdir, parents=True)
# $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH
write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!')
write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar')
write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo')
write_file(os.path.join(app.installdir, 'lib', 'python2.7', 'site-packages', 'foo.egg'), 'foo egg')
write_file(os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'), 'libfoo: foo')
# create module file
app.make_module_step()
self.assertTrue(TMPDIR in app.installdir)
self.assertTrue(TMPDIR in app.installdir_mod)
modtxt = None
for cand_mod_filename in ['3.14', '3.14.lua']:
full_modpath = os.path.join(app.installdir_mod, 'testpypkg', cand_mod_filename)
if os.path.exists(full_modpath):
modtxt = read_file(full_modpath)
break
self.assertFalse(modtxt is None)
regexs = [
(r'^prepend.path.*\WCPATH\W.*include"?\W*$', True),
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WPATH\W.*bin"?\W*$', True),
(r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$', True),
(r'^prepend.path.*\WPYTHONPATH\W.*lib/python2.7/site-packages"?\W*$', True),
# lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False),
]
for (pattern, found) in regexs:
regex = re.compile(pattern, re.M)
if found:
assert_msg = "Pattern '%s' found in: %s" % (regex.pattern, modtxt)
else:
assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern, modtxt)
self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, name=name, version=version, extratxt=extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
os.chdir(orig_workdir)
modfile = os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.3.2')
luamodfile = '%s.lua' % modfile
self.assertTrue(os.path.exists(modfile) or os.path.exists(luamodfile),
"Module file %s or %s was generated" % (modfile, luamodfile))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def install_step(self):
"""Install components, if specified."""
comp_cnt = len(self.cfg['components'])
for idx, cfg in enumerate(self.comp_cfgs):
easyblock = cfg.get('easyblock') or self.cfg['default_easyblock']
if easyblock is None:
raise EasyBuildError(
"No easyblock specified for component %d v%d", cfg['name'],
cfg['version'])
elif easyblock == 'Bundle':
raise EasyBuildError(
"The '%s' easyblock can not be used to install components in a bundle",
easyblock)
print_msg("installing bundle component %s v%s (%d/%d)..." %
(cfg['name'], cfg['version'], idx + 1, comp_cnt))
self.log.info("Installing component %s v%s using easyblock %s",
cfg['name'], cfg['version'], easyblock)
comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg)
# correct build/install dirs
comp.builddir = self.builddir
comp.install_subdir, comp.installdir = self.install_subdir, self.installdir
# figure out correct start directory
comp.guess_start_dir()
# run relevant steps
comp.patch_step()
comp.configure_step()
comp.build_step()
comp.install_step()
# update environment to ensure stuff provided by former components can be picked up by latter components
# once the installation is finalised, this is handled by the generated module
reqs = comp.make_module_req_guess()
for envvar in reqs:
curr_val = os.getenv(envvar, '')
curr_paths = curr_val.split(os.pathsep)
for subdir in reqs[envvar]:
path = os.path.join(self.installdir, subdir)
if path not in curr_paths:
if curr_val:
new_val = '%s:%s' % (path, curr_val)
else:
new_val = path
env.setvar(envvar, new_val)
def install_step(self):
"""Install components, if specified."""
comp_cnt = len(self.cfg['components'])
for idx, cfg in enumerate(self.comp_cfgs):
easyblock = cfg.get('easyblock') or self.cfg['default_easyblock']
if easyblock is None:
raise EasyBuildError("No easyblock specified for component %s v%s", cfg['name'], cfg['version'])
elif easyblock == 'Bundle':
raise EasyBuildError("The '%s' easyblock can not be used to install components in a bundle", easyblock)
print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx+1, comp_cnt))
self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], easyblock)
comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg)
# correct build/install dirs
comp.builddir = self.builddir
comp.install_subdir, comp.installdir = self.install_subdir, self.installdir
# figure out correct start directory
comp.guess_start_dir()
# need to run fetch_patches to ensure per-component patches are applied
comp.fetch_patches()
# location of first unpacked source is used to determine where to apply patch(es)
comp.src = [{'finalpath': comp.cfg['start_dir']}]
# run relevant steps
for step_name in ['patch', 'configure', 'build', 'install']:
if step_name in cfg['skipsteps']:
comp.log.info("Skipping '%s' step for component %s v%s", step_name, cfg['name'], cfg['version'])
else:
comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)])
# update environment to ensure stuff provided by former components can be picked up by latter components
# once the installation is finalised, this is handled by the generated module
reqs = comp.make_module_req_guess()
for envvar in reqs:
curr_val = os.getenv(envvar, '')
curr_paths = curr_val.split(os.pathsep)
for subdir in reqs[envvar]:
path = os.path.join(self.installdir, subdir)
if path not in curr_paths:
if curr_val:
new_val = '%s:%s' % (path, curr_val)
else:
new_val = path
env.setvar(envvar, new_val)
def avail_easyconfig_params(easyblock, output_format):
"""
Compose overview of available easyconfig parameters, in specified format.
"""
params = copy.deepcopy(DEFAULT_CONFIG)
# include list of extra parameters (if any)
extra_params = {}
app = get_easyblock_class(easyblock, default_fallback=False)
if app is not None:
extra_params = app.extra_options()
params.update(extra_params)
# compose title
title = "Available easyconfig parameters"
if extra_params:
title += " (* indicates specific to the %s easyblock)" % app.__name__
# group parameters by category
grouped_params = OrderedDict()
for category in sorted_categories():
# exclude hidden parameters
if category[1].upper() in [HIDDEN]:
continue
grpname = category[1]
grouped_params[grpname] = {}
for name, (dflt, descr, cat) in params.items():
if cat == category:
if name in extra_params:
# mark easyblock-specific parameters
name = '%s*' % name
grouped_params[grpname].update({name: (descr, dflt)})
if not grouped_params[grpname]:
del grouped_params[grpname]
# compose output, according to specified format (txt, rst, ...)
avail_easyconfig_params_functions = {
FORMAT_RST: avail_easyconfig_params_rst,
FORMAT_TXT: avail_easyconfig_params_txt,
}
return avail_easyconfig_params_functions[output_format](title,
grouped_params)
def dump_env_script(easyconfigs):
"""
Dump source scripts that set up build environment for specified easyconfigs.
:param easyconfigs: list of easyconfigs to generate scripts for
"""
ecs_and_script_paths = []
for easyconfig in easyconfigs:
script_path = '%s.env' % os.path.splitext(
os.path.basename(easyconfig['spec']))[0]
ecs_and_script_paths.append((easyconfig['ec'], script_path))
# don't just overwrite existing scripts
existing_scripts = [
s for (_, s) in ecs_and_script_paths if os.path.exists(s)
]
if existing_scripts:
if build_option('force'):
_log.info("Found existing scripts, overwriting them: %s",
' '.join(existing_scripts))
else:
raise EasyBuildError(
"Script(s) already exists, not overwriting them (unless --force is used): %s",
' '.join(existing_scripts))
orig_env = copy.deepcopy(os.environ)
for ec, script_path in ecs_and_script_paths:
# obtain EasyBlock instance
app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
app = app_class(ec)
# mimic dry run, and keep quiet
app.dry_run = app.silent = app.toolchain.dry_run = True
# prepare build environment (in dry run mode)
app.check_readiness_step()
app.prepare_step(start_dir=False)
# create the environment dump
dump_env_easyblock(app,
orig_env=orig_env,
ec_path=ec.path,
script_path=script_path)
def avail_easyconfig_params(easyblock, output_format):
"""
Compose overview of available easyconfig parameters, in specified format.
"""
params = copy.deepcopy(DEFAULT_CONFIG)
# include list of extra parameters (if any)
extra_params = {}
app = get_easyblock_class(easyblock, default_fallback=False)
if app is not None:
extra_params = app.extra_options()
params.update(extra_params)
# compose title
title = "Available easyconfig parameters"
if extra_params:
title += " (* indicates specific to the %s easyblock)" % app.__name__
# group parameters by category
grouped_params = OrderedDict()
for category in sorted_categories():
# exclude hidden parameters
if category[1].upper() in [HIDDEN]:
continue
grpname = category[1]
grouped_params[grpname] = {}
for name, (dflt, descr, cat) in params.items():
if cat == category:
if name in extra_params:
# mark easyblock-specific parameters
name = '%s*' % name
grouped_params[grpname].update({name: (descr, dflt)})
if not grouped_params[grpname]:
del grouped_params[grpname]
# compose output, according to specified format (txt, rst, ...)
avail_easyconfig_params_functions = {
FORMAT_RST: avail_easyconfig_params_rst,
FORMAT_TXT: avail_easyconfig_params_txt,
}
return avail_easyconfig_params_functions[output_format](title, grouped_params)
def install_step(self):
"""Install components, if specified."""
comp_cnt = len(self.cfg['components'])
for idx, cfg in enumerate(self.comp_cfgs):
easyblock = cfg.get('easyblock') or self.cfg['default_easyblock']
if easyblock is None:
raise EasyBuildError("No easyblock specified for component %d v%d", cfg['name'], cfg['version'])
elif easyblock == 'Bundle':
raise EasyBuildError("The '%s' easyblock can not be used to install components in a bundle", easyblock)
print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx+1, comp_cnt))
self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], easyblock)
comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg)
# correct build/install dirs
comp.builddir = self.builddir
comp.install_subdir, comp.installdir = self.install_subdir, self.installdir
# figure out correct start directory
comp.guess_start_dir()
# run relevant steps
comp.patch_step()
comp.configure_step()
comp.build_step()
comp.install_step()
# update environment to ensure stuff provided by former components can be picked up by latter components
# once the installation is finalised, this is handled by the generated module
reqs = comp.make_module_req_guess()
for envvar in reqs:
curr_val = os.getenv(envvar, '')
curr_paths = curr_val.split(os.pathsep)
for subdir in reqs[envvar]:
path = os.path.join(self.installdir, subdir)
if path not in curr_paths:
if curr_val:
new_val = '%s:%s' % (path, curr_val)
else:
new_val = path
env.setvar(envvar, new_val)
def check_sha256_checksums(ecs, whitelist=None):
"""
Check whether all provided (parsed) easyconfigs have SHA256 checksums for sources & patches.
:param whitelist: list of regex patterns on easyconfig filenames; check is skipped for matching easyconfigs
:return: list of strings describing checksum issues (missing checksums, wrong checksum type, etc.)
"""
checksum_issues = []
if whitelist is None:
whitelist = []
for ec in ecs:
# skip whitelisted software
ec_fn = os.path.basename(ec.path)
if any(re.match(regex, ec_fn) for regex in whitelist):
_log.info("Skipping SHA256 checksum check for %s because of whitelist (%s)", ec.path, whitelist)
continue
eb_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
checksum_issues.extend(eb_class(ec).check_checksums())
return checksum_issues
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name, moduleclass
self.assertEqual(ec['name'], name)
self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
extra_opts = app.extra_options()
for key in extra_opts:
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
ec_dict = ec.parser.get_config_dict()
orig_toolchain = ec_dict['toolchain']
for key in ec_dict:
# skip parameters for which value is equal to default value
orig_val = ec_dict[key]
if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
continue
if key in extra_opts and orig_val == extra_opts[key][0]:
continue
if key not in DEFAULT_CONFIG and key not in extra_opts:
continue
orig_val = resolve_template(ec_dict[key], ec.template_values)
dumped_val = resolve_template(dumped_ec[key], ec.template_values)
# take into account that dumped value for *dependencies may include hard-coded subtoolchains
# if no easyconfig was found for the dependency with the 'parent' toolchain,
# if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
if key in DEPENDENCY_PARAMETERS:
# number of dependencies should remain the same
self.assertEqual(len(orig_val), len(dumped_val))
for orig_dep, dumped_dep in zip(orig_val, dumped_val):
# name/version should always match
self.assertEqual(orig_dep[:2], dumped_dep[:2])
# 3rd value is versionsuffix;
if len(dumped_dep) >= 3:
# if no versionsuffix was specified in original dep spec, then dumped value should be empty string
if len(orig_dep) >= 3:
self.assertEqual(dumped_dep[2], orig_dep[2])
else:
self.assertEqual(dumped_dep[2], '')
# 4th value is toolchain spec
if len(dumped_dep) >= 4:
if len(orig_dep) >= 4:
self.assertEqual(dumped_dep[3], orig_dep[3])
else:
# if a subtoolchain is specifed (only) in the dumped easyconfig,
# it should *not* be the same as the parent toolchain
self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version']))
else:
self.assertEqual(orig_val, dumped_val)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def template_init_test(self, easyblock, name='foo', version='1.3.2'):
"""Test whether all easyblocks can be initialized."""
def check_extra_options_format(extra_options):
"""Make sure extra_options value is of correct format."""
# EasyBuild v2.0: dict with <string> keys and <list> values
self.assertTrue(isinstance(extra_options, dict))
extra_options.items()
extra_options.keys()
extra_options.values()
for key in extra_options.keys():
self.assertTrue(isinstance(extra_options[key], list))
self.assertTrue(len(extra_options[key]), 3)
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# make sure error reporting is done correctly (no more log.error, log.exception)
log_method_regexes = [
re.compile(r"log\.error\("),
re.compile(r"log\.exception\("),
re.compile(r"log\.raiseException\("),
]
for regex in log_method_regexes:
self.assertFalse(
regex.search(txt),
"No match for '%s' in %s" % (regex.pattern, easyblock))
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
extra_options = app_class.extra_options()
check_extra_options_format(extra_options)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_txt = ''
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, name=name, version=version, extratxt=extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# check whether easyblock instance is still using functions from a deprecated location
mod = __import__(app.__module__, [], [], ['easybuild.easyblocks'])
moved_functions = [
'modify_env', 'parse_log_for_error', 'read_environment', 'run_cmd',
'run_cmd_qa'
]
for fn in moved_functions:
if hasattr(mod, fn):
tup = (fn, app.__module__, globals()[fn].__module__)
self.assertTrue(
getattr(mod, fn) is globals()[fn],
"%s in %s is imported from %s" % tup)
renamed_functions = [
('source_paths', 'source_path'),
('get_avail_core_count', 'get_core_count'),
('get_os_type', 'get_kernel_name'),
('det_full_ec_version', 'det_installversion'),
]
for (new_fn, old_fn) in renamed_functions:
self.assertFalse(
hasattr(mod, old_fn),
"%s: %s is replaced by %s" % (app.__module__, old_fn, new_fn))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easconfig matches expected filename '%s'" % (spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock'])
# sanity check for software name
self.assertTrue(ec['name'], name)
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
for key in app.extra_options():
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name, moduleclass
self.assertEqual(ec['name'], name)
self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure that $root is not used, since it is not compatible with module files in Lua syntax
res = re.findall('.*\$root.*', ec.rawtxt, re.M)
error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s"
self.assertFalse(res, error_msg % res)
# make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used
old_urls = [
'github.com/hpcugent/easybuild',
'hpcugent.github.com/easybuild',
'hpcugent.github.io/easybuild',
]
for old_url in old_urls:
self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec))
# make sure binutils is included as a build dep if toolchain is GCCcore
if ec['toolchain']['name'] == 'GCCcore':
# with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball
requires_binutils = ec['easyblock'] not in ['Tarball'] and ec['name'] not in ['Eigen']
# let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this
# exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore)
if ec['toolchain']['version'] == 'system':
binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils']
requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies)
# if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled)
requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components'))
if requires_binutils:
dep_names = [d['name'] for d in ec['builddependencies']]
self.assertTrue('binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names))
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
extra_opts = app.extra_options()
for key in extra_opts:
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
ec_dict = ec.parser.get_config_dict()
orig_toolchain = ec_dict['toolchain']
for key in ec_dict:
# skip parameters for which value is equal to default value
orig_val = ec_dict[key]
if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
continue
if key in extra_opts and orig_val == extra_opts[key][0]:
continue
if key not in DEFAULT_CONFIG and key not in extra_opts:
continue
orig_val = resolve_template(ec_dict[key], ec.template_values)
dumped_val = resolve_template(dumped_ec[key], ec.template_values)
# take into account that dumped value for *dependencies may include hard-coded subtoolchains
# if no easyconfig was found for the dependency with the 'parent' toolchain,
# if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
if key in DEPENDENCY_PARAMETERS:
# number of dependencies should remain the same
self.assertEqual(len(orig_val), len(dumped_val))
for orig_dep, dumped_dep in zip(orig_val, dumped_val):
# name/version should always match
self.assertEqual(orig_dep[:2], dumped_dep[:2])
# 3rd value is versionsuffix;
if len(dumped_dep) >= 3:
# if no versionsuffix was specified in original dep spec, then dumped value should be empty string
if len(orig_dep) >= 3:
self.assertEqual(dumped_dep[2], orig_dep[2])
else:
self.assertEqual(dumped_dep[2], '')
# 4th value is toolchain spec
if len(dumped_dep) >= 4:
if len(orig_dep) >= 4:
self.assertEqual(dumped_dep[3], orig_dep[3])
else:
# if a subtoolchain is specifed (only) in the dumped easyconfig,
# it should *not* be the same as the parent toolchain
self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version']))
else:
self.assertEqual(orig_val, dumped_val)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def template_init_test(self, easyblock):
"""Test whether all easyblocks can be initialized."""
def check_extra_options_format(extra_options):
"""Make sure extra_options value is of correct format."""
# EasyBuild v2.0: dict with <string> keys and <list> values
self.assertTrue(isinstance(extra_options, dict))
extra_options.items()
extra_options.keys()
extra_options.values()
for key in extra_options.keys():
self.assertTrue(isinstance(extra_options[key], list))
self.assertTrue(len(extra_options[key]), 3)
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# make sure error reporting is done correctly (no more log.error, log.exception)
log_method_regexes = [
re.compile(r"log\.error\("),
re.compile(r"log\.exception\("),
re.compile(r"log\.raiseException\("),
]
for regex in log_method_regexes:
self.assertFalse(regex.search(txt), "No match for '%s' in %s" % (regex.pattern, easyblock))
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
extra_options = app_class.extra_options()
check_extra_options_format(extra_options)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_txt = ''
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# check whether easyblock instance is still using functions from a deprecated location
mod = __import__(app.__module__, [], [], ['easybuild.easyblocks'])
moved_functions = ['modify_env', 'parse_log_for_error', 'read_environment', 'run_cmd', 'run_cmd_qa']
for fn in moved_functions:
if hasattr(mod, fn):
tup = (fn, app.__module__, globals()[fn].__module__)
self.assertTrue(getattr(mod, fn) is globals()[fn], "%s in %s is imported from %s" % tup)
renamed_functions = [
('source_paths', 'source_path'),
('get_avail_core_count', 'get_core_count'),
('get_os_type', 'get_kernel_name'),
('det_full_ec_version', 'det_installversion'),
]
for (new_fn, old_fn) in renamed_functions:
self.assertFalse(hasattr(mod, old_fn), "%s: %s is replaced by %s" % (app.__module__, old_fn, new_fn))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
subfolders.remove('.git')
for ec_file in files:
if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]:
log.warning("SKIPPING %s/%s" % (root, ec_file))
continue
ec_file = join(root, ec_file)
ec_file = read(ec_file)
try:
ec = EasyConfig(ec_file)
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec.name)
ec.easyblock = None
# check if an easyblock exists
ebclass = get_easyblock_class(None,
name=ec.name,
error_on_missing_easyblock=False)
if ebclass is not None:
module = ebclass.__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
configs.append(ec)
names.append(ec.name)
except Exception, err:
raise EasyBuildError("faulty easyconfig %s: %s", ec_file, err)
log.info("Found easyconfigs: %s" % [x.name for x in configs])
# sort by name
configs = sorted(configs, key=lambda config: config.name.lower())
firstl = ""
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
else:
self.assertTrue(
False,
"easyconfig %s does not contain blocks, yields only one parsed easyconfig"
% spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (
spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt,
['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs',
letter_dir_for(name), name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (
spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name, moduleclass
self.assertEqual(ec['name'], name)
self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure that $root is not used, since it is not compatible with module files in Lua syntax
res = re.findall('.*\$root.*', ec.rawtxt, re.M)
error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s"
self.assertFalse(res, error_msg % res)
# make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used
old_urls = [
'github.com/hpcugent/easybuild',
'hpcugent.github.com/easybuild',
'hpcugent.github.io/easybuild',
]
for old_url in old_urls:
self.assertFalse(old_url in ec.rawtxt,
"Old URL '%s' not found in %s" % (old_url, spec))
# make sure binutils is included as a build dep if toolchain is GCCcore
if ec['toolchain']['name'] == 'GCCcore':
# with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball
requires_binutils = ec['easyblock'] not in [
'Tarball'
] and ec['name'] not in ['Eigen']
# let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this
# exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore)
if ec['toolchain']['version'] == 'system':
binutils_complete_dependencies = [
'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'
]
requires_binutils &= bool(
ec['name'] not in binutils_complete_dependencies)
# if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled)
requires_binutils &= bool(ec['sources'] or ec['exts_list']
or ec.get('components'))
if requires_binutils:
dep_names = [d['name'] for d in ec.builddependencies()]
self.assertTrue(
'binutils' in dep_names,
"binutils is a build dep in %s: %s" % (spec, dep_names))
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
extra_opts = app.extra_options()
for key in extra_opts:
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
ec_dict = ec.parser.get_config_dict()
orig_toolchain = ec_dict['toolchain']
for key in ec_dict:
# skip parameters for which value is equal to default value
orig_val = ec_dict[key]
if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
continue
if key in extra_opts and orig_val == extra_opts[key][0]:
continue
if key not in DEFAULT_CONFIG and key not in extra_opts:
continue
orig_val = resolve_template(ec_dict[key], ec.template_values)
dumped_val = resolve_template(dumped_ec[key], ec.template_values)
# take into account that dumped value for *dependencies may include hard-coded subtoolchains
# if no easyconfig was found for the dependency with the 'parent' toolchain,
# if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
if key in DEPENDENCY_PARAMETERS:
# number of dependencies should remain the same
self.assertEqual(len(orig_val), len(dumped_val))
for orig_dep, dumped_dep in zip(orig_val, dumped_val):
# name/version should always match
self.assertEqual(orig_dep[:2], dumped_dep[:2])
# 3rd value is versionsuffix;
if len(dumped_dep) >= 3:
# if no versionsuffix was specified in original dep spec, then dumped value should be empty string
if len(orig_dep) >= 3:
self.assertEqual(dumped_dep[2], orig_dep[2])
else:
self.assertEqual(dumped_dep[2], '')
# 4th value is toolchain spec
if len(dumped_dep) >= 4:
if len(orig_dep) >= 4:
self.assertEqual(dumped_dep[3], orig_dep[3])
else:
# if a subtoolchain is specifed (only) in the dumped easyconfig,
# it should *not* be the same as the parent toolchain
self.assertNotEqual(dumped_dep[3],
(orig_toolchain['name'],
orig_toolchain['version']))
else:
self.assertEqual(orig_val, dumped_val)
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
if '.git' in subfolders:
log.info("found .git subfolder, ignoring it")
subfolders.remove('.git')
for ec_file in files:
if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]:
log.warning("SKIPPING %s/%s" % (root, ec_file))
continue
ec_file = join(root, ec_file)
ec_file = read(ec_file)
try:
ec = EasyConfig(ec_file)
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec)
# check if an easyblock exists
module = get_easyblock_class(None, name=ec.name).__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
else:
ec.easyblock = None
configs.append(ec)
names.append(ec.name)
except Exception, err:
log.error("faulty easyconfig %s: %s" % (ec_file, err))
log.info("Found easyconfigs: %s" % [x.name for x in configs])
# sort by name
configs = sorted(configs, key=lambda config : config.name.lower())
firstl = ""
# print out the configs in markdown format for the wiki
def template_easyconfig_test(self, spec):
"""Test whether all easyconfigs can be initialized."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec)
# sanity check for software name
name = fetch_parameter_from_easyconfig_file(spec, 'name')
self.assertTrue(ec['name'], name)
# try and fetch easyblock spec from easyconfig
easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock')
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
app.close_log()
os.remove(app.logfile)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
tmpdir = tempfile.mkdtemp()
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname))
toolchain = None
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# easyblocks deriving from IntelBase require a license file to be found for --module-only
if app_class == IntelBase or IntelBase in app_class.__bases__:
os.environ['INTEL_LICENSE_FILE'] = os.path.join(tmpdir, 'intel.lic')
write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license')
if app_class == EB_IMOD:
# $JAVA_HOME must be set for IMOD
os.environ['JAVA_HOME'] = tmpdir
if app_class == EB_OpenFOAM:
# proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER
write_file(os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([
'#%Module',
'setenv EBROOTGCC %s' % tmpdir,
'setenv EBVERSIONGCC 4.9.3',
]))
write_file(os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'), '\n'.join([
'#%Module',
'setenv EBROOTOPENMPI %s' % tmpdir,
'setenv EBVERSIONOPENMPI 1.10.2',
]))
write_file(os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([
'#%Module',
'module load GCC/4.9.3-2.25',
'module load OpenMPI/1.10.2-GCC-4.9.3-2.25',
]))
os.environ['MODULEPATH'] = tmpdir
toolchain = {'name': 'gompi', 'version': '2016a'}
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, name=name, version=version, extratxt=extra_txt, toolchain=toolchain)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
os.chdir(orig_workdir)
modfile = os.path.join(TMPDIR, 'modules', 'all', name, version)
luamodfile = '%s.lua' % modfile
self.assertTrue(os.path.exists(modfile) or os.path.exists(luamodfile),
"Module file %s or %s was generated" % (modfile, luamodfile))
if os.path.exists(modfile):
modtxt = read_file(modfile)
else:
modtxt = read_file(luamodfile)
none_regex = re.compile('None')
self.assertFalse(none_regex.search(modtxt), "None not found in module file: %s" % modtxt)
# cleanup
app.close_log()
os.remove(app.logfile)
shutil.rmtree(tmpdir)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def __init__(self, *args, **kwargs):
"""Initialize easyblock."""
super(Bundle, self).__init__(*args, **kwargs)
self.altroot = None
self.altversion = None
# list of EasyConfig instances for components
self.comp_cfgs = []
# list of EasyConfig instances of components for which to run sanity checks
self.comp_cfgs_sanity_check = []
# list of sources for bundle itself *must* be empty
if self.cfg['sources']:
raise EasyBuildError(
"List of sources for bundle itself must be empty, found %s",
self.cfg['sources'])
if self.cfg['patches']:
raise EasyBuildError(
"List of patches for bundle itself must be empty, found %s",
self.cfg['patches'])
# disable templating to avoid premature resolving of template values
self.cfg.enable_templating = False
# list of checksums for patches (must be included after checksums for sources)
checksums_patches = []
if self.cfg['sanity_check_components'] and self.cfg[
'sanity_check_all_components']:
raise EasyBuildError(
"sanity_check_components and sanity_check_all_components cannot be enabled together"
)
# backup and reset general sanity checks from main body of ec, if component-specific sanity checks are enabled
# necessary to avoid:
# - duplicating the general sanity check across all components running sanity checks
# - general sanity checks taking precedence over those defined in a component's easyblock
self.backup_sanity_paths = self.cfg['sanity_check_paths']
self.backup_sanity_cmds = self.cfg['sanity_check_commands']
if self.cfg['sanity_check_components'] or self.cfg[
'sanity_check_all_components']:
# reset general sanity checks, to be restored later
self.cfg['sanity_check_paths'] = {}
self.cfg['sanity_check_commands'] = {}
for comp in self.cfg['components']:
comp_name, comp_version, comp_specs = comp[0], comp[1], {}
if len(comp) == 3:
comp_specs = comp[2]
comp_cfg = self.cfg.copy()
comp_cfg['name'] = comp_name
comp_cfg['version'] = comp_version
# determine easyblock to use for this component
# - if an easyblock is specified explicitely, that will be used
# - if not, a software-specific easyblock will be considered by get_easyblock_class
# - if no easyblock was found, default_easyblock is considered
comp_easyblock = comp_specs.get('easyblock')
easyblock_class = get_easyblock_class(
comp_easyblock,
name=comp_name,
error_on_missing_easyblock=False)
if easyblock_class is None:
if self.cfg['default_easyblock']:
easyblock = self.cfg['default_easyblock']
easyblock_class = get_easyblock_class(easyblock)
if easyblock_class is None:
raise EasyBuildError(
"No easyblock found for component %s v%s", comp_name,
comp_version)
else:
self.log.info(
"Using default easyblock %s for component %s",
easyblock, comp_name)
else:
easyblock = easyblock_class.__name__
self.log.info("Using easyblock %s for component %s", easyblock,
comp_name)
if easyblock == 'Bundle':
raise EasyBuildError(
"The Bundle easyblock can not be used to install components in a bundle"
)
comp_cfg.easyblock = easyblock_class
# make sure that extra easyconfig parameters are known, so they can be set
extra_opts = comp_cfg.easyblock.extra_options()
comp_cfg.extend_params(copy.deepcopy(extra_opts))
comp_cfg.generate_template_values()
# do not inherit easyblock to use from parent (since that would result in an infinite loop in install_step)
comp_cfg['easyblock'] = None
# reset list of sources/source_urls/checksums
comp_cfg['sources'] = comp_cfg['source_urls'] = comp_cfg[
'checksums'] = comp_cfg['patches'] = []
for key in self.cfg['default_component_specs']:
comp_cfg[key] = self.cfg['default_component_specs'][key]
for key in comp_specs:
comp_cfg[key] = comp_specs[key]
# enable resolving of templates for component-specific EasyConfig instance
comp_cfg.enable_templating = True
# 'sources' is strictly required
if comp_cfg['sources']:
# If per-component source URLs are provided, attach them directly to the relevant sources
if comp_cfg['source_urls']:
for source in comp_cfg['sources']:
if isinstance(source, string_type):
self.cfg.update(
'sources',
[{
'filename': source,
'source_urls': comp_cfg['source_urls']
}])
elif isinstance(source, dict):
# Update source_urls in the 'source' dict to use the one for the components
# (if it doesn't already exist)
if 'source_urls' not in source:
source['source_urls'] = comp_cfg['source_urls']
self.cfg.update('sources', [source])
else:
raise EasyBuildError(
"Source %s for component %s is neither a string nor a dict, cannot "
"process it.", source, comp_cfg['name'])
else:
# add component sources to list of sources
self.cfg.update('sources', comp_cfg['sources'])
else:
raise EasyBuildError(
"No sources specification for component %s v%s", comp_name,
comp_version)
if comp_cfg['checksums']:
src_cnt = len(comp_cfg['sources'])
# add per-component checksums for sources to list of checksums
self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt])
# add per-component checksums for patches to list of checksums for patches
checksums_patches.extend(comp_cfg['checksums'][src_cnt:])
if comp_cfg['patches']:
self.cfg.update('patches', comp_cfg['patches'])
self.comp_cfgs.append(comp_cfg)
self.cfg.update('checksums', checksums_patches)
self.cfg.enable_templating = True
# restore general sanity checks if using component-specific sanity checks
if self.cfg['sanity_check_components'] or self.cfg[
'sanity_check_all_components']:
self.cfg['sanity_check_paths'] = self.backup_sanity_paths
self.cfg['sanity_check_commands'] = self.backup_sanity_cmds
subfolders.remove('.git')
for ec_file in files:
if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]:
log.warning("SKIPPING %s/%s" % (root, ec_file))
continue
ec_file = join(root, ec_file)
ec_file = read(ec_file)
try:
ec = EasyConfig(ec_file)
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec.name)
ec.easyblock = None
# check if an easyblock exists
ebclass = get_easyblock_class(None,
name=ec.name,
default_fallback=False)
if ebclass is not None:
module = ebclass.__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
configs.append(ec)
names.append(ec.name)
except Exception, err:
raise EasyBuildError("faulty easyconfig %s: %s", ec_file, err)
log.info("Found easyconfigs: %s" % [x.name for x in configs])
# sort by name
configs = sorted(configs, key=lambda config: config.name.lower())
firstl = ""
def template_easyconfig_test(self, spec):
"""Test whether all easyconfigs can be initialized."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(
False,
"easyconfig %s does not contain blocks, yields only one parsed easyconfig"
% spec)
# sanity check for software name
name = fetch_parameter_from_easyconfig_file(spec, 'name')
self.assertTrue(ec['name'], name)
# try and fetch easyblock spec from easyconfig
easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock')
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
app.close_log()
os.remove(app.logfile)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def __init__(self, *args, **kwargs):
"""Initialize easyblock."""
super(Bundle, self).__init__(*args, **kwargs)
self.altroot = None
self.altversion = None
# list of EasyConfig instances for components
self.comp_cfgs = []
# list of sources for bundle itself *must* be empty
if self.cfg['sources']:
raise EasyBuildError(
"List of sources for bundle itself must be empty, found %s",
self.cfg['sources'])
# disable templating to avoid premature resolving of template values
self.cfg.enable_templating = False
# list of checksums for patches (must be included after checksums for sources)
checksums_patches = []
for comp in self.cfg['components']:
comp_name, comp_version, comp_specs = comp[0], comp[1], {}
if len(comp) == 3:
comp_specs = comp[2]
comp_cfg = self.cfg.copy()
easyblock = comp_specs.get(
'easyblock') or self.cfg['default_easyblock']
if easyblock is None:
raise EasyBuildError(
"No easyblock specified for component %s v%s",
comp_cfg['name'], comp_cfg['version'])
elif easyblock == 'Bundle':
raise EasyBuildError(
"The Bundle easyblock can not be used to install components in a bundle"
)
comp_cfg.easyblock = get_easyblock_class(easyblock,
name=comp_cfg['name'])
# make sure that extra easyconfig parameters are known, so they can be set
extra_opts = comp_cfg.easyblock.extra_options()
comp_cfg.extend_params(copy.deepcopy(extra_opts))
comp_cfg['name'] = comp_name
comp_cfg['version'] = comp_version
comp_cfg.generate_template_values()
# do not inherit easyblock to use from parent (since that would result in an infinite loop in install_step)
comp_cfg['easyblock'] = None
# reset list of sources/source_urls/checksums
comp_cfg['sources'] = comp_cfg['source_urls'] = comp_cfg[
'checksums'] = []
for key in self.cfg['default_component_specs']:
comp_cfg[key] = self.cfg['default_component_specs'][key]
for key in comp_specs:
comp_cfg[key] = comp_specs[key]
# enable resolving of templates for component-specific EasyConfig instance
comp_cfg.enable_templating = True
# 'sources' is strictly required
if comp_cfg['sources']:
# If per-component source URLs are provided, attach them directly to the relevant sources
if comp_cfg['source_urls']:
for source in comp_cfg['sources']:
if isinstance(source, basestring):
self.cfg.update(
'sources',
[{
'filename': source,
'source_urls': comp_cfg['source_urls']
}])
elif isinstance(source, dict):
# Update source_urls in the 'source' dict to use the one for the components
# (if it doesn't already exist)
if 'source_urls' not in source:
source['source_urls'] = comp_cfg['source_urls']
self.cfg.update('sources', [source])
else:
raise EasyBuildError(
"Source %s for component %s is neither a string nor a dict, cannot "
"process it.", source, comp_cfg['name'])
else:
# add component sources to list of sources
self.cfg.update('sources', comp_cfg['sources'])
else:
raise EasyBuildError(
"No sources specification for component %s v%s", comp_name,
comp_version)
if comp_cfg['checksums']:
src_cnt = len(comp_cfg['sources'])
# add per-component checksums for sources to list of checksums
self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt])
# add per-component checksums for patches to list of checksums for patches
checksums_patches.extend(comp_cfg['checksums'][src_cnt:])
self.comp_cfgs.append(comp_cfg)
self.cfg.update('checksums', checksums_patches)
self.cfg.enable_templating = True
def test_make_module_pythonpackage(self):
"""Test make_module_step of PythonPackage easyblock."""
app_class = get_easyblock_class('PythonPackage')
self.writeEC('PythonPackage', name='testpypkg', version='3.14')
app = app_class(EasyConfig(self.eb_file))
# install dir should not be there yet
self.assertFalse(os.path.exists(app.installdir),
"%s should not exist" % app.installdir)
# create install dir and populate it with subdirs/files
mkdir(app.installdir, parents=True)
# $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH
write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!')
write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar')
write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo')
pyver = '.'.join(map(str, sys.version_info[:2]))
write_file(
os.path.join(app.installdir, 'lib', 'python%s' % pyver,
'site-packages', 'foo.egg'), 'foo egg')
write_file(
os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'),
'libfoo: foo')
# PythonPackage relies on the fact that 'python' points to the right Python version
tmpdir = tempfile.mkdtemp()
python = os.path.join(tmpdir, 'python')
write_file(python, '#!/bin/bash\necho $0 $@\n%s "$@"' % sys.executable)
adjust_permissions(python, stat.S_IXUSR)
os.environ['PATH'] = '%s:%s' % (tmpdir, os.getenv('PATH', ''))
from easybuild.tools.filetools import which
print(which('python'))
# create module file
app.make_module_step()
remove_file(python)
self.assertTrue(TMPDIR in app.installdir)
self.assertTrue(TMPDIR in app.installdir_mod)
modtxt = None
for cand_mod_filename in ['3.14', '3.14.lua']:
full_modpath = os.path.join(app.installdir_mod, 'testpypkg',
cand_mod_filename)
if os.path.exists(full_modpath):
modtxt = read_file(full_modpath)
break
self.assertFalse(modtxt is None)
regexs = [
(r'^prepend.path.*\WCPATH\W.*include"?\W*$', True),
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WPATH\W.*bin"?\W*$', True),
(r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$',
True),
(r'^prepend.path.*\WPYTHONPATH\W.*lib/python[23]\.[0-9]/site-packages"?\W*$',
True),
# lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False),
]
for (pattern, found) in regexs:
regex = re.compile(pattern, re.M)
if found:
assert_msg = "Pattern '%s' found in: %s" % (regex.pattern,
modtxt)
else:
assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern,
modtxt)
self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def template_init_test(self, easyblock, name='foo', version='1.3.2'):
"""Test whether all easyblocks can be initialized."""
def check_extra_options_format(extra_options):
"""Make sure extra_options value is of correct format."""
# EasyBuild v2.0: dict with <string> keys and <list> values
self.assertTrue(isinstance(extra_options, dict))
extra_options.items()
extra_options.keys()
extra_options.values()
for key in extra_options.keys():
self.assertTrue(isinstance(extra_options[key], list))
self.assertTrue(len(extra_options[key]), 3)
class_regex = re.compile(r"^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
regexps = [
# make sure error reporting is done correctly (no more log.error, log.exception)
re.compile(r"log\.error\("),
re.compile(r"log\.exception\("),
re.compile(r"log\.raiseException\("),
# check for use of 'basestring', which is Python 2.x only (should use string_type from tools.py2vs3 instead)
re.compile(r"[^\w]basestring([^\w]|$)"),
# check for use of '.iteritems()', which is Python 2.x only (should use .items instead)
re.compile(r"\.iteritems\(\)"),
# sys.maxint is no longer there in Python 3
re.compile(r"sys\.maxint"),
]
for regexp in regexps:
self.assertFalse(
regexp.search(txt),
"No match for '%s' in %s" % (regexp.pattern, easyblock))
# make sure that (named) arguments get passed down for prepare_step
if re.search('def prepare_step', txt):
regex = re.compile(r"def prepare_step\(self, \*args, \*\*kwargs\):")
self.assertTrue(
regex.search(txt),
"Pattern '%s' found in %s" % (regex.pattern, easyblock))
if re.search(r'\.prepare_step\(', txt):
regex = re.compile(r"\.prepare_step\(.*\*args,.*\*\*kwargs\.*\)")
self.assertTrue(
regex.search(txt),
"Pattern '%s' found in %s" % (regex.pattern, easyblock))
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
extra_options = app_class.extra_options()
check_extra_options_format(extra_options)
# extend easyconfig to make sure mandatory custom easyconfig parameters are defined
extra_txt = ''
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
# use default value if any is set, otherwise use "foo"
if val[0]:
test_param = val[0]
else:
test_param = 'foo'
extra_txt += '%s = "%s"\n' % (key, test_param)
# write easyconfig file
self.writeEC(ebname, name=name, version=version, extratxt=extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# check whether easyblock instance is still using functions from a deprecated location
mod = __import__(app.__module__, [], [], ['easybuild.easyblocks'])
moved_functions = [
'modify_env', 'parse_log_for_error', 'read_environment', 'run_cmd',
'run_cmd_qa'
]
for fn in moved_functions:
if hasattr(mod, fn):
tup = (fn, app.__module__, globals()[fn].__module__)
self.assertTrue(
getattr(mod, fn) is globals()[fn],
"%s in %s is imported from %s" % tup)
renamed_functions = [
('source_paths', 'source_path'),
('get_avail_core_count', 'get_core_count'),
('get_os_type', 'get_kernel_name'),
('det_full_ec_version', 'det_installversion'),
]
for (new_fn, old_fn) in renamed_functions:
self.assertFalse(
hasattr(mod, old_fn),
"%s: %s is replaced by %s" % (app.__module__, old_fn, new_fn))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
if '.git' in subfolders:
log.info("found .git subfolder, ignoring it")
subfolders.remove('.git')
for ec_file in files:
if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]:
log.warning("SKIPPING %s/%s" % (root, ec_file))
continue
ec_file = join(root, ec_file)
ec_file = read(ec_file)
try:
ec = EasyConfig(ec_file)
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec)
# check if an easyblock exists
module = get_easyblock_class(None, name=ec.name).__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
else:
ec.easyblock = None
configs.append(ec)
names.append(ec.name)
except Exception, err:
log.error("faulty easyconfig %s: %s" % (ec_file, err))
log.info("Found easyconfigs: %s" % [x.name for x in configs])
# sort by name
configs = sorted(configs, key=lambda config : config.name.lower())
firstl = ""
# print out the configs in markdown format for the wiki
def template_module_only_test(self,
easyblock,
name='foo',
version='1.3.2',
extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
tmpdir = tempfile.mkdtemp()
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
toolchain = None
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# easyblocks deriving from IntelBase require a license file to be found for --module-only
if app_class == IntelBase or IntelBase in app_class.__bases__:
os.environ['INTEL_LICENSE_FILE'] = os.path.join(
tmpdir, 'intel.lic')
write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license')
if app_class == EB_IMOD:
# $JAVA_HOME must be set for IMOD
os.environ['JAVA_HOME'] = tmpdir
if app_class == EB_OpenFOAM:
# proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER
write_file(
os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([
'#%Module',
'setenv EBROOTGCC %s' % tmpdir,
'setenv EBVERSIONGCC 4.9.3',
]))
write_file(
os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'),
'\n'.join([
'#%Module',
'setenv EBROOTOPENMPI %s' % tmpdir,
'setenv EBVERSIONOPENMPI 1.10.2',
]))
write_file(
os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([
'#%Module',
'module load GCC/4.9.3-2.25',
'module load OpenMPI/1.10.2-GCC-4.9.3-2.25',
]))
os.environ['MODULEPATH'] = tmpdir
toolchain = {'name': 'gompi', 'version': '2016a'}
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname,
name=name,
version=version,
extratxt=extra_txt,
toolchain=toolchain)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
os.chdir(orig_workdir)
modfile = os.path.join(TMPDIR, 'modules', 'all', name, version)
luamodfile = '%s.lua' % modfile
self.assertTrue(
os.path.exists(modfile) or os.path.exists(luamodfile),
"Module file %s or %s was generated" % (modfile, luamodfile))
if os.path.exists(modfile):
modtxt = read_file(modfile)
else:
modtxt = read_file(luamodfile)
none_regex = re.compile('None')
self.assertFalse(none_regex.search(modtxt),
"None not found in module file: %s" % modtxt)
# cleanup
app.close_log()
os.remove(app.logfile)
shutil.rmtree(tmpdir)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
log.info("found .git subfolder, ignoring it")
subfolders.remove('.git')
for ec_file in files:
if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]:
log.warning("SKIPPING %s/%s" % (root, ec_file))
continue
ec_file = join(root, ec_file)
ec_file = read(ec_file)
try:
ec = EasyConfig(ec_file)
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec.name)
ec.easyblock = None
# check if an easyblock exists
ebclass = get_easyblock_class(None, name=ec.name, error_on_missing_easyblock=False)
if ebclass is not None:
module = ebclass.__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
configs.append(ec)
names.append(ec.name)
except Exception, err:
raise EasyBuildError("faulty easyconfig %s: %s", ec_file, err)
log.info("Found easyconfigs: %s" % [x.name for x in configs])
# sort by name
configs = sorted(configs, key=lambda config : config.name.lower())
firstl = ""
# print out the configs in markdown format for the wiki
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(
False,
"easyconfig %s does not contain blocks, yields only one parsed easyconfig"
% spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (
spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt,
['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs',
name.lower()[0], name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (
spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name
self.assertTrue(ec['name'], name)
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
for key in app.extra_options():
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfig(test_ecfile)
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
dumped_ec.template_values.update(dummy_template_values)
for key in sorted(ec._config):
self.assertEqual(ec[key], dumped_ec[key])
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
log.info("found .git subfolder, ignoring it")
subfolders.remove('.git')
for ec_file in files:
if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]:
log.warning("SKIPPING %s/%s" % (root, ec_file))
continue
ec_file = join(root, ec_file)
ec_file = read(ec_file)
try:
ec = EasyConfig(ec_file)
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec)
ec.easyblock = None
# check if an easyblock exists
ebclass = get_easyblock_class(None, name=ec.name, default_fallback=False)
if ebclass is not None:
module = ebclass.__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
configs.append(ec)
names.append(ec.name)
except Exception, err:
log.error("faulty easyconfig %s: %s" % (ec_file, err))
log.info("Found easyconfigs: %s" % [x.name for x in configs])
# sort by name
configs = sorted(configs, key=lambda config : config.name.lower())
firstl = ""
# print out the configs in markdown format for the wiki
def template_module_only_test(self,
easyblock,
name='foo',
version='1.3.2',
extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
tmpdir = tempfile.mkdtemp()
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
toolchain = None
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# easyblocks deriving from IntelBase require a license file to be found for --module-only
bases = list(app_class.__bases__)
for base in copy.copy(bases):
bases.extend(base.__bases__)
if app_class == IntelBase or IntelBase in bases:
os.environ['INTEL_LICENSE_FILE'] = os.path.join(
tmpdir, 'intel.lic')
write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license')
elif app_class == EB_IMOD:
# $JAVA_HOME must be set for IMOD
os.environ['JAVA_HOME'] = tmpdir
elif app_class == PythonBundle:
# $EBROOTPYTHON must be set for PythonBundle easyblock
os.environ[
'EBROOTPYTHON'] = '/fake/install/prefix/Python/2.7.14-foss-2018a'
elif app_class == EB_OpenFOAM:
# proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER
write_file(
os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([
'#%Module',
'setenv EBROOTGCC %s' % tmpdir,
'setenv EBVERSIONGCC 4.9.3',
]))
write_file(
os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'),
'\n'.join([
'#%Module',
'setenv EBROOTOPENMPI %s' % tmpdir,
'setenv EBVERSIONOPENMPI 1.10.2',
]))
write_file(
os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([
'#%Module',
'module load GCC/4.9.3-2.25',
'module load OpenMPI/1.10.2-GCC-4.9.3-2.25',
]))
os.environ['MODULEPATH'] = tmpdir
toolchain = {'name': 'gompi', 'version': '2016a'}
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname,
name=name,
version=version,
extratxt=extra_txt,
toolchain=toolchain)
# take into account that for some easyblock, particular dependencies are hard required early on
# (in prepare_step for exampel);
# we just set the corresponding $EBROOT* environment variables here to fool it...
req_deps = {
# QScintilla easyblock requires that either PyQt or PyQt5 are available as dependency
# (PyQt is easier, since PyQt5 is only supported for sufficiently recent QScintilla versions)
'qscintilla.py': [('PyQt', '4.12')],
# MotionCor2 and Gctf easyblock requires CUDA as dependency
'motioncor2.py': [('CUDA', '10.1.105')],
'gctf.py': [('CUDA', '10.1.105')],
}
easyblock_fn = os.path.basename(easyblock)
for (dep_name, dep_version) in req_deps.get(easyblock_fn, []):
dep_root_envvar = get_software_root_env_var_name(dep_name)
os.environ[dep_root_envvar] = '/value/should/not/matter'
dep_version_envvar = get_software_version_env_var_name(dep_name)
os.environ[dep_version_envvar] = dep_version
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
change_dir(orig_workdir)
if os.path.basename(easyblock) == 'modulerc.py':
# .modulerc must be cleaned up to avoid causing trouble (e.g. "Duplicate version symbol" errors)
modulerc = os.path.join(TMPDIR, 'modules', 'all', name,
'.modulerc')
if os.path.exists(modulerc):
remove_file(modulerc)
modulerc += '.lua'
if os.path.exists(modulerc):
remove_file(modulerc)
else:
modfile = os.path.join(TMPDIR, 'modules', 'all', name, version)
luamodfile = '%s.lua' % modfile
self.assertTrue(
os.path.exists(modfile) or os.path.exists(luamodfile),
"Module file %s or %s was generated" % (modfile, luamodfile))
if os.path.exists(modfile):
modtxt = read_file(modfile)
else:
modtxt = read_file(luamodfile)
none_regex = re.compile('None')
self.assertFalse(none_regex.search(modtxt),
"None not found in module file: %s" % modtxt)
# cleanup
app.close_log()
remove_file(app.logfile)
remove_dir(tmpdir)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(
False,
"easyconfig %s does not contain blocks, yields only one parsed easyconfig"
% spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easconfig matches expected filename '%s'" % (
spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
# sanity check for software name
name = fetch_parameter_from_easyconfig_file(spec, 'name')
self.assertTrue(ec['name'], name)
# try and fetch easyblock spec from easyconfig
easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock')
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
app.close_log()
os.remove(app.logfile)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
