def test_run_cmd_qa_trace(self):
"""Test run_cmd under --trace"""
# replace log.experimental with log.warning to allow experimental code
easybuild.tools.utilities._log.experimental = easybuild.tools.utilities._log.warning
init_config(build_options={'trace': True})
self.mock_stdout(True)
self.mock_stderr(True)
(out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'})
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
self.assertEqual(stderr, '')
pattern = "^ >> running interactive command:\n"
pattern += "\t\[started at: .*\]\n"
pattern += "\t\[output logged in .*\]\n"
pattern += "\techo \'n: \'; read n; seq 1 \$n\n"
pattern += ' >> interactive command completed: exit 0, ran in .*'
self.assertTrue(re.search(pattern, stdout), "Pattern '%s' found in: %s" % (pattern, stdout))
# trace output can be disabled on a per-command basis
self.mock_stdout(True)
self.mock_stderr(True)
(out, ec) = run_cmd("echo hello", trace=False)
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
def test_generaloption_overrides_legacy(self):
"""Test whether generaloption overrides legacy configuration."""
self.purge_environment()
# if both legacy and generaloption style configuration is mixed, generaloption wins
legacy_prefix = os.path.join(self.tmpdir, 'legacy')
go_prefix = os.path.join(self.tmpdir, 'generaloption')
# legacy env vars
os.environ['EASYBUILDPREFIX'] = legacy_prefix
os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build')
# generaloption env vars
os.environ['EASYBUILD_INSTALLPATH'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(legacy_prefix, 'ebfiles_repo'))
del os.environ['EASYBUILDPREFIX']
# legacy env vars
os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'buildagain')
# generaloption env vars
os.environ['EASYBUILD_PREFIX'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(go_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo'))
del os.environ['EASYBUILDBUILDPATH']
def test_get_toolchain_hierarchy(self):
"""Test get_toolchain_hierarchy function."""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'})
self.assertEqual(goolf_hierarchy, [
{'name': 'GCC', 'version': '4.7.2'},
{'name': 'gompi', 'version': '1.4.10'},
{'name': 'goolf', 'version': '1.4.10'},
])
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'})
self.assertEqual(iimpi_hierarchy, [
{'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'},
{'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
])
# test also including dummy
init_config(build_options={
'add_dummy_to_minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
# testing with gompi/1.4.10, since the result for goolf/1.4.10 is cached (and it's hard to reset the cache)
gompi_hierarchy = get_toolchain_hierarchy({'name': 'gompi', 'version': '1.4.10'})
self.assertEqual(gompi_hierarchy, [
{'name': 'dummy', 'version': ''},
{'name': 'GCC', 'version': '4.7.2'},
{'name': 'gompi', 'version': '1.4.10'},
])
def test_check_readiness(self):
"""Test check_readiness method."""
init_config(build_options={'validate': False})
# check that check_readiness step works (adding dependencies, etc.)
ec_file = 'OpenMPI-1.6.4-GCC-4.6.4.eb'
topdir = os.path.dirname(os.path.abspath(__file__))
ec_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'o', 'OpenMPI', ec_file)
ec = EasyConfig(ec_path)
eb = EasyBlock(ec)
eb.check_readiness_step()
# a proper error should be thrown for dependencies that can't be resolved (module should be there)
tmpdir = tempfile.mkdtemp()
shutil.copy2(ec_path, tmpdir)
ec_path = os.path.join(tmpdir, ec_file)
f = open(ec_path, 'a')
f.write("\ndependencies += [('nosuchsoftware', '1.2.3')]\n")
f.close()
ec = EasyConfig(ec_path)
eb = EasyBlock(ec)
try:
eb.check_readiness_step()
except EasyBuildError, err:
err_regex = re.compile("Missing modules for one or more dependencies: nosuchsoftware/1.2.3-GCC-4.6.4")
self.assertTrue(err_regex.search(str(err)), "Pattern '%s' found in '%s'" % (err_regex.pattern, err))
def test_package(self):
"""Test package function."""
init_config(build_options={'silent': True})
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("Contents of installdir %s" % easyblock.installdir)
self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
if DEBUG:
print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
def test_copy_file(self):
""" Test copy_file """
testdir = os.path.dirname(os.path.abspath(__file__))
tmpdir = self.test_prefix
to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
target_path = os.path.join(tmpdir, 'toy.eb')
ft.copy_file(to_copy, target_path)
self.assertTrue(os.path.exists(target_path))
self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
# also test behaviour of extract_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
# remove target file, it shouldn't get copied under dry run
os.remove(target_path)
self.mock_stdout(True)
ft.copy_file(to_copy, target_path)
txt = self.get_stdout()
self.mock_stdout(False)
self.assertFalse(os.path.exists(target_path))
self.assertTrue(re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt))
def test_close_pr(self):
"""Test close_pr function."""
if self.skip_github_tests:
print "Skipping test_close_pr, no GitHub token available?"
return
build_options = {
'dry_run': True,
'github_user': GITHUB_TEST_ACCOUNT,
'pr_target_account': GITHUB_USER,
'pr_target_repo': GITHUB_REPO,
}
init_config(build_options=build_options)
self.mock_stdout(True)
gh.close_pr(2, motivation_msg='just a test')
stdout = self.get_stdout()
self.mock_stdout(False)
patterns = [
"hpcugent/testrepository PR #2 was submitted by migueldiascosta",
"[DRY RUN] Adding comment to testrepository issue #2: '" +
"@migueldiascosta, this PR is being closed for the following reason(s): just a test",
"[DRY RUN] Closed hpcugent/testrepository pull request #2",
]
for pattern in patterns:
self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout))
def test_log_file_format(self):
"""Test for log_file_format()."""
# first test defaults -> no templating when no values are provided
self.assertEqual(log_file_format(), 'easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log')
self.assertEqual(log_file_format(return_directory=True), 'easybuild')
# test whether provided values are used to complete template
ec = {'name': 'foo', 'version': '1.2.3'}
res = log_file_format(ec=ec, date='20190322', timestamp='094356')
self.assertEqual(res, 'easybuild-foo-1.2.3-20190322.094356.log')
res = log_file_format(return_directory=True, ec=ec, date='20190322', timestamp='094356')
self.assertEqual(res, 'easybuild')
# partial templating is done when only some values are provided...
self.assertEqual(log_file_format(ec=ec), 'easybuild-foo-1.2.3-%(date)s.%(time)s.log')
res = log_file_format(date='20190322', timestamp='094356')
self.assertEqual(res, 'easybuild-%(name)s-%(version)s-20190322.094356.log')
# also try with a custom setting
init_config(args=['--logfile-format=eb-%(name)s-%(date)s,log-%(version)s-%(date)s-%(time)s.out'])
self.assertEqual(log_file_format(), 'log-%(version)s-%(date)s-%(time)s.out')
self.assertEqual(log_file_format(return_directory=True), 'eb-%(name)s-%(date)s')
res = log_file_format(ec=ec, date='20190322', timestamp='094356')
self.assertEqual(res, 'log-1.2.3-20190322-094356.out')
res = log_file_format(return_directory=True, ec=ec, date='20190322', timestamp='094356')
self.assertEqual(res, 'eb-foo-20190322')
# test handling of incorrect setting for --logfile-format
init_config(args=['--logfile-format=easybuild,log.txt,thisiswrong'])
error_pattern = "Incorrect log file format specification, should be 2-tuple"
self.assertErrorRegex(EasyBuildError, error_pattern, log_file_format)
def test_check_capability_mapping(self):
"""Test comparing the functionality of two toolchains"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
get_toolchain_hierarchy.clear()
foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True)
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '2016.01'},
incl_capabilities=True)
# Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2
self.assertEqual(foss_hierarchy[0]['name'], 'GCC')
self.assertEqual(foss_hierarchy[1]['name'], 'golf')
self.assertEqual(foss_hierarchy[2]['name'], 'gompi')
self.assertEqual(foss_hierarchy[3]['name'], 'foss')
self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore')
self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort')
self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi')
# golf <-> iimpi (should return False)
self.assertFalse(check_capability_mapping(foss_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs")
# gompi <-> iimpi
self.assertTrue(check_capability_mapping(foss_hierarchy[2], iimpi_hierarchy[2]))
# GCC <-> iimpi
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[2]))
# GCC <-> iccifort
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[1]))
# GCC <-> GCCcore
self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[0]))
def test_map_easyconfig_to_target_tc_hierarchy(self):
"""Test mapping of easyconfig to target hierarchy"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
# The below mapping includes a binutils mapping (2.26 to 2.25)
tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool)
ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb')
tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
tweaked_dict = tweaked_ec['ec'].asdict()
# First check the mapped toolchain
key, value = 'toolchain', iccifort_binutils_tc
self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
# Also check that binutils has been mapped
for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items():
self.assertTrue(key in tweaked_dict['builddependencies'][0] and
value == tweaked_dict['builddependencies'][0][key])
def test_setvar(self):
"""Test setvar function."""
self.mock_stdout(True)
env.setvar('FOO', 'bar')
txt = self.get_stdout()
self.mock_stdout(False)
self.assertEqual(os.getenv('FOO'), 'bar')
self.assertEqual(os.environ['FOO'], 'bar')
# no printing if dry run is not enabled
self.assertEqual(txt, '')
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
self.mock_stdout(True)
env.setvar('FOO', 'foobaz')
txt = self.get_stdout()
self.mock_stdout(False)
self.assertEqual(os.getenv('FOO'), 'foobaz')
self.assertEqual(os.environ['FOO'], 'foobaz')
self.assertEqual(txt, " export FOO=\"foobaz\"\n")
# disabling verbose
self.mock_stdout(True)
env.setvar('FOO', 'barfoo', verbose=False)
txt = self.get_stdout()
self.mock_stdout(False)
self.assertEqual(os.getenv('FOO'), 'barfoo')
self.assertEqual(os.environ['FOO'], 'barfoo')
self.assertEqual(txt, '')
def test_tweak_one_version(self):
"""Test tweak_one function"""
test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')
# test tweaking of software version (--try-software-version)
tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
# checksums should be reset to empty list, only version should be changed, nothing else
self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]:
val = toy_ec_parsed[key]
self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key)
tweaked_val = tweaked_toy_ec_parsed.get(key)
self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val))
# check behaviour if target file already exists
error_pattern = "File exists, not overwriting it without --force"
self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
# existing file does get overwritten when --force is used
init_config(build_options={'force': True, 'silent': True})
write_file(tweaked_toy_ec, '')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
def test_path_to_top_of_module_tree_categorized_hmns(self):
"""
Test function to determine path to top of the module tree for a categorized hierarchical module naming
scheme.
"""
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
init_config(build_options=build_options)
self.setup_categorized_hmns_modules()
modtool = modules_tool()
mod_prefix = os.path.join(self.test_installpath, 'modules', 'all')
init_modpaths = [os.path.join(mod_prefix, 'Core', 'compiler'), os.path.join(mod_prefix, 'Core', 'toolchain')]
deps = ['GCC/4.7.2', 'OpenMPI/1.6.4', 'FFTW/3.3.3', 'OpenBLAS/0.2.6-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2']
path = modtool.path_to_top_of_module_tree(init_modpaths, 'goolf/1.4.10', os.path.join(mod_prefix, 'Core', 'toolchain'), deps)
self.assertEqual(path, [])
path = modtool.path_to_top_of_module_tree(init_modpaths, 'GCC/4.7.2', os.path.join(mod_prefix, 'Core', 'compiler'), [])
self.assertEqual(path, [])
full_mod_subdir = os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'mpi')
deps = ['GCC/4.7.2', 'hwloc/1.6.2']
path = modtool.path_to_top_of_module_tree(init_modpaths, 'OpenMPI/1.6.4', full_mod_subdir, deps)
self.assertEqual(path, ['GCC/4.7.2'])
full_mod_subdir = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4', 'numlib')
deps = ['GCC/4.7.2', 'OpenMPI/1.6.4']
path = modtool.path_to_top_of_module_tree(init_modpaths, 'FFTW/3.3.3', full_mod_subdir, deps)
self.assertEqual(path, ['OpenMPI/1.6.4', 'GCC/4.7.2'])
def test_external_modules_metadata(self):
"""Test --external-modules-metadata."""
# empty list by default
cfg = init_config()
self.assertEqual(cfg.external_modules_metadata, [])
testcfgtxt = EXTERNAL_MODULES_METADATA
testcfg = os.path.join(self.test_prefix, 'test_external_modules_metadata.cfg')
write_file(testcfg, testcfgtxt)
cfg = init_config(args=['--external-modules-metadata=%s' % testcfg])
netcdf = {
'name': ['netCDF', 'netCDF-Fortran'],
'version': ['4.3.2', '4.3.2'],
'prefix': 'NETCDF_DIR',
}
self.assertEqual(cfg.external_modules_metadata['cray-netcdf/4.3.2'], netcdf)
hdf5 = {
'name': ['HDF5'],
'version': ['1.8.13'],
'prefix': 'HDF5_DIR',
}
self.assertEqual(cfg.external_modules_metadata['cray-hdf5/1.8.13'], hdf5)
# impartial metadata is fine
self.assertEqual(cfg.external_modules_metadata['foo'], {'name': ['Foo'], 'prefix': '/foo'})
self.assertEqual(cfg.external_modules_metadata['bar/1.2.3'], {'name': ['bar'], 'version': ['1.2.3']})
# if both names and versions are specified, lists must have same lengths
write_file(testcfg, '\n'.join(['[foo/1.2.3]', 'name = foo,bar', 'version = 1.2.3']))
args = ['--external-modules-metadata=%s' % testcfg]
err_msg = "Different length for lists of names/versions in metadata for external module"
self.assertErrorRegex(EasyBuildError, err_msg, init_config, args=args)
def test_exts_list(self):
"""Test handling of list of extensions."""
os.environ['EASYBUILD_SOURCEPATH'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
init_config()
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = [',
' ("ext1", "ext_ver1", {',
' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail
' "source_urls": ["http://example.com/"]',
' }),',
' ("ext2", "ext_ver2", {',
' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail
' "source_urls": [("http://example.com", "suffix")],'
' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail
' "checksums": [',
' "504c7036558938f997c1c269a01d7458",', # checksum for source (gzip-1.4.eb)
' "ddd5161154f5db67701525123129ff09",', # checksum for patch (toy-0.0.eb)
' ],',
' }),',
']',
])
self.prep()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
exts_sources = eb.fetch_extension_sources()
def test_remove_file(self):
"""Test remove_file"""
testfile = os.path.join(self.test_prefix, 'foo')
ft.write_file(testfile, 'bar')
self.assertTrue(os.path.exists(testfile))
ft.remove_file(testfile)
ft.write_file(testfile, 'bar')
ft.adjust_permissions(self.test_prefix, stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH, add=False)
self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile)
# also test behaviour of remove_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
self.mock_stdout(True)
ft.remove_file(testfile)
txt = self.get_stdout()
self.mock_stdout(False)
regex = re.compile("^file [^ ]* removed$")
self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def test_independence(self):
"""Test independency of toolchain instances."""
# tweaking --optarch is required for Cray toolchains (craypre-<optarch> module must be available)
init_config(build_options={'optarch': 'test'})
tc_cflags = {
'CrayCCE': "-craype-verbose -O2",
'CrayGNU': "-craype-verbose -O2",
'CrayIntel': "-craype-verbose -O2 -ftz -fp-speculation=safe -fp-model source",
'GCC': "-O2 -test",
'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source",
}
toolchains = [
('CrayCCE', '2015.06-XC'),
('CrayGNU', '2015.06-XC'),
('CrayIntel', '2015.06-XC'),
('GCC', '4.7.2'),
('iccifort', '2011.13.367'),
]
# purposely obtain toolchains several times in a row, value for $CFLAGS should not change
for _ in range(3):
for tcname, tcversion in toolchains:
tc = get_toolchain({'name': tcname, 'version': tcversion}, {},
mns=ActiveMNS(), modtool=self.modtool)
tc.set_options({})
tc.prepare()
expected_cflags = tc_cflags[tcname]
msg = "Expected $CFLAGS found for toolchain %s: %s" % (tcname, expected_cflags)
self.assertEqual(str(tc.variables['CFLAGS']), expected_cflags, msg)
self.assertEqual(os.environ['CFLAGS'], expected_cflags, msg)
def test_lmod_specific(self):
"""Lmod-specific test (skipped unless Lmod is used as modules tool)."""
lmod_abspath = which(Lmod.COMMAND)
# only run this test if 'lmod' is available in $PATH
if lmod_abspath is not None:
build_options = {
'allow_modules_tool_mismatch': True,
}
init_config(build_options=build_options)
# drop any location where 'lmod' or 'spider' can be found from $PATH
paths = os.environ.get('PATH', '').split(os.pathsep)
new_paths = []
for path in paths:
lmod_cand_path = os.path.join(path, Lmod.COMMAND)
spider_cand_path = os.path.join(path, 'spider')
if not os.path.isfile(lmod_cand_path) and not os.path.isfile(spider_cand_path):
new_paths.append(path)
os.environ['PATH'] = os.pathsep.join(new_paths)
# make sure $MODULEPATH contains path that provides some modules
os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))
# initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD
os.environ['LMOD_CMD'] = lmod_abspath
lmod = Lmod(testing=True)
# obtain list of availabe modules, should be non-empty
self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty")
# test updating local spider cache (but don't actually update the local cache file!)
self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
self.purge_environment()
# check whether configuration via environment variables works as expected
prefix = os.path.join(self.tmpdir, 'testprefix')
buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build', 'path')
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var
options = init_config(args=[])
self.assertEqual(build_path(), buildpath_env_var)
self.assertEqual(install_path(), os.path.join(prefix, 'software'))
self.assertEqual(get_repositorypath(), [os.path.join(prefix, 'ebfiles_repo')])
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_BUILDPATH']
# check whether configuration via command line arguments works
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--config', config_file, # force empty oldstyle config file
'--prefix', prefix,
'--installpath', install,
'--repositorypath', repopath,
]
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'software'))
self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertEqual(options.config, config_file)
# check mixed command line/env var configuration
prefix = os.path.join(self.tmpdir, 'test3')
install = os.path.join(self.tmpdir, 'test4', 'install')
subdir_software = 'eb-soft'
args = [
'--config', config_file, # force empty oldstyle config file
'--installpath', install,
]
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, subdir_software))
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def test_toolchain_prepare_rpath(self):
"""Test toolchain.prepare under --rpath"""
# put fake 'gcc' command in place that just echos its arguments
fake_gcc = os.path.join(self.test_prefix, 'fake', 'gcc')
write_file(fake_gcc, '#!/bin/bash\necho "$@"')
adjust_permissions(fake_gcc, stat.S_IXUSR)
os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', ''))
# enable --rpath and prepare toolchain
init_config(build_options={'rpath': True, 'rpath_filter': ['/ba.*']})
tc = self.get_toolchain('gompi', version='1.3.12')
# preparing RPATH wrappers requires --experimental, need to bypass that here
tc.log.experimental = lambda x: x
tc.prepare()
# check whether fake gcc was wrapped and that arguments are what they should be
# no -rpath for /bar because of rpath filter
out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""')
expected = ' '.join([
'-Wl,-rpath=$ORIGIN/../lib',
'-Wl,-rpath=$ORIGIN/../lib64',
'-Wl,--disable-new-dtags',
'-Wl,-rpath=/foo',
'%(user)s.c',
'-L/foo',
'-L/bar',
'$FOO',
'-DX=""',
])
self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
def test_prepare_step(self):
"""Test prepare step (setting up build environment)."""
test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0]
mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'dummy-dummy'), parents=True)
eb = EasyBlock(ec['ec'])
eb.silent = True
eb.prepare_step()
self.assertEqual(self.modtool.list(), [])
os.environ['THIS_IS_AN_UNWANTED_ENV_VAR'] = 'foo'
eb.cfg['unwanted_env_vars'] = ['THIS_IS_AN_UNWANTED_ENV_VAR']
eb.cfg['allow_system_deps'] = [('Python', '1.2.3')]
init_config(build_options={'extra_modules': ['GCC/4.7.2']})
eb.prepare_step()
self.assertEqual(os.environ.get('THIS_IS_AN_UNWANTED_ENV_VAR'), None)
self.assertEqual(os.environ.get('EBROOTPYTHON'), 'Python')
self.assertEqual(os.environ.get('EBVERSIONPYTHON'), '1.2.3')
self.assertEqual(len(self.modtool.list()), 1)
self.assertEqual(self.modtool.list()[0]['mod_name'], 'GCC/4.7.2')
def test_robot_archived_easyconfigs(self):
"""Test whether robot can pick up archived easyconfigs when asked."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-ictce-4.1.13.eb')
gzip_ectxt = read_file(gzip_ec)
test_ec = os.path.join(self.test_prefix, 'test.eb')
tc_spec = "toolchain = {'name': 'ictce', 'version': '3.2.2.u3'}"
regex = re.compile("^toolchain = .*", re.M)
test_ectxt = regex.sub(tc_spec, gzip_ectxt)
write_file(test_ec, test_ectxt)
ecs, _ = parse_easyconfigs([(test_ec, False)])
self.assertErrorRegex(EasyBuildError, "Irresolvable dependencies encountered", resolve_dependencies,
ecs, self.modtool, retain_all_deps=True)
# --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs
init_config(build_options={
'consider_archived_easyconfigs': True,
'robot_path': [test_ecs],
})
res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True)
self.assertEqual([ec['full_mod_name'] for ec in res], ['ictce/3.2.2.u3', 'gzip/1.5-ictce-3.2.2.u3'])
expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb')
self.assertTrue(os.path.samefile(res[0]['spec'], expected))
def test_module_mismatch(self):
"""Test whether mismatch detection between modules tool and 'module' function works."""
# redefine 'module' function (deliberate mismatch with used module command in MockModulesTool)
os.environ["module"] = "() { eval `/tmp/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
error_regex = ".*pattern .* not found in defined 'module' function"
self.assertErrorRegex(EasyBuildError, error_regex, MockModulesTool, testing=True)
# check whether escaping error by allowing mismatch via build options works
build_options = {"allow_modules_tool_mismatch": True}
init_config(build_options=build_options)
fancylogger.logToFile(self.logfile)
mt = MockModulesTool(testing=True)
f = open(self.logfile, "r")
logtxt = f.read()
f.close()
warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function")
self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt))
# redefine 'module' function with correct module command
os.environ["module"] = "() { eval `/bin/echo $*`\n}"
mt = MockModulesTool(testing=True)
self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage
# a warning should be logged if the 'module' function is undefined
del os.environ["module"]
mt = MockModulesTool(testing=True)
f = open(self.logfile, "r")
logtxt = f.read()
f.close()
warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*")
self.assertTrue(warn_regex.search(logtxt), "Pattern %s found in %s" % (warn_regex.pattern, logtxt))
fancylogger.logToFile(self.logfile, enable=False)
def test_read_write_file(self):
"""Test reading/writing files."""
fp = os.path.join(self.test_prefix, 'test.txt')
txt = "test123"
ft.write_file(fp, txt)
self.assertEqual(ft.read_file(fp), txt)
txt2 = '\n'.join(['test', '123'])
ft.write_file(fp, txt2, append=True)
self.assertEqual(ft.read_file(fp), txt+txt2)
# also test behaviour of write_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
foo = os.path.join(self.test_prefix, 'foo.txt')
self.mock_stdout(True)
ft.write_file(foo, 'bar')
txt = self.get_stdout()
self.mock_stdout(False)
self.assertFalse(os.path.exists(foo))
self.assertTrue(re.match("^file written: .*/foo.txt$", txt))
ft.write_file(foo, 'bar', forced=True)
self.assertTrue(os.path.exists(foo))
self.assertEqual(ft.read_file(foo), 'bar')
def test_hierarchical_mns(self):
"""Test hierarchical module naming scheme."""
ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, init_modpaths):
"""Test whether active module naming scheme returns expected values."""
ec = EasyConfig(os.path.join(ecs_dir, ecfile))
self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname))
self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname)
self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir)
self.assertEqual(ActiveMNS().det_modpath_extensions(ec), modpath_exts)
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths)
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
# format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions)
iccver = '2013.5.192-GCC-4.8.3'
impi_ec = 'impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb'
imkl_ec = 'imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb'
test_ecs = {
'GCC-4.7.2.eb': ('GCC/4.7.2', 'Core', ['Compiler/GCC/4.7.2'], ['Core']),
'OpenMPI-1.6.4-GCC-4.7.2.eb': ('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2', ['MPI/GCC/4.7.2/OpenMPI/1.6.4'], ['Core']),
'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4', [], ['Core']),
'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core', [], ['Core']),
'icc-2013.5.192-GCC-4.8.3.eb': ('icc/%s' % iccver, 'Core', ['Compiler/intel/%s' % iccver], ['Core']),
'ifort-2013.3.163.eb': ('ifort/2013.3.163', 'Core', ['Compiler/intel/2013.3.163'], ['Core']),
'CUDA-5.5.22-GCC-4.8.2.eb': ('CUDA/5.5.22', 'Compiler/GCC/4.8.2', ['Compiler/GCC-CUDA/4.8.2-5.5.22'], ['Core']),
impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s' % iccver, ['MPI/intel/%s/impi/4.1.3.049' % iccver], ['Core']),
imkl_ec: ('imkl/11.1.2.144', 'MPI/intel/%s/impi/4.1.3.049' % iccver, [], ['Core']),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile, *mns_vals)
# impi with dummy toolchain, which doesn't make sense in a hierarchical context
ec = EasyConfig(os.path.join(ecs_dir, 'impi-4.1.3.049.eb'))
self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec)
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
test_ecs = {
'GCC-4.7.2.eb': ('GCC/4.7.2', '', [], []),
'OpenMPI-1.6.4-GCC-4.7.2.eb': ('OpenMPI/1.6.4-GCC-4.7.2', '', [], []),
'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5-goolf-1.4.10', '', [], []),
'goolf-1.4.10.eb': ('goolf/1.4.10', '', [], []),
'impi-4.1.3.049.eb': ('impi/4.1.3.049', '', [], []),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile, *mns_vals)
def test_strict(self):
"""Test use of --strict."""
# check default
self.assertEqual(build_option('strict'), run.WARN)
for strict_str, strict_val in [('error', run.ERROR), ('ignore', run.IGNORE), ('warn', run.WARN)]:
options = init_config(args=['--strict=%s' % strict_str])
init_config(build_options={'strict': options.strict})
self.assertEqual(build_option('strict'), strict_val)
def test_exclude_path_to_top_of_module_tree(self):
"""
Make sure that modules under the HierarchicalMNS are correct,
w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
"""
self.orig_module_naming_scheme = config.get_module_naming_scheme()
test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all')
mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True)
mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True)
impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049')
imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144')
if get_module_syntax() == 'Lua':
impi_modfile_path += '.lua'
imkl_modfile_path += '.lua'
# example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
# not for the toolchain or any of the toolchain components,
# since both icc/ifort and impi form the path to the top of the module tree
tests = [
('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']),
('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']),
]
for ec_file, modfile_path, excluded_deps in tests:
ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
eb = EasyBlock(ec)
eb.toolchain.prepare()
modpath = eb.make_module_step()
modfile_path = os.path.join(modpath, modfile_path)
modtxt = read_file(modfile_path)
for dep in excluded_deps:
tup = (dep, modfile_path, modtxt)
failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
if get_module_syntax() == 'Tcl':
self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg)
elif get_module_syntax() == 'Lua':
self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
def test_modules_tool_stateless(self):
"""Check whether ModulesTool instance is stateless between runs."""
test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
# copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load
shutil.copytree(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core'))
shutil.copytree(os.path.join(test_modules_path, 'Compiler'), os.path.join(self.test_prefix, 'Compiler'))
modtxt = read_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'))
modpath_extension = os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2')
modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M)
write_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'), modtxt)
modtxt = read_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'))
modpath_extension = os.path.join(self.test_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4')
mkdir(modpath_extension, parents=True)
modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M)
write_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), modtxt)
# force reset of any singletons by reinitiating config
init_config()
os.environ['MODULEPATH'] = os.path.join(self.test_prefix, 'Core')
modtool = modules_tool()
if isinstance(modtool, Lmod):
load_err_msg = "cannot[\s\n]*be[\s\n]*loaded"
else:
load_err_msg = "Unable to locate a modulefile"
# GCC/4.6.3 is *not* an available Core module
self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['GCC/4.6.3'])
# GCC/4.7.2 is one of the available Core modules
modtool.load(['GCC/4.7.2'])
# OpenMPI/1.6.4 becomes available after loading GCC/4.7.2 module
modtool.load(['OpenMPI/1.6.4'])
modtool.purge()
# reset $MODULEPATH, obtain new ModulesTool instance,
# which should not remember anything w.r.t. previous $MODULEPATH value
os.environ['MODULEPATH'] = test_modules_path
modtool = modules_tool()
# GCC/4.6.3 is available
modtool.load(['GCC/4.6.3'])
modtool.purge()
# GCC/4.7.2 is available (note: also as non-Core module outside of hierarchy)
modtool.load(['GCC/4.7.2'])
# OpenMPI/1.6.4 is *not* available with current $MODULEPATH (loaded GCC/4.7.2 was not a hierarchical module)
self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['OpenMPI/1.6.4'])
def test_find_minimally_resolved_modules(self):
"""Test find_minimally_resolved_modules function."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_txt = '\n'.join([
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),", # only available with goolf/1.4.10
"]",
])
write_file(barec, barec_txt)
bar = process_easyconfig(barec)[0]
ecs = [bar]
mods = [
'gompi/1.4.10',
'goolf/1.4.10',
# include modules for dependencies, with subtoolchains rather than full toolchain (except for SQLite)
'OpenMPI/1.6.4-GCC-4.7.2',
'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'SQLite/3.8.10.2-GCC-4.7.2',
]
ordered_ecs, new_easyconfigs, new_avail_modules = find_minimally_resolved_modules(ecs, mods, [])
# all dependencies are resolved for easyconfigs included in ordered_ecs
self.assertEqual(len(ordered_ecs), 1)
self.assertEqual(ordered_ecs[0]['dependencies'], [])
# module is added to list of available modules
self.assertTrue(bar['ec'].full_mod_name in new_avail_modules)
# nothing left
self.assertEqual(new_easyconfigs, [])
def test_wrong_modulepath(self):
"""Test whether modules tool can deal with a broken $MODULEPATH."""
test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
modules_test_installpath = os.path.join(self.test_installpath, 'modules', 'all')
os.environ['MODULEPATH'] = "/some/non-existing/path:/this/doesnt/exists/anywhere:%s" % test_modules_path
init_config()
modtool = modules_tool()
self.assertEqual(len(modtool.mod_paths), 2)
self.assertTrue(os.path.samefile(modtool.mod_paths[0], modules_test_installpath))
self.assertEqual(modtool.mod_paths[1], test_modules_path)
self.assertTrue(len(modtool.available()) > 0)
def test_list_prs(self):
"""Test list_prs function."""
if self.skip_github_tests:
print("Skipping test_list_prs, no GitHub token available?")
return
parameters = ('closed', 'created', 'asc')
init_config(build_options={'pr_target_account': GITHUB_USER,
'pr_target_repo': GITHUB_REPO})
expected = "PR #1: a pr"
self.mock_stdout(True)
output = gh.list_prs(parameters, per_page=1, github_user=GITHUB_TEST_ACCOUNT)
stdout = self.get_stdout()
self.mock_stdout(False)
self.assertTrue(stdout.startswith("== Listing PRs with parameters: "))
self.assertEqual(expected, output)
def test_load(self):
"""Test load part in generated module file."""
expected = [
"",
"if { ![is-loaded mod_name] } {",
" module load mod_name",
"}",
"",
]
self.assertEqual('\n'.join(expected),
self.modgen.load_module("mod_name"))
# with recursive unloading: no if is-loaded guard
init_config(build_options={'recursive_mod_unload': True})
expected = [
"",
"module load mod_name",
"",
]
self.assertEqual('\n'.join(expected),
self.modgen.load_module("mod_name"))
def test_find_patches(self):
""" Test for find_software_name_for_patch """
testdir = os.path.dirname(os.path.abspath(__file__))
ec_path = os.path.join(testdir, 'easyconfigs')
init_config(build_options={
'allow_modules_tool_mismatch': True,
'minimal_toolchains': True,
'use_existing_modules': True,
'external_modules_metadata': ConfigObj(),
'silent': True,
'valid_module_classes': module_classes(),
'validate': False,
})
self.mock_stdout(True)
ec = gh.find_software_name_for_patch('toy-0.0_fix-silly-typo-in-printf-statement.patch', [ec_path])
txt = self.get_stdout()
self.mock_stdout(False)
self.assertTrue(ec == 'toy')
reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked')
self.assertTrue(re.search(reg, txt))
def test_load(self):
"""Test load part in generated module file."""
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
expected = [
'',
"if { ![ is-loaded mod_name ] } {",
" module load mod_name",
"}",
'',
]
self.assertEqual('\n'.join(expected),
self.modgen.load_module("mod_name"))
# with recursive unloading: no if is-loaded guard
init_config(build_options={'recursive_mod_unload': True})
expected = [
'',
"module load mod_name",
'',
]
self.assertEqual('\n'.join(expected),
self.modgen.load_module("mod_name"))
else:
expected = '\n'.join([
'',
'if not isloaded("mod_name") then',
' load("mod_name")',
'end',
'',
])
self.assertEqual(expected, self.modgen.load_module("mod_name"))
init_config(build_options={'recursive_mod_unload': True})
expected = '\n'.join([
'',
'load("mod_name")',
'',
])
self.assertEqual(expected, self.modgen.load_module("mod_name"))
def test_install_path(self):
"""Test install_path function."""
# defaults
self.assertEqual(install_path(), os.path.join(self.test_installpath, 'software'))
self.assertEqual(install_path('software'), os.path.join(self.test_installpath, 'software'))
self.assertEqual(install_path(typ='mod'), os.path.join(self.test_installpath, 'modules'))
self.assertEqual(install_path('modules'), os.path.join(self.test_installpath, 'modules'))
self.assertErrorRegex(EasyBuildError, "Unknown type specified", install_path, typ='foo')
args = [
'--subdir-software', 'SOFT',
'--installpath', '/foo',
]
os.environ['EASYBUILD_SUBDIR_MODULES'] = 'MOD'
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/foo', 'SOFT'))
self.assertEqual(install_path(typ='mod'), os.path.join('/foo', 'MOD'))
del os.environ['EASYBUILD_SUBDIR_MODULES']
args = [
'--installpath', '/prefix',
'--installpath-modules', '/foo',
]
os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] = '/bar/baz'
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/bar', 'baz'))
self.assertEqual(install_path(typ='mod'), '/foo')
del os.environ['EASYBUILD_INSTALLPATH_SOFTWARE']
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/prefix', 'software'))
self.assertEqual(install_path(typ='mod'), '/foo')
def test_independence(self):
"""Test independency of toolchain instances."""
# tweaking --optarch is required for Cray toolchains (craypre-<optarch> module must be available)
init_config(build_options={'optarch': 'test'})
tc_cflags = {
'CrayCCE': "-craype-verbose -O2",
'CrayGNU': "-craype-verbose -O2",
'CrayIntel':
"-craype-verbose -O2 -ftz -fp-speculation=safe -fp-model source",
'GCC': "-O2 -test",
'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source",
}
toolchains = [
('CrayCCE', '2015.06-XC'),
('CrayGNU', '2015.06-XC'),
('CrayIntel', '2015.06-XC'),
('GCC', '4.7.2'),
('iccifort', '2011.13.367'),
]
# purposely obtain toolchains several times in a row, value for $CFLAGS should not change
for _ in range(3):
for tcname, tcversion in toolchains:
tc = get_toolchain({
'name': tcname,
'version': tcversion
}, {},
mns=ActiveMNS(),
modtool=self.modtool)
tc.set_options({})
tc.prepare()
expected_cflags = tc_cflags[tcname]
msg = "Expected $CFLAGS found for toolchain %s: %s" % (
tcname, expected_cflags)
self.assertEqual(str(tc.variables['CFLAGS']), expected_cflags,
msg)
self.assertEqual(os.environ['CFLAGS'], expected_cflags, msg)
def test_map_easyconfig_to_target_tc_hierarchy(self):
"""Test mapping of easyconfig to target hierarchy"""
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
init_config(
build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
iccifort_binutils_tc = {
'name': 'iccifort',
'version': '2016.1.150-GCC-4.9.3-2.25'
}
# The below mapping includes a binutils mapping (2.26 to 2.25)
tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc,
iccifort_binutils_tc,
self.modtool)
ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc',
'hwloc-1.6.2-GCC-4.9.3-2.26.eb')
tweaked_spec = map_easyconfig_to_target_tc_hierarchy(
ec_spec, tc_mapping)
tweaked_ec = process_easyconfig(tweaked_spec)[0]
tweaked_dict = tweaked_ec['ec'].asdict()
# First check the mapped toolchain
key, value = 'toolchain', iccifort_binutils_tc
self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
# Also check that binutils has been mapped
for key, value in {
'name': 'binutils',
'version': '2.25',
'versionsuffix': ''
}.items():
self.assertTrue(
key in tweaked_dict['builddependencies'][0]
and value == tweaked_dict['builddependencies'][0][key])
def test_package(self):
"""Test package function."""
init_config(build_options={'silent': True})
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# write a dummy log and report file to make sure they don't get packaged
logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log")
write_file(logfile, "I'm a logfile")
reportfile = os.path.join(easyblock.installdir, log_path(), "report.md")
write_file(reportfile, "I'm a reportfile")
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" % easyblock.installdir)
self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
no_logfiles_regex = re.compile(r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS', re.DOTALL|re.MULTILINE)
self.assertFalse(no_logfiles_regex.search(pkgtxt), "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt))
if DEBUG:
print "The FPM script debug output"
print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
print "The Package File"
print read_file(pkgfile)
def test_optarch_generic(self):
"""Test whether --optarch=GENERIC works as intended."""
for generic in [False, True]:
if generic:
build_options = {'optarch': 'GENERIC'}
init_config(build_options=build_options)
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']
tcs = {
'gompi': ('1.3.12', "-march=x86-64 -mtune=generic"),
'iccifort': ('2011.13.367', "-xSSE2 -ftz -fp-speculation=safe -fp-model source"),
}
for tcopt_optarch in [False, True]:
for tcname in tcs:
tcversion, generic_flags = tcs[tcname]
tc = self.get_toolchain(tcname, version=tcversion)
tc.set_options({'optarch': tcopt_optarch})
tc.prepare()
for var in flag_vars:
if generic:
self.assertTrue(generic_flags in tc.get_variable(var))
else:
self.assertFalse(generic_flags in tc.get_variable(var))
def test_dep_tree_of_toolchain(self):
"""Test getting list of dependencies of a toolchain (as EasyConfig objects)"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
'check_osdeps': False,
})
toolchain_spec = {'name': 'foss', 'version': '2018a'}
list_of_deps = get_dep_tree_of_toolchain(toolchain_spec, self.modtool)
expected_deps = [
['GCC', '6.4.0'],
['OpenBLAS', '0.2.20'],
['hwloc', '1.11.8'],
['OpenMPI', '2.1.2'],
['gompi', '2018a'],
['FFTW', '3.3.7'],
['ScaLAPACK', '2.0.2'],
['foss', '2018a']
]
actual_deps = [[dep['name'], dep['version']] for dep in list_of_deps]
self.assertEqual(expected_deps, actual_deps)
def test_reasons_for_closing(self):
"""Test reasons_for_closing function."""
if self.skip_github_tests:
print("Skipping test_reasons_for_closing, no GitHub token available?")
return
repo_owner = gh.GITHUB_EB_MAIN
repo_name = gh.GITHUB_EASYCONFIGS_REPO
build_options = {
'dry_run': True,
'github_user': GITHUB_TEST_ACCOUNT,
'pr_target_account': repo_owner,
'pr_target_repo': repo_name,
'robot_path': [],
}
init_config(build_options=build_options)
pr_data, _ = gh.fetch_pr_data(1844, repo_owner, repo_name, GITHUB_TEST_ACCOUNT, full=True)
self.mock_stdout(True)
self.mock_stderr(True)
# can't easily check return value, since auto-detected reasons may change over time if PR is touched
res = gh.reasons_for_closing(pr_data)
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
self.assertTrue(isinstance(res, list))
self.assertEqual(stderr.strip(), "WARNING: Using easyconfigs from closed PR #1844")
patterns = [
"Status of last commit is SUCCESS",
"Last comment on",
"No activity since",
"* QEMU-2.4.0",
]
for pattern in patterns:
self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout))
def test_make_module_dep_of_dep_hmns(self):
"""Test for make_module_dep under HMNS with dependencies of dependencies"""
test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
# GCC and OpenMPI extend $MODULEPATH; hwloc is a dependency of OpenMPI.
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
'dependencies = [',
" ('GCC', '4.7.2', '', True),"
" ('hwloc', '1.6.2', '', ('GCC', '4.7.2')),",
" ('OpenMPI', '1.6.4', '', ('GCC', '4.7.2')),"
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
# GCC, OpenMPI and hwloc modules should *not* be included in loads for dependencies
mod_dep_txt = eb.make_module_dep()
for mod in ['GCC/4.7.2', 'OpenMPI/1.6.4', 'hwloc/1.6.2']:
regex = re.compile('load.*%s' % mod)
self.assertFalse(regex.search(mod_dep_txt), "Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
def test_run_hook(self):
"""Test for run_hook function."""
hooks = load_hooks(self.test_hooks_pymod)
init_config(build_options={'debug': True})
self.mock_stdout(True)
self.mock_stderr(True)
run_hook('start', hooks)
run_hook('parse',
hooks,
args=['<EasyConfig instance>'],
msg="Running parse hook for example.eb...")
run_hook('configure', hooks, pre_step_hook=True, args=[None])
run_hook('configure', hooks, post_step_hook=True, args=[None])
run_hook('build', hooks, pre_step_hook=True, args=[None])
run_hook('build', hooks, post_step_hook=True, args=[None])
run_hook('install', hooks, pre_step_hook=True, args=[None])
run_hook('install', hooks, post_step_hook=True, args=[None])
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
expected_stdout = '\n'.join([
"== Running start hook...",
"this is triggered at the very beginning",
"== Running parse hook for example.eb...",
"Parse hook with argument <EasyConfig instance>",
"== Running post-configure hook...",
"this is run after configure step",
"running foo helper method",
"== Running pre-install hook...",
"this is run before install step",
])
self.assertEqual(stdout.strip(), expected_stdout)
self.assertEqual(stderr, '')
def test_det_user_modpath(self):
"""Test for generic det_user_modpath method."""
# None by default
self.assertEqual(self.modgen.det_user_modpath(None), None)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
self.assertEqual(self.modgen.det_user_modpath('my/own/modules'),
'"my/own/modules" "all"')
else:
self.assertEqual(self.modgen.det_user_modpath('my/own/modules'),
'"my/own/modules", "all"')
# result is affected by --suffix-modules-path
# {RUNTIME_ENV::FOO} gets translated into Tcl/Lua syntax for resolving $FOO at runtime
init_config(build_options={'suffix_modules_path': ''})
user_modpath = 'my/{RUNTIME_ENV::TEST123}/modules'
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
self.assertEqual(self.modgen.det_user_modpath(user_modpath),
'"my" $::env(TEST123) "modules"')
else:
self.assertEqual(self.modgen.det_user_modpath(user_modpath),
'"my", os.getenv("TEST123"), "modules"')
def test_check_python_version(self):
"""Test check_python_version function."""
init_config(build_options={'silence_deprecation_warnings': []})
def mock_python_ver(py_maj_ver, py_min_ver):
"""Helper function to mock a particular Python version."""
st.sys.version_info = (py_maj_ver, py_min_ver) + sys.version_info[2:]
# mock running with different Python versions
mock_python_ver(1, 4)
error_pattern = r"EasyBuild is not compatible \(yet\) with Python 1.4"
self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version)
mock_python_ver(4, 0)
error_pattern = r"EasyBuild is not compatible \(yet\) with Python 4.0"
self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version)
mock_python_ver(2, 6)
error_pattern = r"Python 2.7 is required when using Python 2, found Python 2.6"
self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version)
# no problems when running with a supported Python version
for pyver in [(2, 7), (3, 5), (3, 6), (3, 7)]:
mock_python_ver(*pyver)
self.assertEqual(check_python_version(), pyver)
# shouldn't raise any errors, since Python version used to run tests should be supported;
self.mock_stderr(True)
(py_maj_ver, py_min_ver) = check_python_version()
stderr = self.get_stderr()
self.mock_stderr(False)
self.assertFalse(stderr)
self.assertTrue(py_maj_ver in [2, 3])
if py_maj_ver == 2:
self.assertTrue(py_min_ver == 7)
else:
self.assertTrue(py_min_ver >= 5)
def test_copy_file(self):
""" Test copy_file """
testdir = os.path.dirname(os.path.abspath(__file__))
tmpdir = self.test_prefix
to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
target_path = os.path.join(tmpdir, 'toy.eb')
ft.copy_file(to_copy, target_path)
self.assertTrue(os.path.exists(target_path))
self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
# also test behaviour of copy_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
# remove target file, it shouldn't get copied under dry run
os.remove(target_path)
self.mock_stdout(True)
ft.copy_file(to_copy, target_path)
txt = self.get_stdout()
self.mock_stdout(False)
self.assertFalse(os.path.exists(target_path))
self.assertTrue(
re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt))
# forced copy, even in dry run mode
self.mock_stdout(True)
ft.copy_file(to_copy, target_path, force_in_dry_run=True)
txt = self.get_stdout()
self.mock_stdout(False)
self.assertTrue(os.path.exists(target_path))
self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
self.assertEqual(txt, '')
def test_module_mismatch(self):
"""Test whether mismatch detection between modules tool and 'module' function works."""
# redefine 'module' function (deliberate mismatch with used module command in MockModulesTool)
os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
self.assertErrorRegex(EasyBuildError, ".*pattern .* not found in defined 'module' function", MockModulesTool)
# check whether escaping error by allowing mismatch via build options works
build_options = {
'allow_modules_tool_mismatch': True,
}
init_config(build_options=build_options)
fancylogger.logToFile(self.logfile)
mt = MockModulesTool()
f = open(self.logfile, 'r')
logtxt = f.read()
f.close()
warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function")
self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt))
# redefine 'module' function with correct module command
os.environ['module'] = "() { eval `/bin/echo $*`\n}"
MockModulesTool._instances.pop(MockModulesTool)
mt = MockModulesTool()
self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage
# a warning should be logged if the 'module' function is undefined
del os.environ['module']
MockModulesTool._instances.pop(MockModulesTool)
mt = MockModulesTool()
f = open(self.logfile, 'r')
logtxt = f.read()
f.close()
warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*")
self.assertTrue(warn_regex.search(logtxt), "Pattern %s found in %s" % (warn_regex.pattern, logtxt))
fancylogger.logToFile(self.logfile, enable=False)
def test_lmod_specific(self):
"""Lmod-specific test (skipped unless Lmod is used as modules tool)."""
lmod_abspath = which(Lmod.COMMAND)
# only run this test if 'lmod' is available in $PATH
if lmod_abspath is not None:
build_options = {
'allow_modules_tool_mismatch': True,
'update_modules_tool_cache': True,
}
init_config(build_options=build_options)
# drop any location where 'lmod' or 'spider' can be found from $PATH
paths = os.environ.get('PATH', '').split(os.pathsep)
new_paths = []
for path in paths:
lmod_cand_path = os.path.join(path, Lmod.COMMAND)
spider_cand_path = os.path.join(path, 'spider')
if not os.path.isfile(lmod_cand_path) and not os.path.isfile(
spider_cand_path):
new_paths.append(path)
os.environ['PATH'] = os.pathsep.join(new_paths)
# make sure $MODULEPATH contains path that provides some modules
os.environ['MODULEPATH'] = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'modules'))
# initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD
os.environ['LMOD_CMD'] = lmod_abspath
lmod = Lmod(testing=True)
# obtain list of availabe modules, should be non-empty
self.assertTrue(
lmod.available(),
"List of available modules obtained using Lmod is non-empty")
# test updating local spider cache (but don't actually update the local cache file!)
self.assertTrue(lmod.update(),
"Updated local Lmod spider cache is non-empty")
def test_download_file(self):
"""Test download_file function."""
fn = 'toy-0.0.tar.gz'
target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn)
# provide local file path as source URL
test_dir = os.path.abspath(os.path.dirname(__file__))
source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn)
res = ft.download_file(fn, source_url, target_location)
self.assertEqual(res, target_location, "'download' of local file works")
# non-existing files result in None return value
self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None)
# install broken proxy handler for opening local files
# this should make urllib2.urlopen use this broken proxy for downloading from a file:// URL
proxy_handler = urllib2.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir})
urllib2.install_opener(urllib2.build_opener(proxy_handler))
# downloading over a broken proxy results in None return value (failed download)
# this tests whether proxies are taken into account by download_file
self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails")
# restore a working file handler, and retest download of local file
urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler()))
res = ft.download_file(fn, source_url, target_location)
self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy")
# make sure specified timeout is parsed correctly (as a float, not a string)
opts = init_config(args=['--download-timeout=5.3'])
init_config(build_options={'download_timeout': opts.download_timeout})
target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt')
url = 'https://jenkins1.ugent.be/robots.txt'
try:
urllib2.urlopen(url)
res = ft.download_file(fn, url, target_location)
self.assertEqual(res, target_location, "download with specified timeout works")
except urllib2.URLError:
print "Skipping timeout test in test_download_file (working offline)"
def test_external_module_toolchain(self):
"""Test specifying external (build) dependencies in yaml format."""
if 'yaml' not in sys.modules:
print("Skipping test_external_module_toolchain (no PyYAML available)")
return
ecpath = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'yeb', 'CrayCCE-5.1.29.yeb')
metadata = {
'name': ['foo', 'bar'],
'version': ['1.2.3', '3.2.1'],
'prefix': '/foo/bar',
}
build_options = {
'external_modules_metadata': {'fftw/3.3.4.0': metadata},
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
easybuild.tools.build_log.EXPERIMENTAL = True
ec = EasyConfig(ecpath)
self.assertEqual(ec.dependencies()[1]['full_mod_name'], 'fftw/3.3.4.0')
self.assertEqual(ec.dependencies()[1]['external_module_metadata'], metadata)
def test_obtain_file(self):
"""Test obtain_file method."""
toy_tarball = 'toy-0.0.tar.gz'
testdir = os.path.abspath(os.path.dirname(__file__))
sandbox_sources = os.path.join(testdir, 'sandbox', 'sources')
toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball)
tmpdir = tempfile.mkdtemp()
tmpdir_subdir = os.path.join(tmpdir, 'testing')
mkdir(tmpdir_subdir, parents=True)
del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp
ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0]
eb = EasyBlock(ec['ec'])
# 'downloading' a file to (first) sourcepath works
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)])
shutil.copy2(toy_tarball_path, tmpdir_subdir)
res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball))
# finding a file in sourcepath works
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)])
res = eb.obtain_file(toy_tarball)
self.assertEqual(res, toy_tarball_path)
# sourcepath has preference over downloading
res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
self.assertEqual(res, toy_tarball_path)
# obtain_file yields error for non-existing files
fn = 'thisisclearlyanonexistingfile'
error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn
self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir])
# file specifications via URL also work, are downloaded to (first) sourcepath
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)])
urls = ["http://hpcugent.github.io/easybuild/index.html", "https://hpcugent.github.io/easybuild/index.html"]
for file_url in urls:
fn = os.path.basename(file_url)
res = None
try:
res = eb.obtain_file(file_url)
except EasyBuildError, err:
# if this fails, it should be because there's no online access
download_fail_regex = re.compile('socket error')
self.assertTrue(download_fail_regex.search(str(err)))
# result may be None during offline testing
if res is not None:
loc = os.path.join(tmpdir, 't', 'toy', fn)
self.assertEqual(res, loc)
self.assertTrue(os.path.exists(loc), "%s file is found at %s" % (fn, loc))
txt = open(loc, 'r').read()
eb_regex = re.compile("EasyBuild: building software with ease")
self.assertTrue(eb_regex.search(txt))
else:
print "ignoring failure to download %s in test_obtain_file, testing offline?" % file_url
def test_match_minimum_tc_specs(self):
"""Test matching a toolchain to lowest possible in a hierarchy"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'robot_path': test_easyconfigs,
'silent': True,
'valid_module_classes': module_classes(),
})
get_toolchain_hierarchy.clear()
foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True)
iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '2016.01'},
incl_capabilities=True)
# Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2
self.assertEqual(foss_hierarchy[0]['name'], 'GCC')
self.assertEqual(foss_hierarchy[1]['name'], 'golf')
self.assertEqual(foss_hierarchy[2]['name'], 'gompi')
self.assertEqual(foss_hierarchy[3]['name'], 'foss')
self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore')
self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort')
self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi')
# base compiler first (GCCcore which maps to GCC/6.4.0-2.28)
self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[0], foss_hierarchy),
{'name': 'GCC', 'version': '6.4.0-2.28'})
# then iccifort (which also maps to GCC/6.4.0-2.28)
self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[1], foss_hierarchy),
{'name': 'GCC', 'version': '6.4.0-2.28'})
# Then MPI
self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[2], foss_hierarchy),
{'name': 'gompi', 'version': '2018a'})
# Check against own math only subtoolchain for math
self.assertEqual(match_minimum_tc_specs(foss_hierarchy[1], foss_hierarchy),
{'name': 'golf', 'version': '2018a'})
# Make sure there's an error when we can't do the mapping
error_msg = "No possible mapping from source toolchain spec .*"
self.assertErrorRegex(EasyBuildError, error_msg, match_minimum_tc_specs,
foss_hierarchy[3], iimpi_hierarchy)
def test_lmod_specific(self):
"""Lmod-specific test (skipped unless Lmod is used as modules tool)."""
lmod_abspath = which(Lmod.COMMAND)
# only run this test if 'lmod' is available in $PATH
if lmod_abspath is not None:
build_options = {
'allow_modules_tool_mismatch': True,
'update_modules_tool_cache': True,
}
init_config(build_options=build_options)
lmod = Lmod(testing=True)
self.assertTrue(os.path.samefile(lmod.cmd, lmod_abspath))
# drop any location where 'lmod' or 'spider' can be found from $PATH
paths = os.environ.get('PATH', '').split(os.pathsep)
new_paths = []
for path in paths:
lmod_cand_path = os.path.join(path, Lmod.COMMAND)
spider_cand_path = os.path.join(path, 'spider')
if not os.path.isfile(lmod_cand_path) and not os.path.isfile(
spider_cand_path):
new_paths.append(path)
os.environ['PATH'] = os.pathsep.join(new_paths)
# make sure $MODULEPATH contains path that provides some modules
os.environ['MODULEPATH'] = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'modules'))
# initialize Lmod modules tool, pass (fake) full path to 'lmod' via $LMOD_CMD
fake_path = os.path.join(self.test_installpath, 'lmod')
fake_lmod_txt = '\n'.join([
'#!/bin/bash',
'echo "Modules based on Lua: Version %s " >&2' %
Lmod.REQ_VERSION,
'echo "os.environ[\'FOO\'] = \'foo\'"',
])
write_file(fake_path, fake_lmod_txt)
os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR)
os.environ['LMOD_CMD'] = fake_path
init_config(build_options=build_options)
lmod = Lmod(testing=True)
self.assertTrue(os.path.samefile(lmod.cmd, fake_path))
# use correct full path for 'lmod' via $LMOD_CMD
os.environ['LMOD_CMD'] = lmod_abspath
init_config(build_options=build_options)
lmod = Lmod(testing=True)
# obtain list of availabe modules, should be non-empty
self.assertTrue(
lmod.available(),
"List of available modules obtained using Lmod is non-empty")
# test updating local spider cache (but don't actually update the local cache file!)
self.assertTrue(lmod.update(),
"Updated local Lmod spider cache is non-empty")
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs
except ImportError:
print "GC3Pie not available, skipping test"
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False, recursive=False)
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
options = init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs)
topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path)
jobs = build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0'))
self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def test_obtain_file(self):
"""Test obtain_file method."""
toy_tarball = 'toy-0.0.tar.gz'
testdir = os.path.abspath(os.path.dirname(__file__))
sandbox_sources = os.path.join(testdir, 'sandbox', 'sources')
toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball)
tmpdir = tempfile.mkdtemp()
tmpdir_subdir = os.path.join(tmpdir, 'testing')
mkdir(tmpdir_subdir, parents=True)
del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp
ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0]
eb = EasyBlock(ec['ec'])
# 'downloading' a file to (first) sourcepath works
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)])
shutil.copy2(toy_tarball_path, tmpdir_subdir)
res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)])
self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball))
# finding a file in sourcepath works
init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)])
res = eb.obtain_file(toy_tarball)
self.assertEqual(res, toy_tarball_path)
# sourcepath has preference over downloading
res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)])
self.assertEqual(res, toy_tarball_path)
# obtain_file yields error for non-existing files
fn = 'thisisclearlyanonexistingfile'
try:
eb.obtain_file(fn, urls=[os.path.join('file://', tmpdir_subdir)])
except EasyBuildError, err:
fail_regex = re.compile("Couldn't find file %s anywhere, and downloading it didn't work either" % fn)
self.assertTrue(fail_regex.search(str(err)))
def test_override_optarch(self):
"""Test whether overriding the optarch flag works."""
flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']
for optarch_var in ['march=lovelylovelysandybridge', None]:
build_options = {'optarch': optarch_var}
init_config(build_options=build_options)
for enable in [True, False]:
tc = self.get_toolchain("goalf", version="1.1.0-no-OFED")
tc.set_options({'optarch': enable})
tc.prepare()
flag = None
if optarch_var is not None:
flag = '-%s' % optarch_var
else:
# default optarch flag
flag = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch]
for var in flag_vars:
flags = tc.get_variable(var)
if enable:
self.assertTrue(flag in flags, "optarch: True means %s in %s" % (flag, flags))
else:
self.assertFalse(flag in flags, "optarch: False means no %s in %s" % (flag, flags))
modules.modules_tool().purge()
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
args = [
'--configfiles',
config_file,
'--debug',
'--buildpath',
testpath1,
'--installpath-software',
installpath_software,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [
os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')
]) # default
self.assertEqual(install_path(),
installpath_software) # via cmdline arg
self.assertEqual(install_path('mod'),
os.path.join(testpath2, 'modules')) # via config file
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
tmpdir = tempfile.mkdtemp(
prefix='easybuild-easyconfigs-pkg-install-path')
mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'easyconfigs')
shutil.copytree(test_ecs_dir,
os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
orig_sys_path = sys.path[:]
sys.path.insert(
0, tmpdir
) # prepend to give it preference over possible other installed easyconfigs pkgs
# test with config file passed via environment variable
# also test for existence of HOME and USER by adding paths to robot-paths
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
'sourcepath = %(DEFAULT_REPOSITORYPATH)s',
'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir',
'robot-paths=/tmp/foo:%(sourcepath)s:%(HOME)s:/tmp/%(USER)s:%(DEFAULT_ROBOT_PATHS)s',
'installpath-modules=%s' % installpath_modules,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath',
testpath2,
]
options = init_config(args=args)
topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild')
self.assertEqual(install_path(), os.path.join(topdir,
'software')) # default
self.assertEqual(install_path('mod'),
installpath_modules), # via config file
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
self.assertEqual(get_repositorypath(),
[os.path.join(topdir, 'ebfiles_repo'), 'somesubdir'
]) # via config file
# hardcoded first entry
self.assertEqual(options.robot_paths[0], '/tmp/foo')
# resolved value for %(sourcepath)s template
self.assertEqual(
options.robot_paths[1],
os.path.join(os.getenv('HOME'), '.local', 'easybuild',
'ebfiles_repo'))
# resolved value for HOME constant
self.assertEqual(options.robot_paths[2], os.getenv('HOME'))
# resolved value that uses USER constant
self.assertEqual(options.robot_paths[3],
os.path.join('/tmp', os.getenv('USER')))
# first path in DEFAULT_ROBOT_PATHS
self.assertEqual(options.robot_paths[4],
os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath',
testpath3,
]
options = init_config(args=args)
self.assertEqual(
source_paths(),
[testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(),
os.path.join(testpath3,
'software')) # via command line
self.assertEqual(install_path('mod'),
installpath_modules), # via config file
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
sys.path[:] = orig_sys_path
def test_modules_tool_stateless(self):
"""Check whether ModulesTool instance is stateless between runs."""
test_modules_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'modules')
# copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load
shutil.copytree(os.path.join(test_modules_path, 'Core'),
os.path.join(self.test_prefix, 'Core'))
shutil.copytree(os.path.join(test_modules_path, 'Compiler'),
os.path.join(self.test_prefix, 'Compiler'))
modtxt = read_file(
os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'))
modpath_extension = os.path.join(self.test_prefix, 'Compiler', 'GCC',
'4.7.2')
modtxt = re.sub('module use .*', 'module use %s' % modpath_extension,
modtxt, re.M)
write_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'),
modtxt)
modtxt = read_file(
os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2',
'OpenMPI', '1.6.4'))
modpath_extension = os.path.join(self.test_prefix, 'MPI', 'GCC',
'4.7.2', 'OpenMPI', '1.6.4')
mkdir(modpath_extension, parents=True)
modtxt = re.sub('module use .*', 'module use %s' % modpath_extension,
modtxt, re.M)
write_file(
os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2',
'OpenMPI', '1.6.4'), modtxt)
# force reset of any singletons by reinitiating config
init_config()
os.environ['MODULEPATH'] = os.path.join(self.test_prefix, 'Core')
modtool = modules_tool()
if isinstance(modtool, Lmod):
load_err_msg = "cannot[\s\n]*be[\s\n]*loaded"
else:
load_err_msg = "Unable to locate a modulefile"
# GCC/4.6.3 is *not* an available Core module
self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load,
['GCC/4.6.3'])
# GCC/4.7.2 is one of the available Core modules
modtool.load(['GCC/4.7.2'])
# OpenMPI/1.6.4 becomes available after loading GCC/4.7.2 module
modtool.load(['OpenMPI/1.6.4'])
modtool.purge()
# reset $MODULEPATH, obtain new ModulesTool instance,
# which should not remember anything w.r.t. previous $MODULEPATH value
os.environ['MODULEPATH'] = test_modules_path
modtool = modules_tool()
# GCC/4.6.3 is available
modtool.load(['GCC/4.6.3'])
modtool.purge()
# GCC/4.7.2 is available (note: also as non-Core module outside of hierarchy)
modtool.load(['GCC/4.7.2'])
# OpenMPI/1.6.4 is *not* available with current $MODULEPATH (loaded GCC/4.7.2 was not a hierarchical module)
self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load,
['OpenMPI/1.6.4'])
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
self.purge_environment()
# check whether configuration via environment variables works as expected
prefix = os.path.join(self.tmpdir, 'testprefix')
buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build',
'path')
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var
options = init_config(args=[])
self.assertEqual(build_path(), buildpath_env_var)
self.assertEqual(install_path(), os.path.join(prefix, 'software'))
self.assertEqual(get_repositorypath(),
[os.path.join(prefix, 'ebfiles_repo')])
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_BUILDPATH']
# check whether configuration via command line arguments works
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--configfiles',
config_file, # force empty config file
'--prefix',
prefix,
'--installpath',
install,
'--repositorypath',
repopath,
'--subdir-software',
'APPS',
]
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'APPS'))
self.assertEqual(install_path(typ='mod'),
os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertTrue(config_file in options.configfiles)
# check mixed command line/env var configuration
prefix = os.path.join(self.tmpdir, 'test3')
install = os.path.join(self.tmpdir, 'test4', 'install')
subdir_software = 'eb-soft'
args = [
'--configfiles',
config_file, # force empty config file
'--installpath',
install,
]
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
os.environ['EASYBUILD_INSTALLPATH_MODULES'] = installpath_modules
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install,
subdir_software))
self.assertEqual(install_path('mod'), installpath_modules)
# subdir options *must* be relative (to --installpath)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = installpath_software
error_regex = r"Found problems validating the options.*'subdir_software' must specify a \*relative\* path"
self.assertErrorRegex(EasyBuildError, error_regex, init_config)
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def configure(self, args=None):
"""(re)Configure and return configfile"""
options = init_config(args=args)
return options.config