def test_module_caches(self):
"""Test module caches and invalidate_module_caches_for function."""
self.assertEqual(mod.MODULE_AVAIL_CACHE, {})
# purposely extending $MODULEPATH with non-existing path, should be handled fine
nonpath = os.path.join(self.test_prefix, 'nosuchfileordirectory')
self.modtool.use(nonpath)
modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p]
self.assertTrue(any([os.path.samefile(nonpath, mp) for mp in modulepaths]))
shutil.rmtree(nonpath)
# create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH
# invalidate_module_caches_for should be able to deal with this
test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
mods_symlink = os.path.join(self.test_prefix, 'modules_symlink')
os.symlink(test_mods_path, mods_symlink)
self.modtool.use(mods_symlink)
# no caching for 'avail' commands with an argument
self.assertTrue(self.modtool.available('GCC'))
self.assertEqual(mod.MODULE_AVAIL_CACHE, {})
# run 'avail' without argument, result should get cached
res = self.modtool.available()
# just a single cache entry
self.assertEqual(len(mod.MODULE_AVAIL_CACHE), 1)
# fetch cache entry
avail_cache_key = mod.MODULE_AVAIL_CACHE.keys()[0]
cached_res = mod.MODULE_AVAIL_CACHE[avail_cache_key]
self.assertTrue(cached_res == res)
# running avail again results in getting cached result, exactly the same result as before
# depending on the modules tool being used, it may not be the same list instance, because of post-processing
self.assertTrue(self.modtool.available() == res)
# run 'show', should be all cached
show_res_gcc = self.modtool.show('GCC/4.7.2')
show_res_fftw = self.modtool.show('FFTW')
self.assertEqual(len(mod.MODULE_SHOW_CACHE), 2)
self.assertTrue(show_res_gcc in mod.MODULE_SHOW_CACHE.values())
self.assertTrue(show_res_fftw in mod.MODULE_SHOW_CACHE.values())
self.assertTrue(self.modtool.show('GCC/4.7.2') is show_res_gcc)
self.assertTrue(self.modtool.show('FFTW') is show_res_fftw)
# invalidate caches with correct path
modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p]
self.assertTrue(any([os.path.exists(mp) and os.path.samefile(test_mods_path, mp) for mp in modulepaths]))
paths_in_key = [p for p in avail_cache_key[0].split('=')[1].split(os.pathsep) if p]
self.assertTrue(any([os.path.exists(p) and os.path.samefile(test_mods_path, p) for p in paths_in_key]))
# verify cache invalidation, caches should be empty again
invalidate_module_caches_for(test_mods_path)
self.assertEqual(mod.MODULE_AVAIL_CACHE, {})
self.assertEqual(mod.MODULE_SHOW_CACHE, {})
def test_module_caches(self):
"""Test module caches and invalidate_module_caches_for function."""
self.assertEqual(mod.MODULE_AVAIL_CACHE, {})
# purposely extending $MODULEPATH with non-existing path, should be handled fine
nonpath = os.path.join(self.test_prefix, 'nosuchfileordirectory')
self.modtool.use(nonpath)
modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p]
self.assertTrue(any([os.path.samefile(nonpath, mp) for mp in modulepaths]))
shutil.rmtree(nonpath)
# create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH
# invalidate_module_caches_for should be able to deal with this
test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
mods_symlink = os.path.join(self.test_prefix, 'modules_symlink')
os.symlink(test_mods_path, mods_symlink)
self.modtool.use(mods_symlink)
# no caching for 'avail' commands with an argument
self.assertTrue(self.modtool.available('GCC'))
self.assertEqual(mod.MODULE_AVAIL_CACHE, {})
# run 'avail' without argument, result should get cached
res = self.modtool.available()
# just a single cache entry
self.assertEqual(len(mod.MODULE_AVAIL_CACHE), 1)
# fetch cache entry
avail_cache_key = mod.MODULE_AVAIL_CACHE.keys()[0]
cached_res = mod.MODULE_AVAIL_CACHE[avail_cache_key]
self.assertTrue(cached_res == res)
# running avail again results in getting cached result, exactly the same result as before
# depending on the modules tool being used, it may not be the same list instance, because of post-processing
self.assertTrue(self.modtool.available() == res)
# run 'show', should be all cached
show_res_gcc = self.modtool.show('GCC/4.7.2')
show_res_fftw = self.modtool.show('FFTW')
self.assertEqual(len(mod.MODULE_SHOW_CACHE), 2)
self.assertTrue(show_res_gcc in mod.MODULE_SHOW_CACHE.values())
self.assertTrue(show_res_fftw in mod.MODULE_SHOW_CACHE.values())
self.assertTrue(self.modtool.show('GCC/4.7.2') is show_res_gcc)
self.assertTrue(self.modtool.show('FFTW') is show_res_fftw)
# invalidate caches with correct path
modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p]
self.assertTrue(any([os.path.exists(mp) and os.path.samefile(test_mods_path, mp) for mp in modulepaths]))
paths_in_key = [p for p in avail_cache_key[0].split('=')[1].split(os.pathsep) if p]
self.assertTrue(any([os.path.exists(p) and os.path.samefile(test_mods_path, p) for p in paths_in_key]))
# verify cache invalidation, caches should be empty again
invalidate_module_caches_for(test_mods_path)
self.assertEqual(mod.MODULE_AVAIL_CACHE, {})
self.assertEqual(mod.MODULE_SHOW_CACHE, {})
def test_robot_find_minimal_toolchain_of_dependency(self):
"""Test robot_find_minimal_toolchain_of_dependency."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
#
# First test that it can do basic resolution
#
gzip15 = {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
get_toolchain_hierarchy.clear()
new_gzip15_toolchain = robot_find_minimal_toolchain_of_dependency(gzip15, self.modtool)
self.assertEqual(new_gzip15_toolchain, gzip15['toolchain'])
# no easyconfig for gzip 1.4 with matching non-dummy (sub)toolchain
gzip14 = {
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
get_toolchain_hierarchy.clear()
self.assertEqual(robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool), None)
gzip14['toolchain'] = {'name': 'gompi', 'version': '1.4.10'}
#
# Second test also including dummy toolchain
#
init_config(build_options={
'add_dummy_to_minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
# specify alternative parent toolchain
gompi_1410 = {'name': 'gompi', 'version': '1.4.10'}
get_toolchain_hierarchy.clear()
new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool, parent_tc=gompi_1410)
self.assertTrue(new_gzip14_toolchain != gzip14['toolchain'])
self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''})
# default: use toolchain from dependency
gzip14['toolchain'] = gompi_1410
get_toolchain_hierarchy.clear()
new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool)
self.assertTrue(new_gzip14_toolchain != gzip14['toolchain'])
self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''})
# check reversed order (parent tc first) and skipping of parent tc itself
dep = {
'name': 'SQLite',
'version': '3.8.10.2',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool)
self.assertEqual(res, {'name': 'GCC', 'version': '4.7.2'})
res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool, parent_first=True)
self.assertEqual(res, {'name': 'goolf', 'version': '1.4.10'})
#
# Finally test if it can recognise existing modules and use those
#
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_txt = '\n'.join([
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),", # available with goolf/1.4.10, gompi/1.4.10 and GCC/4.7.2
"]",
])
write_file(barec, barec_txt)
# check without --minimal-toolchains
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
bar = EasyConfig(barec)
expected_dep_versions = [
'1.6.4-GCC-4.7.2',
'0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'3.8.10.2-goolf-1.4.10',
]
for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions):
self.assertEqual(det_full_ec_version(dep), expected_dep_version)
# check with --minimal-toolchains enabled
init_config(build_options={
'minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
bar = EasyConfig(barec)
# check that all bar dependencies have been processed as expected
expected_dep_versions[-1] = '3.8.10.2-GCC-4.7.2'
for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions):
self.assertEqual(det_full_ec_version(dep), expected_dep_version)
# Add the gompi/1.4.10 version of SQLite as an available module
module_parent = os.path.join(self.test_prefix, 'minimal_toolchain_modules')
module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-gompi-1.4.10')
module_txt = '\n'.join([
"#%Module",
"set root /tmp/SQLite/3.8.10.2",
"setenv EBROOTSQLITE $root",
"setenv EBVERSIONSQLITE 3.8.10.2",
"setenv EBDEVELSQLITE $root/easybuild/SQLite-3.8.10.2-easybuild-devel",
])
write_file(module_file, module_txt)
os.environ['MODULEPATH'] = module_parent # Add the parent directory to the MODULEPATH
invalidate_module_caches_for(module_parent)
# Reinitialize the environment for the updated MODULEPATH and use_existing_modules
init_config(build_options={
'minimal_toolchains': True,
'use_existing_modules': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
# Check gompi is now being picked up
bar = EasyConfig(barec) # Re-parse the parent easyconfig
sqlite = bar.dependencies()[3]
self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-gompi-1.4.10')
# Add the goolf version as an available version and check that gets precedence over the gompi version
module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-goolf-1.4.10')
write_file(module_file, module_txt)
invalidate_module_caches_for(module_parent)
bar = EasyConfig(barec) # Re-parse the parent easyconfig
sqlite = bar.dependencies()[3]
self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-goolf-1.4.10')
