def test_module_caches(self): """Test module caches and invalidate_module_caches_for function.""" self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) # purposely extending $MODULEPATH with non-existing path, should be handled fine nonpath = os.path.join(self.test_prefix, 'nosuchfileordirectory') self.modtool.use(nonpath) modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] self.assertTrue(any([os.path.samefile(nonpath, mp) for mp in modulepaths])) shutil.rmtree(nonpath) # create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH # invalidate_module_caches_for should be able to deal with this test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') mods_symlink = os.path.join(self.test_prefix, 'modules_symlink') os.symlink(test_mods_path, mods_symlink) self.modtool.use(mods_symlink) # no caching for 'avail' commands with an argument self.assertTrue(self.modtool.available('GCC')) self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) # run 'avail' without argument, result should get cached res = self.modtool.available() # just a single cache entry self.assertEqual(len(mod.MODULE_AVAIL_CACHE), 1) # fetch cache entry avail_cache_key = mod.MODULE_AVAIL_CACHE.keys()[0] cached_res = mod.MODULE_AVAIL_CACHE[avail_cache_key] self.assertTrue(cached_res == res) # running avail again results in getting cached result, exactly the same result as before # depending on the modules tool being used, it may not be the same list instance, because of post-processing self.assertTrue(self.modtool.available() == res) # run 'show', should be all cached show_res_gcc = self.modtool.show('GCC/4.7.2') show_res_fftw = self.modtool.show('FFTW') self.assertEqual(len(mod.MODULE_SHOW_CACHE), 2) self.assertTrue(show_res_gcc in mod.MODULE_SHOW_CACHE.values()) self.assertTrue(show_res_fftw in mod.MODULE_SHOW_CACHE.values()) self.assertTrue(self.modtool.show('GCC/4.7.2') is show_res_gcc) self.assertTrue(self.modtool.show('FFTW') is show_res_fftw) # invalidate caches with correct path modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] self.assertTrue(any([os.path.exists(mp) and os.path.samefile(test_mods_path, mp) for mp in modulepaths])) paths_in_key = [p for p in avail_cache_key[0].split('=')[1].split(os.pathsep) if p] self.assertTrue(any([os.path.exists(p) and os.path.samefile(test_mods_path, p) for p in paths_in_key])) # verify cache invalidation, caches should be empty again invalidate_module_caches_for(test_mods_path) self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) self.assertEqual(mod.MODULE_SHOW_CACHE, {})
def test_module_caches(self): """Test module caches and invalidate_module_caches_for function.""" self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) # purposely extending $MODULEPATH with non-existing path, should be handled fine nonpath = os.path.join(self.test_prefix, 'nosuchfileordirectory') self.modtool.use(nonpath) modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] self.assertTrue(any([os.path.samefile(nonpath, mp) for mp in modulepaths])) shutil.rmtree(nonpath) # create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH # invalidate_module_caches_for should be able to deal with this test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') mods_symlink = os.path.join(self.test_prefix, 'modules_symlink') os.symlink(test_mods_path, mods_symlink) self.modtool.use(mods_symlink) # no caching for 'avail' commands with an argument self.assertTrue(self.modtool.available('GCC')) self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) # run 'avail' without argument, result should get cached res = self.modtool.available() # just a single cache entry self.assertEqual(len(mod.MODULE_AVAIL_CACHE), 1) # fetch cache entry avail_cache_key = mod.MODULE_AVAIL_CACHE.keys()[0] cached_res = mod.MODULE_AVAIL_CACHE[avail_cache_key] self.assertTrue(cached_res == res) # running avail again results in getting cached result, exactly the same result as before # depending on the modules tool being used, it may not be the same list instance, because of post-processing self.assertTrue(self.modtool.available() == res) # run 'show', should be all cached show_res_gcc = self.modtool.show('GCC/4.7.2') show_res_fftw = self.modtool.show('FFTW') self.assertEqual(len(mod.MODULE_SHOW_CACHE), 2) self.assertTrue(show_res_gcc in mod.MODULE_SHOW_CACHE.values()) self.assertTrue(show_res_fftw in mod.MODULE_SHOW_CACHE.values()) self.assertTrue(self.modtool.show('GCC/4.7.2') is show_res_gcc) self.assertTrue(self.modtool.show('FFTW') is show_res_fftw) # invalidate caches with correct path modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] self.assertTrue(any([os.path.exists(mp) and os.path.samefile(test_mods_path, mp) for mp in modulepaths])) paths_in_key = [p for p in avail_cache_key[0].split('=')[1].split(os.pathsep) if p] self.assertTrue(any([os.path.exists(p) and os.path.samefile(test_mods_path, p) for p in paths_in_key])) # verify cache invalidation, caches should be empty again invalidate_module_caches_for(test_mods_path) self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) self.assertEqual(mod.MODULE_SHOW_CACHE, {})
def test_robot_find_minimal_toolchain_of_dependency(self): """Test robot_find_minimal_toolchain_of_dependency.""" # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # # First test that it can do basic resolution # gzip15 = { 'name': 'gzip', 'version': '1.5', 'versionsuffix': '', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } get_toolchain_hierarchy.clear() new_gzip15_toolchain = robot_find_minimal_toolchain_of_dependency(gzip15, self.modtool) self.assertEqual(new_gzip15_toolchain, gzip15['toolchain']) # no easyconfig for gzip 1.4 with matching non-dummy (sub)toolchain gzip14 = { 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } get_toolchain_hierarchy.clear() self.assertEqual(robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool), None) gzip14['toolchain'] = {'name': 'gompi', 'version': '1.4.10'} # # Second test also including dummy toolchain # init_config(build_options={ 'add_dummy_to_minimal_toolchains': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # specify alternative parent toolchain gompi_1410 = {'name': 'gompi', 'version': '1.4.10'} get_toolchain_hierarchy.clear() new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool, parent_tc=gompi_1410) self.assertTrue(new_gzip14_toolchain != gzip14['toolchain']) self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''}) # default: use toolchain from dependency gzip14['toolchain'] = gompi_1410 get_toolchain_hierarchy.clear() new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool) self.assertTrue(new_gzip14_toolchain != gzip14['toolchain']) self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''}) # check reversed order (parent tc first) and skipping of parent tc itself dep = { 'name': 'SQLite', 'version': '3.8.10.2', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool) self.assertEqual(res, {'name': 'GCC', 'version': '4.7.2'}) res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool, parent_first=True) self.assertEqual(res, {'name': 'goolf', 'version': '1.4.10'}) # # Finally test if it can recognise existing modules and use those # barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_txt = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", "toolchain = {'name': 'goolf', 'version': '1.4.10'}", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2 " ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('SQLite', '3.8.10.2'),", # available with goolf/1.4.10, gompi/1.4.10 and GCC/4.7.2 "]", ]) write_file(barec, barec_txt) # check without --minimal-toolchains init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) bar = EasyConfig(barec) expected_dep_versions = [ '1.6.4-GCC-4.7.2', '0.2.6-gompi-1.4.10-LAPACK-3.4.2', '2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', '3.8.10.2-goolf-1.4.10', ] for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions): self.assertEqual(det_full_ec_version(dep), expected_dep_version) # check with --minimal-toolchains enabled init_config(build_options={ 'minimal_toolchains': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) bar = EasyConfig(barec) # check that all bar dependencies have been processed as expected expected_dep_versions[-1] = '3.8.10.2-GCC-4.7.2' for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions): self.assertEqual(det_full_ec_version(dep), expected_dep_version) # Add the gompi/1.4.10 version of SQLite as an available module module_parent = os.path.join(self.test_prefix, 'minimal_toolchain_modules') module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-gompi-1.4.10') module_txt = '\n'.join([ "#%Module", "set root /tmp/SQLite/3.8.10.2", "setenv EBROOTSQLITE $root", "setenv EBVERSIONSQLITE 3.8.10.2", "setenv EBDEVELSQLITE $root/easybuild/SQLite-3.8.10.2-easybuild-devel", ]) write_file(module_file, module_txt) os.environ['MODULEPATH'] = module_parent # Add the parent directory to the MODULEPATH invalidate_module_caches_for(module_parent) # Reinitialize the environment for the updated MODULEPATH and use_existing_modules init_config(build_options={ 'minimal_toolchains': True, 'use_existing_modules': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # Check gompi is now being picked up bar = EasyConfig(barec) # Re-parse the parent easyconfig sqlite = bar.dependencies()[3] self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-gompi-1.4.10') # Add the goolf version as an available version and check that gets precedence over the gompi version module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-goolf-1.4.10') write_file(module_file, module_txt) invalidate_module_caches_for(module_parent) bar = EasyConfig(barec) # Re-parse the parent easyconfig sqlite = bar.dependencies()[3] self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-goolf-1.4.10')