def test_run_cmd_qa_trace(self): """Test run_cmd under --trace""" # replace log.experimental with log.warning to allow experimental code easybuild.tools.utilities._log.experimental = easybuild.tools.utilities._log.warning init_config(build_options={'trace': True}) self.mock_stdout(True) self.mock_stderr(True) (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) self.assertEqual(stderr, '') pattern = "^ >> running interactive command:\n" pattern += "\t\[started at: .*\]\n" pattern += "\t\[output logged in .*\]\n" pattern += "\techo \'n: \'; read n; seq 1 \$n\n" pattern += ' >> interactive command completed: exit 0, ran in .*' self.assertTrue(re.search(pattern, stdout), "Pattern '%s' found in: %s" % (pattern, stdout)) # trace output can be disabled on a per-command basis self.mock_stdout(True) self.mock_stderr(True) (out, ec) = run_cmd("echo hello", trace=False) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) self.assertEqual(stdout, '') self.assertEqual(stderr, '')
def test_generaloption_overrides_legacy(self): """Test whether generaloption overrides legacy configuration.""" self.purge_environment() # if both legacy and generaloption style configuration is mixed, generaloption wins legacy_prefix = os.path.join(self.tmpdir, 'legacy') go_prefix = os.path.join(self.tmpdir, 'generaloption') # legacy env vars os.environ['EASYBUILDPREFIX'] = legacy_prefix os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build') # generaloption env vars os.environ['EASYBUILD_INSTALLPATH'] = go_prefix init_config() self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build')) self.assertEqual(install_path(), os.path.join(go_prefix, 'software')) repo = init_repository(get_repository(), get_repositorypath()) self.assertEqual(repo.repo, os.path.join(legacy_prefix, 'ebfiles_repo')) del os.environ['EASYBUILDPREFIX'] # legacy env vars os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'buildagain') # generaloption env vars os.environ['EASYBUILD_PREFIX'] = go_prefix init_config() self.assertEqual(build_path(), os.path.join(go_prefix, 'build')) self.assertEqual(install_path(), os.path.join(go_prefix, 'software')) repo = init_repository(get_repository(), get_repositorypath()) self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo')) del os.environ['EASYBUILDBUILDPATH']
def test_get_toolchain_hierarchy(self): """Test get_toolchain_hierarchy function.""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}) self.assertEqual(goolf_hierarchy, [ {'name': 'GCC', 'version': '4.7.2'}, {'name': 'gompi', 'version': '1.4.10'}, {'name': 'goolf', 'version': '1.4.10'}, ]) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}) self.assertEqual(iimpi_hierarchy, [ {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'}, {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'}, ]) # test also including dummy init_config(build_options={ 'add_dummy_to_minimal_toolchains': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # testing with gompi/1.4.10, since the result for goolf/1.4.10 is cached (and it's hard to reset the cache) gompi_hierarchy = get_toolchain_hierarchy({'name': 'gompi', 'version': '1.4.10'}) self.assertEqual(gompi_hierarchy, [ {'name': 'dummy', 'version': ''}, {'name': 'GCC', 'version': '4.7.2'}, {'name': 'gompi', 'version': '1.4.10'}, ])
def test_check_readiness(self): """Test check_readiness method.""" init_config(build_options={'validate': False}) # check that check_readiness step works (adding dependencies, etc.) ec_file = 'OpenMPI-1.6.4-GCC-4.6.4.eb' topdir = os.path.dirname(os.path.abspath(__file__)) ec_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'o', 'OpenMPI', ec_file) ec = EasyConfig(ec_path) eb = EasyBlock(ec) eb.check_readiness_step() # a proper error should be thrown for dependencies that can't be resolved (module should be there) tmpdir = tempfile.mkdtemp() shutil.copy2(ec_path, tmpdir) ec_path = os.path.join(tmpdir, ec_file) f = open(ec_path, 'a') f.write("\ndependencies += [('nosuchsoftware', '1.2.3')]\n") f.close() ec = EasyConfig(ec_path) eb = EasyBlock(ec) try: eb.check_readiness_step() except EasyBuildError, err: err_regex = re.compile("Missing modules for one or more dependencies: nosuchsoftware/1.2.3-GCC-4.6.4") self.assertTrue(err_regex.search(str(err)), "Pattern '%s' found in '%s'" % (err_regex.pattern, err))
def test_package(self): """Test package function.""" init_config(build_options={'silent': True}) test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False) mock_fpm(self.test_prefix) # import needs to be done here, since test easyblocks are only included later from easybuild.easyblocks.toy import EB_toy easyblock = EB_toy(ec) # build & install first easyblock.run_all_steps(False) # package using default packaging configuration (FPM to build RPM packages) pkgdir = package(easyblock) pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION) self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile) pkgtxt = read_file(pkgfile) pkgtxt_regex = re.compile("Contents of installdir %s" % easyblock.installdir) self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt)) if DEBUG: print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
def test_copy_file(self): """ Test copy_file """ testdir = os.path.dirname(os.path.abspath(__file__)) tmpdir = self.test_prefix to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') target_path = os.path.join(tmpdir, 'toy.eb') ft.copy_file(to_copy, target_path) self.assertTrue(os.path.exists(target_path)) self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) # also test behaviour of extract_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) # remove target file, it shouldn't get copied under dry run os.remove(target_path) self.mock_stdout(True) ft.copy_file(to_copy, target_path) txt = self.get_stdout() self.mock_stdout(False) self.assertFalse(os.path.exists(target_path)) self.assertTrue(re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt))
def test_close_pr(self): """Test close_pr function.""" if self.skip_github_tests: print "Skipping test_close_pr, no GitHub token available?" return build_options = { 'dry_run': True, 'github_user': GITHUB_TEST_ACCOUNT, 'pr_target_account': GITHUB_USER, 'pr_target_repo': GITHUB_REPO, } init_config(build_options=build_options) self.mock_stdout(True) gh.close_pr(2, motivation_msg='just a test') stdout = self.get_stdout() self.mock_stdout(False) patterns = [ "hpcugent/testrepository PR #2 was submitted by migueldiascosta", "[DRY RUN] Adding comment to testrepository issue #2: '" + "@migueldiascosta, this PR is being closed for the following reason(s): just a test", "[DRY RUN] Closed hpcugent/testrepository pull request #2", ] for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout))
def test_log_file_format(self): """Test for log_file_format().""" # first test defaults -> no templating when no values are provided self.assertEqual(log_file_format(), 'easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log') self.assertEqual(log_file_format(return_directory=True), 'easybuild') # test whether provided values are used to complete template ec = {'name': 'foo', 'version': '1.2.3'} res = log_file_format(ec=ec, date='20190322', timestamp='094356') self.assertEqual(res, 'easybuild-foo-1.2.3-20190322.094356.log') res = log_file_format(return_directory=True, ec=ec, date='20190322', timestamp='094356') self.assertEqual(res, 'easybuild') # partial templating is done when only some values are provided... self.assertEqual(log_file_format(ec=ec), 'easybuild-foo-1.2.3-%(date)s.%(time)s.log') res = log_file_format(date='20190322', timestamp='094356') self.assertEqual(res, 'easybuild-%(name)s-%(version)s-20190322.094356.log') # also try with a custom setting init_config(args=['--logfile-format=eb-%(name)s-%(date)s,log-%(version)s-%(date)s-%(time)s.out']) self.assertEqual(log_file_format(), 'log-%(version)s-%(date)s-%(time)s.out') self.assertEqual(log_file_format(return_directory=True), 'eb-%(name)s-%(date)s') res = log_file_format(ec=ec, date='20190322', timestamp='094356') self.assertEqual(res, 'log-1.2.3-20190322-094356.out') res = log_file_format(return_directory=True, ec=ec, date='20190322', timestamp='094356') self.assertEqual(res, 'eb-foo-20190322') # test handling of incorrect setting for --logfile-format init_config(args=['--logfile-format=easybuild,log.txt,thisiswrong']) error_pattern = "Incorrect log file format specification, should be 2-tuple" self.assertErrorRegex(EasyBuildError, error_pattern, log_file_format)
def test_check_capability_mapping(self): """Test comparing the functionality of two toolchains""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) get_toolchain_hierarchy.clear() foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '2016.01'}, incl_capabilities=True) # Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2 self.assertEqual(foss_hierarchy[0]['name'], 'GCC') self.assertEqual(foss_hierarchy[1]['name'], 'golf') self.assertEqual(foss_hierarchy[2]['name'], 'gompi') self.assertEqual(foss_hierarchy[3]['name'], 'foss') self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore') self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort') self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi') # golf <-> iimpi (should return False) self.assertFalse(check_capability_mapping(foss_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs") # gompi <-> iimpi self.assertTrue(check_capability_mapping(foss_hierarchy[2], iimpi_hierarchy[2])) # GCC <-> iimpi self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[2])) # GCC <-> iccifort self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[1])) # GCC <-> GCCcore self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[0]))
def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'robot_path': test_easyconfigs, 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): self.assertTrue(key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key])
def test_setvar(self): """Test setvar function.""" self.mock_stdout(True) env.setvar('FOO', 'bar') txt = self.get_stdout() self.mock_stdout(False) self.assertEqual(os.getenv('FOO'), 'bar') self.assertEqual(os.environ['FOO'], 'bar') # no printing if dry run is not enabled self.assertEqual(txt, '') build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) self.mock_stdout(True) env.setvar('FOO', 'foobaz') txt = self.get_stdout() self.mock_stdout(False) self.assertEqual(os.getenv('FOO'), 'foobaz') self.assertEqual(os.environ['FOO'], 'foobaz') self.assertEqual(txt, " export FOO=\"foobaz\"\n") # disabling verbose self.mock_stdout(True) env.setvar('FOO', 'barfoo', verbose=False) txt = self.get_stdout() self.mock_stdout(False) self.assertEqual(os.getenv('FOO'), 'barfoo') self.assertEqual(os.environ['FOO'], 'barfoo') self.assertEqual(txt, '')
def test_tweak_one_version(self): """Test tweak_one function""" test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb') # test tweaking of software version (--try-software-version) tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb') tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict() tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() # checksums should be reset to empty list, only version should be changed, nothing else self.assertEqual(tweaked_toy_ec_parsed['checksums'], []) self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]: val = toy_ec_parsed[key] self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) tweaked_val = tweaked_toy_ec_parsed.get(key) self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val)) # check behaviour if target file already exists error_pattern = "File exists, not overwriting it without --force" self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) # existing file does get overwritten when --force is used init_config(build_options={'force': True, 'silent': True}) write_file(tweaked_toy_ec, '') tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
def test_path_to_top_of_module_tree_categorized_hmns(self): """ Test function to determine path to top of the module tree for a categorized hierarchical module naming scheme. """ ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, 'robot_path': [ecs_dir], 'valid_stops': all_stops, 'validate': False, } os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' init_config(build_options=build_options) self.setup_categorized_hmns_modules() modtool = modules_tool() mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') init_modpaths = [os.path.join(mod_prefix, 'Core', 'compiler'), os.path.join(mod_prefix, 'Core', 'toolchain')] deps = ['GCC/4.7.2', 'OpenMPI/1.6.4', 'FFTW/3.3.3', 'OpenBLAS/0.2.6-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2'] path = modtool.path_to_top_of_module_tree(init_modpaths, 'goolf/1.4.10', os.path.join(mod_prefix, 'Core', 'toolchain'), deps) self.assertEqual(path, []) path = modtool.path_to_top_of_module_tree(init_modpaths, 'GCC/4.7.2', os.path.join(mod_prefix, 'Core', 'compiler'), []) self.assertEqual(path, []) full_mod_subdir = os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'mpi') deps = ['GCC/4.7.2', 'hwloc/1.6.2'] path = modtool.path_to_top_of_module_tree(init_modpaths, 'OpenMPI/1.6.4', full_mod_subdir, deps) self.assertEqual(path, ['GCC/4.7.2']) full_mod_subdir = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4', 'numlib') deps = ['GCC/4.7.2', 'OpenMPI/1.6.4'] path = modtool.path_to_top_of_module_tree(init_modpaths, 'FFTW/3.3.3', full_mod_subdir, deps) self.assertEqual(path, ['OpenMPI/1.6.4', 'GCC/4.7.2'])
def test_external_modules_metadata(self): """Test --external-modules-metadata.""" # empty list by default cfg = init_config() self.assertEqual(cfg.external_modules_metadata, []) testcfgtxt = EXTERNAL_MODULES_METADATA testcfg = os.path.join(self.test_prefix, 'test_external_modules_metadata.cfg') write_file(testcfg, testcfgtxt) cfg = init_config(args=['--external-modules-metadata=%s' % testcfg]) netcdf = { 'name': ['netCDF', 'netCDF-Fortran'], 'version': ['4.3.2', '4.3.2'], 'prefix': 'NETCDF_DIR', } self.assertEqual(cfg.external_modules_metadata['cray-netcdf/4.3.2'], netcdf) hdf5 = { 'name': ['HDF5'], 'version': ['1.8.13'], 'prefix': 'HDF5_DIR', } self.assertEqual(cfg.external_modules_metadata['cray-hdf5/1.8.13'], hdf5) # impartial metadata is fine self.assertEqual(cfg.external_modules_metadata['foo'], {'name': ['Foo'], 'prefix': '/foo'}) self.assertEqual(cfg.external_modules_metadata['bar/1.2.3'], {'name': ['bar'], 'version': ['1.2.3']}) # if both names and versions are specified, lists must have same lengths write_file(testcfg, '\n'.join(['[foo/1.2.3]', 'name = foo,bar', 'version = 1.2.3'])) args = ['--external-modules-metadata=%s' % testcfg] err_msg = "Different length for lists of names/versions in metadata for external module" self.assertErrorRegex(EasyBuildError, err_msg, init_config, args=args)
def test_exts_list(self): """Test handling of list of extensions.""" os.environ['EASYBUILD_SOURCEPATH'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') init_config() self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'exts_list = [', ' ("ext1", "ext_ver1", {', ' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail ' "source_urls": ["http://example.com/"]', ' }),', ' ("ext2", "ext_ver2", {', ' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail ' "source_urls": [("http://example.com", "suffix")],' ' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail ' "checksums": [', ' "504c7036558938f997c1c269a01d7458",', # checksum for source (gzip-1.4.eb) ' "ddd5161154f5db67701525123129ff09",', # checksum for patch (toy-0.0.eb) ' ],', ' }),', ']', ]) self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) exts_sources = eb.fetch_extension_sources()
def test_remove_file(self): """Test remove_file""" testfile = os.path.join(self.test_prefix, 'foo') ft.write_file(testfile, 'bar') self.assertTrue(os.path.exists(testfile)) ft.remove_file(testfile) ft.write_file(testfile, 'bar') ft.adjust_permissions(self.test_prefix, stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH, add=False) self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile) # also test behaviour of remove_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) self.mock_stdout(True) ft.remove_file(testfile) txt = self.get_stdout() self.mock_stdout(False) regex = re.compile("^file [^ ]* removed$") self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def test_independence(self): """Test independency of toolchain instances.""" # tweaking --optarch is required for Cray toolchains (craypre-<optarch> module must be available) init_config(build_options={'optarch': 'test'}) tc_cflags = { 'CrayCCE': "-craype-verbose -O2", 'CrayGNU': "-craype-verbose -O2", 'CrayIntel': "-craype-verbose -O2 -ftz -fp-speculation=safe -fp-model source", 'GCC': "-O2 -test", 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source", } toolchains = [ ('CrayCCE', '2015.06-XC'), ('CrayGNU', '2015.06-XC'), ('CrayIntel', '2015.06-XC'), ('GCC', '4.7.2'), ('iccifort', '2011.13.367'), ] # purposely obtain toolchains several times in a row, value for $CFLAGS should not change for _ in range(3): for tcname, tcversion in toolchains: tc = get_toolchain({'name': tcname, 'version': tcversion}, {}, mns=ActiveMNS(), modtool=self.modtool) tc.set_options({}) tc.prepare() expected_cflags = tc_cflags[tcname] msg = "Expected $CFLAGS found for toolchain %s: %s" % (tcname, expected_cflags) self.assertEqual(str(tc.variables['CFLAGS']), expected_cflags, msg) self.assertEqual(os.environ['CFLAGS'], expected_cflags, msg)
def test_lmod_specific(self): """Lmod-specific test (skipped unless Lmod is used as modules tool).""" lmod_abspath = which(Lmod.COMMAND) # only run this test if 'lmod' is available in $PATH if lmod_abspath is not None: build_options = { 'allow_modules_tool_mismatch': True, } init_config(build_options=build_options) # drop any location where 'lmod' or 'spider' can be found from $PATH paths = os.environ.get('PATH', '').split(os.pathsep) new_paths = [] for path in paths: lmod_cand_path = os.path.join(path, Lmod.COMMAND) spider_cand_path = os.path.join(path, 'spider') if not os.path.isfile(lmod_cand_path) and not os.path.isfile(spider_cand_path): new_paths.append(path) os.environ['PATH'] = os.pathsep.join(new_paths) # make sure $MODULEPATH contains path that provides some modules os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) # initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD os.environ['LMOD_CMD'] = lmod_abspath lmod = Lmod(testing=True) # obtain list of availabe modules, should be non-empty self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty") # test updating local spider cache (but don't actually update the local cache file!) self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")
def test_generaloption_config(self): """Test new-style configuration (based on generaloption).""" self.purge_environment() # check whether configuration via environment variables works as expected prefix = os.path.join(self.tmpdir, 'testprefix') buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build', 'path') os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var options = init_config(args=[]) self.assertEqual(build_path(), buildpath_env_var) self.assertEqual(install_path(), os.path.join(prefix, 'software')) self.assertEqual(get_repositorypath(), [os.path.join(prefix, 'ebfiles_repo')]) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_BUILDPATH'] # check whether configuration via command line arguments works prefix = os.path.join(self.tmpdir, 'test1') install = os.path.join(self.tmpdir, 'test2', 'install') repopath = os.path.join(self.tmpdir, 'test2', 'repo') config_file = os.path.join(self.tmpdir, 'nooldconfig.py') write_file(config_file, '') args = [ '--config', config_file, # force empty oldstyle config file '--prefix', prefix, '--installpath', install, '--repositorypath', repopath, ] options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, 'software')) self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) self.assertEqual(options.config, config_file) # check mixed command line/env var configuration prefix = os.path.join(self.tmpdir, 'test3') install = os.path.join(self.tmpdir, 'test4', 'install') subdir_software = 'eb-soft' args = [ '--config', config_file, # force empty oldstyle config file '--installpath', install, ] os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, subdir_software)) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" # put fake 'gcc' command in place that just echos its arguments fake_gcc = os.path.join(self.test_prefix, 'fake', 'gcc') write_file(fake_gcc, '#!/bin/bash\necho "$@"') adjust_permissions(fake_gcc, stat.S_IXUSR) os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', '')) # enable --rpath and prepare toolchain init_config(build_options={'rpath': True, 'rpath_filter': ['/ba.*']}) tc = self.get_toolchain('gompi', version='1.3.12') # preparing RPATH wrappers requires --experimental, need to bypass that here tc.log.experimental = lambda x: x tc.prepare() # check whether fake gcc was wrapped and that arguments are what they should be # no -rpath for /bar because of rpath filter out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""') expected = ' '.join([ '-Wl,-rpath=$ORIGIN/../lib', '-Wl,-rpath=$ORIGIN/../lib64', '-Wl,--disable-new-dtags', '-Wl,-rpath=/foo', '%(user)s.c', '-L/foo', '-L/bar', '$FOO', '-DX=""', ]) self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
def test_prepare_step(self): """Test prepare step (setting up build environment).""" test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'dummy-dummy'), parents=True) eb = EasyBlock(ec['ec']) eb.silent = True eb.prepare_step() self.assertEqual(self.modtool.list(), []) os.environ['THIS_IS_AN_UNWANTED_ENV_VAR'] = 'foo' eb.cfg['unwanted_env_vars'] = ['THIS_IS_AN_UNWANTED_ENV_VAR'] eb.cfg['allow_system_deps'] = [('Python', '1.2.3')] init_config(build_options={'extra_modules': ['GCC/4.7.2']}) eb.prepare_step() self.assertEqual(os.environ.get('THIS_IS_AN_UNWANTED_ENV_VAR'), None) self.assertEqual(os.environ.get('EBROOTPYTHON'), 'Python') self.assertEqual(os.environ.get('EBVERSIONPYTHON'), '1.2.3') self.assertEqual(len(self.modtool.list()), 1) self.assertEqual(self.modtool.list()[0]['mod_name'], 'GCC/4.7.2')
def test_robot_archived_easyconfigs(self): """Test whether robot can pick up archived easyconfigs when asked.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-ictce-4.1.13.eb') gzip_ectxt = read_file(gzip_ec) test_ec = os.path.join(self.test_prefix, 'test.eb') tc_spec = "toolchain = {'name': 'ictce', 'version': '3.2.2.u3'}" regex = re.compile("^toolchain = .*", re.M) test_ectxt = regex.sub(tc_spec, gzip_ectxt) write_file(test_ec, test_ectxt) ecs, _ = parse_easyconfigs([(test_ec, False)]) self.assertErrorRegex(EasyBuildError, "Irresolvable dependencies encountered", resolve_dependencies, ecs, self.modtool, retain_all_deps=True) # --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs init_config(build_options={ 'consider_archived_easyconfigs': True, 'robot_path': [test_ecs], }) res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True) self.assertEqual([ec['full_mod_name'] for ec in res], ['ictce/3.2.2.u3', 'gzip/1.5-ictce-3.2.2.u3']) expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb') self.assertTrue(os.path.samefile(res[0]['spec'], expected))
def test_module_mismatch(self): """Test whether mismatch detection between modules tool and 'module' function works.""" # redefine 'module' function (deliberate mismatch with used module command in MockModulesTool) os.environ["module"] = "() { eval `/tmp/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}" error_regex = ".*pattern .* not found in defined 'module' function" self.assertErrorRegex(EasyBuildError, error_regex, MockModulesTool, testing=True) # check whether escaping error by allowing mismatch via build options works build_options = {"allow_modules_tool_mismatch": True} init_config(build_options=build_options) fancylogger.logToFile(self.logfile) mt = MockModulesTool(testing=True) f = open(self.logfile, "r") logtxt = f.read() f.close() warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function") self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt)) # redefine 'module' function with correct module command os.environ["module"] = "() { eval `/bin/echo $*`\n}" mt = MockModulesTool(testing=True) self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage # a warning should be logged if the 'module' function is undefined del os.environ["module"] mt = MockModulesTool(testing=True) f = open(self.logfile, "r") logtxt = f.read() f.close() warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*") self.assertTrue(warn_regex.search(logtxt), "Pattern %s found in %s" % (warn_regex.pattern, logtxt)) fancylogger.logToFile(self.logfile, enable=False)
def test_read_write_file(self): """Test reading/writing files.""" fp = os.path.join(self.test_prefix, 'test.txt') txt = "test123" ft.write_file(fp, txt) self.assertEqual(ft.read_file(fp), txt) txt2 = '\n'.join(['test', '123']) ft.write_file(fp, txt2, append=True) self.assertEqual(ft.read_file(fp), txt+txt2) # also test behaviour of write_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) foo = os.path.join(self.test_prefix, 'foo.txt') self.mock_stdout(True) ft.write_file(foo, 'bar') txt = self.get_stdout() self.mock_stdout(False) self.assertFalse(os.path.exists(foo)) self.assertTrue(re.match("^file written: .*/foo.txt$", txt)) ft.write_file(foo, 'bar', forced=True) self.assertTrue(os.path.exists(foo)) self.assertEqual(ft.read_file(foo), 'bar')
def test_hierarchical_mns(self): """Test hierarchical module naming scheme.""" ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, 'robot_path': [ecs_dir], 'valid_stops': all_stops, 'validate': False, } def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, init_modpaths): """Test whether active module naming scheme returns expected values.""" ec = EasyConfig(os.path.join(ecs_dir, ecfile)) self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname)) self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname) self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir) self.assertEqual(ActiveMNS().det_modpath_extensions(ec), modpath_exts) self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' init_config(build_options=build_options) # format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions) iccver = '2013.5.192-GCC-4.8.3' impi_ec = 'impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb' imkl_ec = 'imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb' test_ecs = { 'GCC-4.7.2.eb': ('GCC/4.7.2', 'Core', ['Compiler/GCC/4.7.2'], ['Core']), 'OpenMPI-1.6.4-GCC-4.7.2.eb': ('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2', ['MPI/GCC/4.7.2/OpenMPI/1.6.4'], ['Core']), 'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4', [], ['Core']), 'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core', [], ['Core']), 'icc-2013.5.192-GCC-4.8.3.eb': ('icc/%s' % iccver, 'Core', ['Compiler/intel/%s' % iccver], ['Core']), 'ifort-2013.3.163.eb': ('ifort/2013.3.163', 'Core', ['Compiler/intel/2013.3.163'], ['Core']), 'CUDA-5.5.22-GCC-4.8.2.eb': ('CUDA/5.5.22', 'Compiler/GCC/4.8.2', ['Compiler/GCC-CUDA/4.8.2-5.5.22'], ['Core']), impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s' % iccver, ['MPI/intel/%s/impi/4.1.3.049' % iccver], ['Core']), imkl_ec: ('imkl/11.1.2.144', 'MPI/intel/%s/impi/4.1.3.049' % iccver, [], ['Core']), } for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals) # impi with dummy toolchain, which doesn't make sense in a hierarchical context ec = EasyConfig(os.path.join(ecs_dir, 'impi-4.1.3.049.eb')) self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme init_config(build_options=build_options) test_ecs = { 'GCC-4.7.2.eb': ('GCC/4.7.2', '', [], []), 'OpenMPI-1.6.4-GCC-4.7.2.eb': ('OpenMPI/1.6.4-GCC-4.7.2', '', [], []), 'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5-goolf-1.4.10', '', [], []), 'goolf-1.4.10.eb': ('goolf/1.4.10', '', [], []), 'impi-4.1.3.049.eb': ('impi/4.1.3.049', '', [], []), } for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals)
def test_strict(self): """Test use of --strict.""" # check default self.assertEqual(build_option('strict'), run.WARN) for strict_str, strict_val in [('error', run.ERROR), ('ignore', run.IGNORE), ('warn', run.WARN)]: options = init_config(args=['--strict=%s' % strict_str]) init_config(build_options={'strict': options.strict}) self.assertEqual(build_option('strict'), strict_val)
def test_exclude_path_to_top_of_module_tree(self): """ Make sure that modules under the HierarchicalMNS are correct, w.r.t. not including any load statements for modules that build up the path to the top of the module tree. """ self.orig_module_naming_scheme = config.get_module_naming_scheme() test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, 'robot_path': [test_ecs_path], 'valid_stops': all_stops, 'validate': False, } os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' init_config(build_options=build_options) self.setup_hierarchical_modules() modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all') mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True) mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True) impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049') imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144') if get_module_syntax() == 'Lua': impi_modfile_path += '.lua' imkl_modfile_path += '.lua' # example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included # not for the toolchain or any of the toolchain components, # since both icc/ifort and impi form the path to the top of the module tree tests = [ ('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']), ('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']), ] for ec_file, modfile_path, excluded_deps in tests: ec = EasyConfig(os.path.join(test_ecs_path, ec_file)) eb = EasyBlock(ec) eb.toolchain.prepare() modpath = eb.make_module_step() modfile_path = os.path.join(modpath, modfile_path) modtxt = read_file(modfile_path) for dep in excluded_deps: tup = (dep, modfile_path, modtxt) failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup if get_module_syntax() == 'Tcl': self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg) elif get_module_syntax() == 'Lua': self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme init_config(build_options=build_options)
def test_modules_tool_stateless(self): """Check whether ModulesTool instance is stateless between runs.""" test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') # copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load shutil.copytree(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core')) shutil.copytree(os.path.join(test_modules_path, 'Compiler'), os.path.join(self.test_prefix, 'Compiler')) modtxt = read_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2')) modpath_extension = os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2') modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M) write_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'), modtxt) modtxt = read_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4')) modpath_extension = os.path.join(self.test_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4') mkdir(modpath_extension, parents=True) modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M) write_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), modtxt) # force reset of any singletons by reinitiating config init_config() os.environ['MODULEPATH'] = os.path.join(self.test_prefix, 'Core') modtool = modules_tool() if isinstance(modtool, Lmod): load_err_msg = "cannot[\s\n]*be[\s\n]*loaded" else: load_err_msg = "Unable to locate a modulefile" # GCC/4.6.3 is *not* an available Core module self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['GCC/4.6.3']) # GCC/4.7.2 is one of the available Core modules modtool.load(['GCC/4.7.2']) # OpenMPI/1.6.4 becomes available after loading GCC/4.7.2 module modtool.load(['OpenMPI/1.6.4']) modtool.purge() # reset $MODULEPATH, obtain new ModulesTool instance, # which should not remember anything w.r.t. previous $MODULEPATH value os.environ['MODULEPATH'] = test_modules_path modtool = modules_tool() # GCC/4.6.3 is available modtool.load(['GCC/4.6.3']) modtool.purge() # GCC/4.7.2 is available (note: also as non-Core module outside of hierarchy) modtool.load(['GCC/4.7.2']) # OpenMPI/1.6.4 is *not* available with current $MODULEPATH (loaded GCC/4.7.2 was not a hierarchical module) self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['OpenMPI/1.6.4'])
def test_find_minimally_resolved_modules(self): """Test find_minimally_resolved_modules function.""" # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_txt = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", "toolchain = {'name': 'goolf', 'version': '1.4.10'}", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2 " ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('SQLite', '3.8.10.2'),", # only available with goolf/1.4.10 "]", ]) write_file(barec, barec_txt) bar = process_easyconfig(barec)[0] ecs = [bar] mods = [ 'gompi/1.4.10', 'goolf/1.4.10', # include modules for dependencies, with subtoolchains rather than full toolchain (except for SQLite) 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', 'SQLite/3.8.10.2-GCC-4.7.2', ] ordered_ecs, new_easyconfigs, new_avail_modules = find_minimally_resolved_modules(ecs, mods, []) # all dependencies are resolved for easyconfigs included in ordered_ecs self.assertEqual(len(ordered_ecs), 1) self.assertEqual(ordered_ecs[0]['dependencies'], []) # module is added to list of available modules self.assertTrue(bar['ec'].full_mod_name in new_avail_modules) # nothing left self.assertEqual(new_easyconfigs, [])
def test_wrong_modulepath(self): """Test whether modules tool can deal with a broken $MODULEPATH.""" test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') modules_test_installpath = os.path.join(self.test_installpath, 'modules', 'all') os.environ['MODULEPATH'] = "/some/non-existing/path:/this/doesnt/exists/anywhere:%s" % test_modules_path init_config() modtool = modules_tool() self.assertEqual(len(modtool.mod_paths), 2) self.assertTrue(os.path.samefile(modtool.mod_paths[0], modules_test_installpath)) self.assertEqual(modtool.mod_paths[1], test_modules_path) self.assertTrue(len(modtool.available()) > 0)
def test_list_prs(self): """Test list_prs function.""" if self.skip_github_tests: print("Skipping test_list_prs, no GitHub token available?") return parameters = ('closed', 'created', 'asc') init_config(build_options={'pr_target_account': GITHUB_USER, 'pr_target_repo': GITHUB_REPO}) expected = "PR #1: a pr" self.mock_stdout(True) output = gh.list_prs(parameters, per_page=1, github_user=GITHUB_TEST_ACCOUNT) stdout = self.get_stdout() self.mock_stdout(False) self.assertTrue(stdout.startswith("== Listing PRs with parameters: ")) self.assertEqual(expected, output)
def test_load(self): """Test load part in generated module file.""" expected = [ "", "if { ![is-loaded mod_name] } {", " module load mod_name", "}", "", ] self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) # with recursive unloading: no if is-loaded guard init_config(build_options={'recursive_mod_unload': True}) expected = [ "", "module load mod_name", "", ] self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name"))
def test_find_patches(self): """ Test for find_software_name_for_patch """ testdir = os.path.dirname(os.path.abspath(__file__)) ec_path = os.path.join(testdir, 'easyconfigs') init_config(build_options={ 'allow_modules_tool_mismatch': True, 'minimal_toolchains': True, 'use_existing_modules': True, 'external_modules_metadata': ConfigObj(), 'silent': True, 'valid_module_classes': module_classes(), 'validate': False, }) self.mock_stdout(True) ec = gh.find_software_name_for_patch('toy-0.0_fix-silly-typo-in-printf-statement.patch', [ec_path]) txt = self.get_stdout() self.mock_stdout(False) self.assertTrue(ec == 'toy') reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked') self.assertTrue(re.search(reg, txt))
def test_load(self): """Test load part in generated module file.""" if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: expected = [ '', "if { ![ is-loaded mod_name ] } {", " module load mod_name", "}", '', ] self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) # with recursive unloading: no if is-loaded guard init_config(build_options={'recursive_mod_unload': True}) expected = [ '', "module load mod_name", '', ] self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) else: expected = '\n'.join([ '', 'if not isloaded("mod_name") then', ' load("mod_name")', 'end', '', ]) self.assertEqual(expected, self.modgen.load_module("mod_name")) init_config(build_options={'recursive_mod_unload': True}) expected = '\n'.join([ '', 'load("mod_name")', '', ]) self.assertEqual(expected, self.modgen.load_module("mod_name"))
def test_install_path(self): """Test install_path function.""" # defaults self.assertEqual(install_path(), os.path.join(self.test_installpath, 'software')) self.assertEqual(install_path('software'), os.path.join(self.test_installpath, 'software')) self.assertEqual(install_path(typ='mod'), os.path.join(self.test_installpath, 'modules')) self.assertEqual(install_path('modules'), os.path.join(self.test_installpath, 'modules')) self.assertErrorRegex(EasyBuildError, "Unknown type specified", install_path, typ='foo') args = [ '--subdir-software', 'SOFT', '--installpath', '/foo', ] os.environ['EASYBUILD_SUBDIR_MODULES'] = 'MOD' init_config(args=args) self.assertEqual(install_path(), os.path.join('/foo', 'SOFT')) self.assertEqual(install_path(typ='mod'), os.path.join('/foo', 'MOD')) del os.environ['EASYBUILD_SUBDIR_MODULES'] args = [ '--installpath', '/prefix', '--installpath-modules', '/foo', ] os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] = '/bar/baz' init_config(args=args) self.assertEqual(install_path(), os.path.join('/bar', 'baz')) self.assertEqual(install_path(typ='mod'), '/foo') del os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] init_config(args=args) self.assertEqual(install_path(), os.path.join('/prefix', 'software')) self.assertEqual(install_path(typ='mod'), '/foo')
def test_independence(self): """Test independency of toolchain instances.""" # tweaking --optarch is required for Cray toolchains (craypre-<optarch> module must be available) init_config(build_options={'optarch': 'test'}) tc_cflags = { 'CrayCCE': "-craype-verbose -O2", 'CrayGNU': "-craype-verbose -O2", 'CrayIntel': "-craype-verbose -O2 -ftz -fp-speculation=safe -fp-model source", 'GCC': "-O2 -test", 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source", } toolchains = [ ('CrayCCE', '2015.06-XC'), ('CrayGNU', '2015.06-XC'), ('CrayIntel', '2015.06-XC'), ('GCC', '4.7.2'), ('iccifort', '2011.13.367'), ] # purposely obtain toolchains several times in a row, value for $CFLAGS should not change for _ in range(3): for tcname, tcversion in toolchains: tc = get_toolchain({ 'name': tcname, 'version': tcversion }, {}, mns=ActiveMNS(), modtool=self.modtool) tc.set_options({}) tc.prepare() expected_cflags = tc_cflags[tcname] msg = "Expected $CFLAGS found for toolchain %s: %s" % ( tcname, expected_cflags) self.assertEqual(str(tc.variables['CFLAGS']), expected_cflags, msg) self.assertEqual(os.environ['CFLAGS'], expected_cflags, msg)
def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" test_easyconfigs = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config( build_options={ 'robot_path': test_easyconfigs, 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = { 'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25' } # The below mapping includes a binutils mapping (2.26 to 2.25) tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') tweaked_spec = map_easyconfig_to_target_tc_hierarchy( ec_spec, tc_mapping) tweaked_ec = process_easyconfig(tweaked_spec)[0] tweaked_dict = tweaked_ec['ec'].asdict() # First check the mapped toolchain key, value = 'toolchain', iccifort_binutils_tc self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in { 'name': 'binutils', 'version': '2.25', 'versionsuffix': '' }.items(): self.assertTrue( key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key])
def test_package(self): """Test package function.""" init_config(build_options={'silent': True}) test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False) mock_fpm(self.test_prefix) # import needs to be done here, since test easyblocks are only included later from easybuild.easyblocks.toy import EB_toy easyblock = EB_toy(ec) # build & install first easyblock.run_all_steps(False) # write a dummy log and report file to make sure they don't get packaged logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log") write_file(logfile, "I'm a logfile") reportfile = os.path.join(easyblock.installdir, log_path(), "report.md") write_file(reportfile, "I'm a reportfile") # package using default packaging configuration (FPM to build RPM packages) pkgdir = package(easyblock) pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION) self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile) pkgtxt = read_file(pkgfile) pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" % easyblock.installdir) self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt)) no_logfiles_regex = re.compile(r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS', re.DOTALL|re.MULTILINE) self.assertFalse(no_logfiles_regex.search(pkgtxt), "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt)) if DEBUG: print "The FPM script debug output" print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE)) print "The Package File" print read_file(pkgfile)
def test_optarch_generic(self): """Test whether --optarch=GENERIC works as intended.""" for generic in [False, True]: if generic: build_options = {'optarch': 'GENERIC'} init_config(build_options=build_options) flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] tcs = { 'gompi': ('1.3.12', "-march=x86-64 -mtune=generic"), 'iccifort': ('2011.13.367', "-xSSE2 -ftz -fp-speculation=safe -fp-model source"), } for tcopt_optarch in [False, True]: for tcname in tcs: tcversion, generic_flags = tcs[tcname] tc = self.get_toolchain(tcname, version=tcversion) tc.set_options({'optarch': tcopt_optarch}) tc.prepare() for var in flag_vars: if generic: self.assertTrue(generic_flags in tc.get_variable(var)) else: self.assertFalse(generic_flags in tc.get_variable(var))
def test_dep_tree_of_toolchain(self): """Test getting list of dependencies of a toolchain (as EasyConfig objects)""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, 'check_osdeps': False, }) toolchain_spec = {'name': 'foss', 'version': '2018a'} list_of_deps = get_dep_tree_of_toolchain(toolchain_spec, self.modtool) expected_deps = [ ['GCC', '6.4.0'], ['OpenBLAS', '0.2.20'], ['hwloc', '1.11.8'], ['OpenMPI', '2.1.2'], ['gompi', '2018a'], ['FFTW', '3.3.7'], ['ScaLAPACK', '2.0.2'], ['foss', '2018a'] ] actual_deps = [[dep['name'], dep['version']] for dep in list_of_deps] self.assertEqual(expected_deps, actual_deps)
def test_reasons_for_closing(self): """Test reasons_for_closing function.""" if self.skip_github_tests: print("Skipping test_reasons_for_closing, no GitHub token available?") return repo_owner = gh.GITHUB_EB_MAIN repo_name = gh.GITHUB_EASYCONFIGS_REPO build_options = { 'dry_run': True, 'github_user': GITHUB_TEST_ACCOUNT, 'pr_target_account': repo_owner, 'pr_target_repo': repo_name, 'robot_path': [], } init_config(build_options=build_options) pr_data, _ = gh.fetch_pr_data(1844, repo_owner, repo_name, GITHUB_TEST_ACCOUNT, full=True) self.mock_stdout(True) self.mock_stderr(True) # can't easily check return value, since auto-detected reasons may change over time if PR is touched res = gh.reasons_for_closing(pr_data) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) self.assertTrue(isinstance(res, list)) self.assertEqual(stderr.strip(), "WARNING: Using easyconfigs from closed PR #1844") patterns = [ "Status of last commit is SUCCESS", "Last comment on", "No activity since", "* QEMU-2.4.0", ] for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout))
def test_make_module_dep_of_dep_hmns(self): """Test for make_module_dep under HMNS with dependencies of dependencies""" test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, 'robot_path': [test_ecs_path], 'valid_stops': all_stops, 'validate': False, } os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' init_config(build_options=build_options) self.setup_hierarchical_modules() # GCC and OpenMPI extend $MODULEPATH; hwloc is a dependency of OpenMPI. self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', "toolchain = {'name': 'goolf', 'version': '1.4.10'}", 'dependencies = [', " ('GCC', '4.7.2', '', True)," " ('hwloc', '1.6.2', '', ('GCC', '4.7.2')),", " ('OpenMPI', '1.6.4', '', ('GCC', '4.7.2'))," ']', ]) self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') eb.check_readiness_step() # GCC, OpenMPI and hwloc modules should *not* be included in loads for dependencies mod_dep_txt = eb.make_module_dep() for mod in ['GCC/4.7.2', 'OpenMPI/1.6.4', 'hwloc/1.6.2']: regex = re.compile('load.*%s' % mod) self.assertFalse(regex.search(mod_dep_txt), "Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
def test_run_hook(self): """Test for run_hook function.""" hooks = load_hooks(self.test_hooks_pymod) init_config(build_options={'debug': True}) self.mock_stdout(True) self.mock_stderr(True) run_hook('start', hooks) run_hook('parse', hooks, args=['<EasyConfig instance>'], msg="Running parse hook for example.eb...") run_hook('configure', hooks, pre_step_hook=True, args=[None]) run_hook('configure', hooks, post_step_hook=True, args=[None]) run_hook('build', hooks, pre_step_hook=True, args=[None]) run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) run_hook('install', hooks, post_step_hook=True, args=[None]) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) expected_stdout = '\n'.join([ "== Running start hook...", "this is triggered at the very beginning", "== Running parse hook for example.eb...", "Parse hook with argument <EasyConfig instance>", "== Running post-configure hook...", "this is run after configure step", "running foo helper method", "== Running pre-install hook...", "this is run before install step", ]) self.assertEqual(stdout.strip(), expected_stdout) self.assertEqual(stderr, '')
def test_det_user_modpath(self): """Test for generic det_user_modpath method.""" # None by default self.assertEqual(self.modgen.det_user_modpath(None), None) if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: self.assertEqual(self.modgen.det_user_modpath('my/own/modules'), '"my/own/modules" "all"') else: self.assertEqual(self.modgen.det_user_modpath('my/own/modules'), '"my/own/modules", "all"') # result is affected by --suffix-modules-path # {RUNTIME_ENV::FOO} gets translated into Tcl/Lua syntax for resolving $FOO at runtime init_config(build_options={'suffix_modules_path': ''}) user_modpath = 'my/{RUNTIME_ENV::TEST123}/modules' if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" $::env(TEST123) "modules"') else: self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", os.getenv("TEST123"), "modules"')
def test_check_python_version(self): """Test check_python_version function.""" init_config(build_options={'silence_deprecation_warnings': []}) def mock_python_ver(py_maj_ver, py_min_ver): """Helper function to mock a particular Python version.""" st.sys.version_info = (py_maj_ver, py_min_ver) + sys.version_info[2:] # mock running with different Python versions mock_python_ver(1, 4) error_pattern = r"EasyBuild is not compatible \(yet\) with Python 1.4" self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) mock_python_ver(4, 0) error_pattern = r"EasyBuild is not compatible \(yet\) with Python 4.0" self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) mock_python_ver(2, 6) error_pattern = r"Python 2.7 is required when using Python 2, found Python 2.6" self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) # no problems when running with a supported Python version for pyver in [(2, 7), (3, 5), (3, 6), (3, 7)]: mock_python_ver(*pyver) self.assertEqual(check_python_version(), pyver) # shouldn't raise any errors, since Python version used to run tests should be supported; self.mock_stderr(True) (py_maj_ver, py_min_ver) = check_python_version() stderr = self.get_stderr() self.mock_stderr(False) self.assertFalse(stderr) self.assertTrue(py_maj_ver in [2, 3]) if py_maj_ver == 2: self.assertTrue(py_min_ver == 7) else: self.assertTrue(py_min_ver >= 5)
def test_copy_file(self): """ Test copy_file """ testdir = os.path.dirname(os.path.abspath(__file__)) tmpdir = self.test_prefix to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') target_path = os.path.join(tmpdir, 'toy.eb') ft.copy_file(to_copy, target_path) self.assertTrue(os.path.exists(target_path)) self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) # also test behaviour of copy_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) # remove target file, it shouldn't get copied under dry run os.remove(target_path) self.mock_stdout(True) ft.copy_file(to_copy, target_path) txt = self.get_stdout() self.mock_stdout(False) self.assertFalse(os.path.exists(target_path)) self.assertTrue( re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt)) # forced copy, even in dry run mode self.mock_stdout(True) ft.copy_file(to_copy, target_path, force_in_dry_run=True) txt = self.get_stdout() self.mock_stdout(False) self.assertTrue(os.path.exists(target_path)) self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) self.assertEqual(txt, '')
def test_module_mismatch(self): """Test whether mismatch detection between modules tool and 'module' function works.""" # redefine 'module' function (deliberate mismatch with used module command in MockModulesTool) os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}" self.assertErrorRegex(EasyBuildError, ".*pattern .* not found in defined 'module' function", MockModulesTool) # check whether escaping error by allowing mismatch via build options works build_options = { 'allow_modules_tool_mismatch': True, } init_config(build_options=build_options) fancylogger.logToFile(self.logfile) mt = MockModulesTool() f = open(self.logfile, 'r') logtxt = f.read() f.close() warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function") self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt)) # redefine 'module' function with correct module command os.environ['module'] = "() { eval `/bin/echo $*`\n}" MockModulesTool._instances.pop(MockModulesTool) mt = MockModulesTool() self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage # a warning should be logged if the 'module' function is undefined del os.environ['module'] MockModulesTool._instances.pop(MockModulesTool) mt = MockModulesTool() f = open(self.logfile, 'r') logtxt = f.read() f.close() warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*") self.assertTrue(warn_regex.search(logtxt), "Pattern %s found in %s" % (warn_regex.pattern, logtxt)) fancylogger.logToFile(self.logfile, enable=False)
def test_lmod_specific(self): """Lmod-specific test (skipped unless Lmod is used as modules tool).""" lmod_abspath = which(Lmod.COMMAND) # only run this test if 'lmod' is available in $PATH if lmod_abspath is not None: build_options = { 'allow_modules_tool_mismatch': True, 'update_modules_tool_cache': True, } init_config(build_options=build_options) # drop any location where 'lmod' or 'spider' can be found from $PATH paths = os.environ.get('PATH', '').split(os.pathsep) new_paths = [] for path in paths: lmod_cand_path = os.path.join(path, Lmod.COMMAND) spider_cand_path = os.path.join(path, 'spider') if not os.path.isfile(lmod_cand_path) and not os.path.isfile( spider_cand_path): new_paths.append(path) os.environ['PATH'] = os.pathsep.join(new_paths) # make sure $MODULEPATH contains path that provides some modules os.environ['MODULEPATH'] = os.path.abspath( os.path.join(os.path.dirname(__file__), 'modules')) # initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD os.environ['LMOD_CMD'] = lmod_abspath lmod = Lmod(testing=True) # obtain list of availabe modules, should be non-empty self.assertTrue( lmod.available(), "List of available modules obtained using Lmod is non-empty") # test updating local spider cache (but don't actually update the local cache file!) self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")
def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works") # non-existing files result in None return value self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) # install broken proxy handler for opening local files # this should make urllib2.urlopen use this broken proxy for downloading from a file:// URL proxy_handler = urllib2.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) urllib2.install_opener(urllib2.build_opener(proxy_handler)) # downloading over a broken proxy results in None return value (failed download) # this tests whether proxies are taken into account by download_file self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails") # restore a working file handler, and retest download of local file urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler())) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy") # make sure specified timeout is parsed correctly (as a float, not a string) opts = init_config(args=['--download-timeout=5.3']) init_config(build_options={'download_timeout': opts.download_timeout}) target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt') url = 'https://jenkins1.ugent.be/robots.txt' try: urllib2.urlopen(url) res = ft.download_file(fn, url, target_location) self.assertEqual(res, target_location, "download with specified timeout works") except urllib2.URLError: print "Skipping timeout test in test_download_file (working offline)"
def test_external_module_toolchain(self): """Test specifying external (build) dependencies in yaml format.""" if 'yaml' not in sys.modules: print("Skipping test_external_module_toolchain (no PyYAML available)") return ecpath = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'yeb', 'CrayCCE-5.1.29.yeb') metadata = { 'name': ['foo', 'bar'], 'version': ['1.2.3', '3.2.1'], 'prefix': '/foo/bar', } build_options = { 'external_modules_metadata': {'fftw/3.3.4.0': metadata}, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) easybuild.tools.build_log.EXPERIMENTAL = True ec = EasyConfig(ecpath) self.assertEqual(ec.dependencies()[1]['full_mod_name'], 'fftw/3.3.4.0') self.assertEqual(ec.dependencies()[1]['external_module_metadata'], metadata)
def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' testdir = os.path.abspath(os.path.dirname(__file__)) sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball) tmpdir = tempfile.mkdtemp() tmpdir_subdir = os.path.join(tmpdir, 'testing') mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) # finding a file in sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)]) res = eb.obtain_file(toy_tarball) self.assertEqual(res, toy_tarball_path) # sourcepath has preference over downloading res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, toy_tarball_path) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir]) # file specifications via URL also work, are downloaded to (first) sourcepath init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)]) urls = ["http://hpcugent.github.io/easybuild/index.html", "https://hpcugent.github.io/easybuild/index.html"] for file_url in urls: fn = os.path.basename(file_url) res = None try: res = eb.obtain_file(file_url) except EasyBuildError, err: # if this fails, it should be because there's no online access download_fail_regex = re.compile('socket error') self.assertTrue(download_fail_regex.search(str(err))) # result may be None during offline testing if res is not None: loc = os.path.join(tmpdir, 't', 'toy', fn) self.assertEqual(res, loc) self.assertTrue(os.path.exists(loc), "%s file is found at %s" % (fn, loc)) txt = open(loc, 'r').read() eb_regex = re.compile("EasyBuild: building software with ease") self.assertTrue(eb_regex.search(txt)) else: print "ignoring failure to download %s in test_obtain_file, testing offline?" % file_url
def test_match_minimum_tc_specs(self): """Test matching a toolchain to lowest possible in a hierarchy""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'robot_path': test_easyconfigs, 'silent': True, 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True) iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '2016.01'}, incl_capabilities=True) # Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2 self.assertEqual(foss_hierarchy[0]['name'], 'GCC') self.assertEqual(foss_hierarchy[1]['name'], 'golf') self.assertEqual(foss_hierarchy[2]['name'], 'gompi') self.assertEqual(foss_hierarchy[3]['name'], 'foss') self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore') self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort') self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi') # base compiler first (GCCcore which maps to GCC/6.4.0-2.28) self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[0], foss_hierarchy), {'name': 'GCC', 'version': '6.4.0-2.28'}) # then iccifort (which also maps to GCC/6.4.0-2.28) self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[1], foss_hierarchy), {'name': 'GCC', 'version': '6.4.0-2.28'}) # Then MPI self.assertEqual(match_minimum_tc_specs(iimpi_hierarchy[2], foss_hierarchy), {'name': 'gompi', 'version': '2018a'}) # Check against own math only subtoolchain for math self.assertEqual(match_minimum_tc_specs(foss_hierarchy[1], foss_hierarchy), {'name': 'golf', 'version': '2018a'}) # Make sure there's an error when we can't do the mapping error_msg = "No possible mapping from source toolchain spec .*" self.assertErrorRegex(EasyBuildError, error_msg, match_minimum_tc_specs, foss_hierarchy[3], iimpi_hierarchy)
def test_lmod_specific(self): """Lmod-specific test (skipped unless Lmod is used as modules tool).""" lmod_abspath = which(Lmod.COMMAND) # only run this test if 'lmod' is available in $PATH if lmod_abspath is not None: build_options = { 'allow_modules_tool_mismatch': True, 'update_modules_tool_cache': True, } init_config(build_options=build_options) lmod = Lmod(testing=True) self.assertTrue(os.path.samefile(lmod.cmd, lmod_abspath)) # drop any location where 'lmod' or 'spider' can be found from $PATH paths = os.environ.get('PATH', '').split(os.pathsep) new_paths = [] for path in paths: lmod_cand_path = os.path.join(path, Lmod.COMMAND) spider_cand_path = os.path.join(path, 'spider') if not os.path.isfile(lmod_cand_path) and not os.path.isfile( spider_cand_path): new_paths.append(path) os.environ['PATH'] = os.pathsep.join(new_paths) # make sure $MODULEPATH contains path that provides some modules os.environ['MODULEPATH'] = os.path.abspath( os.path.join(os.path.dirname(__file__), 'modules')) # initialize Lmod modules tool, pass (fake) full path to 'lmod' via $LMOD_CMD fake_path = os.path.join(self.test_installpath, 'lmod') fake_lmod_txt = '\n'.join([ '#!/bin/bash', 'echo "Modules based on Lua: Version %s " >&2' % Lmod.REQ_VERSION, 'echo "os.environ[\'FOO\'] = \'foo\'"', ]) write_file(fake_path, fake_lmod_txt) os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR) os.environ['LMOD_CMD'] = fake_path init_config(build_options=build_options) lmod = Lmod(testing=True) self.assertTrue(os.path.samefile(lmod.cmd, fake_path)) # use correct full path for 'lmod' via $LMOD_CMD os.environ['LMOD_CMD'] = lmod_abspath init_config(build_options=build_options) lmod = Lmod(testing=True) # obtain list of availabe modules, should be non-empty self.assertTrue( lmod.available(), "List of available modules obtained using Lmod is non-empty") # test updating local spider cache (but don't actually update the local cache file!) self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")
def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" try: import gc3libs except ImportError: print "GC3Pie not available, skipping test" return # put GC3Pie config in place to use local host and fork/exec resourcedir = os.path.join(self.test_prefix, 'gc3pie') gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { 'resourcedir': resourcedir, 'time': which('time'), } write_file(gc3pie_cfgfile, gc3pie_cfgtxt) output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) mkdir(output_dir, parents=True) write_file(os.path.join(output_dir, 'foo'), 'bar') # remove write permissions on parent dir of specified output dir, # to check that GC3Pie does not try to rename the (already existing) output directory... adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', 'robot_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } options = init_config(args=['--job-backend=GC3Pie'], build_options=build_options) ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs) topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path) jobs = build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')) self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' testdir = os.path.abspath(os.path.dirname(__file__)) sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball) tmpdir = tempfile.mkdtemp() tmpdir_subdir = os.path.join(tmpdir, 'testing') mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) # finding a file in sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)]) res = eb.obtain_file(toy_tarball) self.assertEqual(res, toy_tarball_path) # sourcepath has preference over downloading res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) self.assertEqual(res, toy_tarball_path) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' try: eb.obtain_file(fn, urls=[os.path.join('file://', tmpdir_subdir)]) except EasyBuildError, err: fail_regex = re.compile("Couldn't find file %s anywhere, and downloading it didn't work either" % fn) self.assertTrue(fail_regex.search(str(err)))
def test_override_optarch(self): """Test whether overriding the optarch flag works.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] for optarch_var in ['march=lovelylovelysandybridge', None]: build_options = {'optarch': optarch_var} init_config(build_options=build_options) for enable in [True, False]: tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") tc.set_options({'optarch': enable}) tc.prepare() flag = None if optarch_var is not None: flag = '-%s' % optarch_var else: # default optarch flag flag = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch] for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue(flag in flags, "optarch: True means %s in %s" % (flag, flags)) else: self.assertFalse(flag in flags, "optarch: False means no %s in %s" % (flag, flags)) modules.modules_tool().purge()
def test_generaloption_config_file(self): """Test use of new-style configuration file.""" self.purge_environment() config_file = os.path.join(self.tmpdir, 'testconfig.cfg') testpath1 = os.path.join(self.tmpdir, 'test1') testpath2 = os.path.join(self.tmpdir, 'testtwo') # test with config file passed via command line cfgtxt = '\n'.join([ '[config]', 'installpath = %s' % testpath2, ]) write_file(config_file, cfgtxt) installpath_software = tempfile.mkdtemp(prefix='installpath-software') args = [ '--configfiles', config_file, '--debug', '--buildpath', testpath1, '--installpath-software', installpath_software, ] options = init_config(args=args) self.assertEqual(build_path(), testpath1) # via command line self.assertEqual(source_paths(), [ os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources') ]) # default self.assertEqual(install_path(), installpath_software) # via cmdline arg self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory # to check whether easyconfigs install path is auto-included in robot path tmpdir = tempfile.mkdtemp( prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert( 0, tmpdir ) # prepend to give it preference over possible other installed easyconfigs pkgs # test with config file passed via environment variable # also test for existence of HOME and USER by adding paths to robot-paths installpath_modules = tempfile.mkdtemp(prefix='installpath-modules') cfgtxt = '\n'.join([ '[config]', 'buildpath = %s' % testpath1, 'sourcepath = %(DEFAULT_REPOSITORYPATH)s', 'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir', 'robot-paths=/tmp/foo:%(sourcepath)s:%(HOME)s:/tmp/%(USER)s:%(DEFAULT_ROBOT_PATHS)s', 'installpath-modules=%s' % installpath_modules, ]) write_file(config_file, cfgtxt) os.environ['EASYBUILD_CONFIGFILES'] = config_file args = [ '--debug', '--sourcepath', testpath2, ] options = init_config(args=args) topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild') self.assertEqual(install_path(), os.path.join(topdir, 'software')) # default self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir' ]) # via config file # hardcoded first entry self.assertEqual(options.robot_paths[0], '/tmp/foo') # resolved value for %(sourcepath)s template self.assertEqual( options.robot_paths[1], os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'ebfiles_repo')) # resolved value for HOME constant self.assertEqual(options.robot_paths[2], os.getenv('HOME')) # resolved value that uses USER constant self.assertEqual(options.robot_paths[3], os.path.join('/tmp', os.getenv('USER'))) # first path in DEFAULT_ROBOT_PATHS self.assertEqual(options.robot_paths[4], os.path.join(tmpdir, 'easybuild', 'easyconfigs')) testpath3 = os.path.join(self.tmpdir, 'testTHREE') os.environ['EASYBUILD_SOURCEPATH'] = testpath2 args = [ '--debug', '--installpath', testpath3, ] options = init_config(args=args) self.assertEqual( source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(build_path(), testpath1) # via config file del os.environ['EASYBUILD_CONFIGFILES'] sys.path[:] = orig_sys_path
def test_modules_tool_stateless(self): """Check whether ModulesTool instance is stateless between runs.""" test_modules_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'modules') # copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load shutil.copytree(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core')) shutil.copytree(os.path.join(test_modules_path, 'Compiler'), os.path.join(self.test_prefix, 'Compiler')) modtxt = read_file( os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2')) modpath_extension = os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2') modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M) write_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'), modtxt) modtxt = read_file( os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4')) modpath_extension = os.path.join(self.test_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4') mkdir(modpath_extension, parents=True) modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M) write_file( os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), modtxt) # force reset of any singletons by reinitiating config init_config() os.environ['MODULEPATH'] = os.path.join(self.test_prefix, 'Core') modtool = modules_tool() if isinstance(modtool, Lmod): load_err_msg = "cannot[\s\n]*be[\s\n]*loaded" else: load_err_msg = "Unable to locate a modulefile" # GCC/4.6.3 is *not* an available Core module self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['GCC/4.6.3']) # GCC/4.7.2 is one of the available Core modules modtool.load(['GCC/4.7.2']) # OpenMPI/1.6.4 becomes available after loading GCC/4.7.2 module modtool.load(['OpenMPI/1.6.4']) modtool.purge() # reset $MODULEPATH, obtain new ModulesTool instance, # which should not remember anything w.r.t. previous $MODULEPATH value os.environ['MODULEPATH'] = test_modules_path modtool = modules_tool() # GCC/4.6.3 is available modtool.load(['GCC/4.6.3']) modtool.purge() # GCC/4.7.2 is available (note: also as non-Core module outside of hierarchy) modtool.load(['GCC/4.7.2']) # OpenMPI/1.6.4 is *not* available with current $MODULEPATH (loaded GCC/4.7.2 was not a hierarchical module) self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['OpenMPI/1.6.4'])
def test_generaloption_config(self): """Test new-style configuration (based on generaloption).""" self.purge_environment() # check whether configuration via environment variables works as expected prefix = os.path.join(self.tmpdir, 'testprefix') buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build', 'path') os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var options = init_config(args=[]) self.assertEqual(build_path(), buildpath_env_var) self.assertEqual(install_path(), os.path.join(prefix, 'software')) self.assertEqual(get_repositorypath(), [os.path.join(prefix, 'ebfiles_repo')]) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_BUILDPATH'] # check whether configuration via command line arguments works prefix = os.path.join(self.tmpdir, 'test1') install = os.path.join(self.tmpdir, 'test2', 'install') repopath = os.path.join(self.tmpdir, 'test2', 'repo') config_file = os.path.join(self.tmpdir, 'nooldconfig.py') write_file(config_file, '') args = [ '--configfiles', config_file, # force empty config file '--prefix', prefix, '--installpath', install, '--repositorypath', repopath, '--subdir-software', 'APPS', ] options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, 'APPS')) self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) self.assertTrue(config_file in options.configfiles) # check mixed command line/env var configuration prefix = os.path.join(self.tmpdir, 'test3') install = os.path.join(self.tmpdir, 'test4', 'install') subdir_software = 'eb-soft' args = [ '--configfiles', config_file, # force empty config file '--installpath', install, ] os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software installpath_modules = tempfile.mkdtemp(prefix='installpath-modules') os.environ['EASYBUILD_INSTALLPATH_MODULES'] = installpath_modules options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, subdir_software)) self.assertEqual(install_path('mod'), installpath_modules) # subdir options *must* be relative (to --installpath) installpath_software = tempfile.mkdtemp(prefix='installpath-software') os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = installpath_software error_regex = r"Found problems validating the options.*'subdir_software' must specify a \*relative\* path" self.assertErrorRegex(EasyBuildError, error_regex, init_config) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def configure(self, args=None): """(re)Configure and return configfile""" options = init_config(args=args) return options.config