def configure_step(self): """Configure MATLAB installation: create license file.""" # create license file licserv = self.cfg['license_server'] licport = self.cfg['license_server_port'] lictxt = '\n'.join([ "SERVER %s 000000000000 %s" % (licserv, licport), "USE_SERVER", ]) licfile = os.path.join(self.builddir, 'matlab.lic') write_file(licfile, lictxt) try: shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile) config = read_file(self.configfile) regdest = re.compile(r"^# destinationFolder=.*", re.M) regkey = re.compile(r"^# fileInstallationKey=.*", re.M) regagree = re.compile(r"^# agreeToLicense=.*", re.M) regmode = re.compile(r"^# mode=.*", re.M) reglicpath = re.compile(r"^# licensePath=.*", re.M) config = regdest.sub("destinationFolder=%s" % self.installdir, config) key = self.cfg['key'] config = regkey.sub("fileInstallationKey=%s" % key, config) config = regagree.sub("agreeToLicense=Yes", config) config = regmode.sub("mode=silent", config) config = reglicpath.sub("licensePath=%s" % licfile, config) write_file(self.configfile, config) except IOError, err: raise EasyBuildError("Failed to create installation config file %s: %s", self.configfile, err)
def test_toy_build(self): """Perform a toy build.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) # adjust PYTHONPATH such that test easyblocks are found orig_sys_path = sys.path[:] sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'sandbox'))) import easybuild.easyblocks reload(easybuild.easyblocks) reload(easybuild.tools.module_naming_scheme) # clear log write_file(self.logfile, '') buildpath = tempfile.mkdtemp() installpath = tempfile.mkdtemp() sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') args = [ os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'), '--sourcepath=%s' % sourcepath, '--buildpath=%s' % buildpath, '--installpath=%s' % installpath, '--debug', '--unittest-file=%s' % self.logfile, '--force', ] try: main((args, dummylogfn, True)) except (SystemExit, Exception), err: print err
def extensions_step(self): """Build & Install both Python and R extension""" # we start with the python bindings self.py_ext.src = os.path.join(self.mxnet_src_dir, "python") change_dir(self.py_ext.src) self.py_ext.prerun() self.py_ext.run(unpack_src=False) self.py_ext.postrun() # next up, the R bindings self.r_ext.src = os.path.join(self.mxnet_src_dir, "R-package") change_dir(self.r_ext.src) mkdir("inst") symlink(os.path.join(self.installdir, "lib"), os.path.join("inst", "libs")) symlink(os.path.join(self.installdir, "include"), os.path.join("inst", "include")) # MXNet doesn't provide a list of its R dependencies by default write_file("NAMESPACE", R_NAMESPACE) change_dir(self.mxnet_src_dir) self.r_ext.prerun() # MXNet is just weird. To install the R extension, we have to: # - First install the extension like it is # - Let R export the extension again. By doing this, all the dependencies get # correctly filled and some mappings are done # - Reinstal the exported version self.r_ext.run() run_cmd("R_LIBS=%s Rscript -e \"require(mxnet); mxnet:::mxnet.export(\\\"R-package\\\")\"" % self.installdir) change_dir(self.r_ext.src) self.r_ext.run() self.r_ext.postrun()
def configure_step(self): """Custom configuration procedure for pplacer.""" # install dir has to be non-existing when we start (it may be there from a previous (failed) install try: if os.path.exists(self.installdir): shutil.rmtree(self.installdir) self.log.warning("Existing install directory %s removed", self.installdir) except OSError as err: raise EasyBuildError("Failed to remove %s: %s", self.installdir, err) # configure OPAM to install pplacer dependencies env.setvar('OPAMROOT', self.installdir) opam_init_cmd = mk_opam_init_cmd() run_cmd(opam_init_cmd) run_cmd("opam repo add pplacer-deps %s/pplacer-opam-repository*/" % self.builddir) run_cmd("opam update pplacer-deps") env.setvar('OCAML_BACKEND', 'gcc') run_cmd("eval `opam config env` && cat opam-requirements.txt | xargs -t opam install -y") txt = "let version = \"v%s\"\n" % self.version write_file(os.path.join(self.builddir, 'pplacer-%s' % self.version, 'common_src', 'version.ml'), txt)
def test_tweak_one_version(self): """Test tweak_one function""" test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb') # test tweaking of software version (--try-software-version) tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb') tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict() tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() # checksums should be reset to empty list, only version should be changed, nothing else self.assertEqual(tweaked_toy_ec_parsed['checksums'], []) self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]: val = toy_ec_parsed[key] self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) tweaked_val = tweaked_toy_ec_parsed.get(key) self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val)) # check behaviour if target file already exists error_pattern = "File exists, not overwriting it without --force" self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) # existing file does get overwritten when --force is used init_config(build_options={'force': True, 'silent': True}) write_file(tweaked_toy_ec, '') tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
def test_include_mns(self): """Test include_module_naming_schemes().""" testdir = os.path.dirname(os.path.abspath(__file__)) test_mns = os.path.join(testdir, 'sandbox', 'easybuild', 'module_naming_scheme') my_mns = os.path.join(self.test_prefix, 'my_mns') mkdir(my_mns) # include __init__.py file that should be ignored, and shouldn't cause trouble (bug #1697) write_file(os.path.join(my_mns, '__init__.py'), "# dummy init, should not get included") my_mns_txt = '\n'.join([ "from easybuild.tools.module_naming_scheme import ModuleNamingScheme", "class MyMNS(ModuleNamingScheme):", " pass", ]) write_file(os.path.join(my_mns, 'my_mns.py'), my_mns_txt) # include custom MNS included_mns_path = include_module_naming_schemes(self.test_prefix, [os.path.join(my_mns, '*.py')]) expected_paths = ['__init__.py', 'tools/__init__.py', 'tools/module_naming_scheme/__init__.py', 'tools/module_naming_scheme/my_mns.py'] for filepath in expected_paths: fullpath = os.path.join(included_mns_path, 'easybuild', filepath) self.assertTrue(os.path.exists(fullpath), "%s exists" % fullpath) # path to included MNSs should be prepended to Python search path self.assertEqual(sys.path[0], included_mns_path) # importing custom MNS should work import easybuild.tools.module_naming_scheme.my_mns my_mns_pyc_path = easybuild.tools.module_naming_scheme.my_mns.__file__ my_mns_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_mns_pyc_path), 'my_mns.py')) self.assertTrue(os.path.samefile(up(my_mns_real_py_path, 1), my_mns))
def test_download_repo(self): """Test download_repo function.""" if self.github_token is None: print "Skipping test_download_repo, no GitHub token available?" return # default: download tarball for master branch of hpcugent/easybuild-easyconfigs repo path = gh.download_repo(path=self.test_prefix) repodir = os.path.join(self.test_prefix, 'hpcugent', 'easybuild-easyconfigs-master') self.assertTrue(os.path.samefile(path, repodir)) self.assertTrue(os.path.exists(repodir)) shafile = os.path.join(repodir, 'latest-sha') self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2015a.eb'))) # existing downloaded repo is not reperformed, except if SHA is different account, repo, branch = 'boegel', 'easybuild-easyblocks', 'develop' repodir = os.path.join(self.test_prefix, account, '%s-%s' % (repo, branch)) latest_sha = gh.fetch_latest_commit_sha(repo, account, branch=branch) # put 'latest-sha' fail in place, check whether repo was (re)downloaded (should not) shafile = os.path.join(repodir, 'latest-sha') write_file(shafile, latest_sha) path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix) self.assertTrue(os.path.samefile(path, repodir)) self.assertEqual(os.listdir(repodir), ['latest-sha']) # remove 'latest-sha' file and verify that download was performed os.remove(shafile) path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix) self.assertTrue(os.path.samefile(path, repodir)) self.assertTrue('easybuild' in os.listdir(repodir)) self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')))
def configure_step(self): """Configure MCR installation: create license file.""" configfile = os.path.join(self.builddir, self.configfilename) if LooseVersion(self.version) < LooseVersion('R2015a'): shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), configfile) config = read_file(configfile) # compile regex first since re.sub doesn't accept re.M flag for multiline regex in Python 2.6 regdest = re.compile(r"^# destinationFolder=.*", re.M) regagree = re.compile(r"^# agreeToLicense=.*", re.M) regmode = re.compile(r"^# mode=.*", re.M) config = regdest.sub("destinationFolder=%s" % self.installdir, config) config = regagree.sub("agreeToLicense=Yes", config) config = regmode.sub("mode=silent", config) else: config = '\n'.join([ "destinationFolder=%s" % self.installdir, "agreeToLicense=Yes", "mode=silent", ]) write_file(configfile, config) self.log.debug("configuration file written to %s:\n %s", configfile, config)
def test_include_easyblocks_priority(self): """Test whether easyblocks included via include_easyblocks() get prioroity over others.""" test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks') # make sure that test 'foo' easyblocks is there import easybuild.easyblocks.foo foo_path = os.path.dirname(os.path.dirname(easybuild.easyblocks.foo.__file__)) self.assertTrue(os.path.samefile(foo_path, test_easyblocks)) # inject custom 'foo' easyblocks myeasyblocks = os.path.join(self.test_prefix, 'myeasyblocks') mkdir(myeasyblocks) # include __init__.py file that should be ignored, and shouldn't cause trouble (bug #1697) write_file(os.path.join(myeasyblocks, '__init__.py'), "# dummy init, should not get included") # 'undo' import of foo easyblock del sys.modules['easybuild.easyblocks.foo'] foo_easyblock_txt = '\n'.join([ "from easybuild.framework.easyblock import EasyBlock", "class EB_Foo(EasyBlock):", " pass", ]) write_file(os.path.join(myeasyblocks, 'foo.py'), foo_easyblock_txt) include_easyblocks(self.test_prefix, [os.path.join(myeasyblocks, 'foo.py')]) foo_pyc_path = easybuild.easyblocks.foo.__file__ foo_real_py_path = os.path.realpath(os.path.join(os.path.dirname(foo_pyc_path), 'foo.py')) self.assertFalse(os.path.samefile(os.path.dirname(foo_pyc_path), test_easyblocks)) self.assertTrue(os.path.samefile(foo_real_py_path, os.path.join(myeasyblocks, 'foo.py'))) # 'undo' import of foo easyblock del sys.modules['easybuild.easyblocks.foo']
def test_robot_archived_easyconfigs(self): """Test whether robot can pick up archived easyconfigs when asked.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-ictce-4.1.13.eb') gzip_ectxt = read_file(gzip_ec) test_ec = os.path.join(self.test_prefix, 'test.eb') tc_spec = "toolchain = {'name': 'ictce', 'version': '3.2.2.u3'}" regex = re.compile("^toolchain = .*", re.M) test_ectxt = regex.sub(tc_spec, gzip_ectxt) write_file(test_ec, test_ectxt) ecs, _ = parse_easyconfigs([(test_ec, False)]) self.assertErrorRegex(EasyBuildError, "Irresolvable dependencies encountered", resolve_dependencies, ecs, self.modtool, retain_all_deps=True) # --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs init_config(build_options={ 'consider_archived_easyconfigs': True, 'robot_path': [test_ecs], }) res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True) self.assertEqual([ec['full_mod_name'] for ec in res], ['ictce/3.2.2.u3', 'gzip/1.5-ictce-3.2.2.u3']) expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb') self.assertTrue(os.path.samefile(res[0]['spec'], expected))
def test_checksums(self): """Test checksum functionality.""" fh, fp = tempfile.mkstemp() os.close(fh) ft.write_file(fp, "easybuild\n") known_checksums = { 'adler32': '0x379257805', 'crc32': '0x1457143216', 'md5': '7167b64b1ca062b9674ffef46f9325db', 'sha1': 'db05b79e09a4cc67e9dd30b313b5488813db3190', } # make sure checksums computation/verification is correct for checksum_type, checksum in known_checksums.items(): self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum) self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum))) # md5 is default self.assertEqual(ft.compute_checksum(fp), known_checksums['md5']) self.assertTrue(ft.verify_checksum(fp, known_checksums['md5'])) # make sure faulty checksums are reported broken_checksums = dict([(typ, val + 'foo') for (typ, val) in known_checksums.items()]) for checksum_type, checksum in broken_checksums.items(): self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum) self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum))) # md5 is default self.assertFalse(ft.compute_checksum(fp) == broken_checksums['md5']) self.assertFalse(ft.verify_checksum(fp, broken_checksums['md5'])) # cleanup os.remove(fp)
def test_job(self): """Test submitting build as a job.""" # use gzip-1.4.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4.eb') # check log message with --job for job_args in [ # options passed are reordered, so order here matters to make tests pass ['--debug'], ['--debug', '--stop=configure', '--try-software-name=foo'], ]: # clear log file write_file(self.logfile, '') args = [ eb_file, '--job', ] + job_args outtxt = self.eb_main(args) job_msg = "INFO.* Command template for jobs: .* && eb %%\(spec\)s.* %s.*\n" % ' .*'.join(job_args) assertmsg = "Info log message with job command template when using --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt) self.assertTrue(re.search(job_msg, outtxt), assertmsg) modify_env(os.environ, self.orig_environ) tempfile.tempdir = None
def generate_recipe(self): """ This method will make use of resolve_template and resolve_template_data methods in order to generate the container recipe. """ template = self.resolve_template() data = self.resolve_template_data() if self.img_name: file_label = os.path.splitext(self.img_name)[0] else: file_label = data['mod_names'].split(' ')[0].replace('/', '-') recipe_path = os.path.join(self.container_path, "%s.%s" % (self.RECIPE_FILE_NAME, file_label)) if os.path.exists(recipe_path): if build_option('force'): print_msg("WARNING: overwriting existing container recipe at %s due to --force" % recipe_path) else: raise EasyBuildError("Container recipe at %s already exists, not overwriting it without --force", recipe_path) recipe_content = template % data write_file(recipe_path, recipe_content) print_msg("%s definition file created at %s" % (self.RECIPE_FILE_NAME, recipe_path), log=self.log) return recipe_path
def test_generaloption_config(self): """Test new-style configuration (based on generaloption).""" prefix = os.path.join(self.tmpdir, 'test1') install = os.path.join(self.tmpdir, 'test2', 'install') repopath = os.path.join(self.tmpdir, 'test2', 'repo') config_file = os.path.join(self.tmpdir, 'nooldconfig.py') write_file(config_file, '') args = [ '--config', config_file, # force empty oldstyle config file '--prefix', prefix, '--installpath', install, '--repositorypath', repopath, ] options = self.configure_options(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, 'software')) self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) self.assertEqual(options.config, config_file)
def test_force(self): """Test forcing installation even if the module is already available.""" # use GCC-4.6.3.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'GCC-4.6.3.eb') # check log message without --force args = [ eb_file, '--debug', ] outtxt, error_thrown = self.eb_main(args, return_error=True) self.assertTrue(not error_thrown, "No error is thrown if software is already installed (error_thrown: %s)" % error_thrown) already_msg = "GCC/4.6.3 is already installed" self.assertTrue(re.search(already_msg, outtxt), "Already installed message without --force, outtxt: %s" % outtxt) # clear log file, clean up environment write_file(self.logfile, '') modify_env(os.environ, self.orig_environ) tempfile.tempdir = None # check that --force works args = [ eb_file, '--force', '--debug', ] outtxt = self.eb_main(args) self.assertTrue(not re.search(already_msg, outtxt), "Already installed message not there with --force")
def setup_sandbox_for_intel_fftw(self, moddir, imklver='10.3.12.361'): """Set up sandbox for Intel FFTW""" # hack to make Intel FFTW lib check pass # create dummy imkl module and put required lib*.a files in place imkl_module_path = os.path.join(moddir, 'imkl', imklver) imkl_dir = os.path.join(self.test_prefix, 'software', 'imkl', imklver) imkl_mod_txt = '\n'.join([ "#%Module", "setenv EBROOTIMKL %s" % imkl_dir, "setenv EBVERSIONIMKL %s" % imklver, ]) write_file(imkl_module_path, imkl_mod_txt) fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] fftw_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] if LooseVersion(imklver) >= LooseVersion('11'): fftw_libs.extend(['fftw3x_cdft_ilp64', 'fftw3x_cdft_lp64']) else: fftw_libs.append('fftw3x_cdft') for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: os.makedirs(os.path.join(imkl_dir, subdir)) for fftlib in fftw_libs: write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo')
def add_easyconfig(self, cfg, name, version, stats, previous): """ Add the eb-file for software name and version to the repository. stats should be a dict containing statistics. if previous is true -> append the statistics to the file This will return the path to the created file (for use in subclasses) """ # create directory for eb file full_path = os.path.join(self.wc, self.subdir, name) mkdir(full_path, parents=True) # destination dest = os.path.join(full_path, "%s-%s.eb" % (name, version)) txt = "# Built with EasyBuild version %s on %s\n" % (VERBOSE_VERSION, time.strftime("%Y-%m-%d_%H-%M-%S")) # copy file txt += read_file(cfg) # append a line to the eb file so that we don't have git merge conflicts if not previous: statsprefix = "\n# Build statistics\nbuildstats = [" statssuffix = "]\n" else: # statstemplate = "\nbuildstats.append(%s)\n" statsprefix = "\nbuildstats.append(" statssuffix = ")\n" txt += statsprefix + stats_to_str(stats) + statssuffix write_file(dest, txt) return dest
def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state): """ Upload/dump overall test report @param ecs_with_res: processed easyconfigs with build result (success/failure) @param orig_cnt: number of original easyconfig paths @param success: boolean indicating whether all builds were successful @param msg: message to be included in test report @param init_session_state: initial session state info to include in test report """ dump_path = build_option('dump_test_report') pr_nr = build_option('from_pr') upload = build_option('upload_test_report') if upload: msg = msg + " (%d easyconfigs in this PR)" % orig_cnt test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True) if pr_nr: # upload test report to gist and issue a comment in the PR to notify txt = post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, success) else: # only upload test report as a gist gist_url = upload_test_report_as_gist(test_report) txt = "Test report uploaded to %s" % gist_url else: test_report = create_test_report(msg, ecs_with_res, init_session_state) txt = None _log.debug("Test report: %s" % test_report) if dump_path is not None: write_file(dump_path, test_report) _log.info("Test report dumped to %s" % dump_path) return txt
def test_toy_download_sources(self): """Test toy build with sources that still need to be 'downloaded'.""" tmpdir = tempfile.mkdtemp() # copy toy easyconfig file, and append source_urls to it shutil.copy2(os.path.join(os.path.dirname(__file__), "easyconfigs", "toy-0.0.eb"), tmpdir) source_url = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sandbox", "sources", "toy") ec_file = os.path.join(tmpdir, "toy-0.0.eb") write_file(ec_file, '\nsource_urls = ["file://%s"]\n' % source_url, append=True) # unset $EASYBUILD_XPATH env vars, to make sure --prefix is picked up for cfg_opt in ["build", "install", "source"]: del os.environ["EASYBUILD_%sPATH" % cfg_opt.upper()] sourcepath = os.path.join(tmpdir, "mysources") args = [ ec_file, "--prefix=%s" % tmpdir, "--sourcepath=%s" % ":".join([sourcepath, "/bar"]), # include senseless path which should be ignored "--debug", "--unittest-file=%s" % self.logfile, "--force", ] outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True) self.check_toy(tmpdir, outtxt) self.assertTrue(os.path.exists(os.path.join(sourcepath, "t", "toy", "toy-0.0.tar.gz"))) shutil.rmtree(tmpdir)
def process_easyconfig_file(ec_file): """Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested).""" ectxt = read_file(ec_file) name, easyblock = fetch_parameters_from_easyconfig(ectxt, ['name', 'easyblock']) derived_easyblock_class = get_easyblock_class(easyblock, name=name, default_fallback=False) fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class) if ectxt != fixed_ectxt: if go.options.backup: try: backup_ec_file = '%s.bk' % ec_file i = 1 while os.path.exists(backup_ec_file): backup_ec_file = '%s.bk%d' % (ec_file, i) i += 1 os.rename(ec_file, backup_ec_file) log.info("Backed up %s to %s" % (ec_file, backup_ec_file)) except OSError, err: raise EasyBuildError("Failed to backup %s before rewriting it: %s", ec_file, err) write_file(ec_file, fixed_ectxt) log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt) log.info("%s: fixed" % ec_file)
def test_apply_regex_substitutions(self): """Test apply_regex_substitutions function.""" testfile = os.path.join(self.test_prefix, 'test.txt') testtxt = '\n'.join([ "CC = gcc", "CFLAGS = -O3 -g", "FC = gfortran", "FFLAGS = -O3 -g -ffixed-form", ]) ft.write_file(testfile, testtxt) regex_subs = [ (r"^(CC)\s*=\s*.*$", r"\1 = ${CC}"), (r"^(FC\s*=\s*).*$", r"\1${FC}"), (r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"), ] ft.apply_regex_substitutions(testfile, regex_subs) expected_testtxt = '\n'.join([ "CC = ${CC}", "CFLAGS = -O2", "FC = ${FC}", "FFLAGS = -O2 -ffixed-form", ]) new_testtxt = ft.read_file(testfile) self.assertEqual(new_testtxt, expected_testtxt)
def test_toy_download_sources(self): """Test toy build with sources that still need to be 'downloaded'.""" tmpdir = tempfile.mkdtemp() # copy toy easyconfig file, and append source_urls to it shutil.copy2(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'), tmpdir) source_url = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sandbox', 'sources', 'toy') ec_file = os.path.join(tmpdir, 'toy-0.0.eb') write_file(ec_file, '\nsource_urls = ["file://%s"]\n' % source_url, append=True) # unset $EASYBUILD_XPATH env vars, to make sure --prefix is picked up for cfg_opt in ['build', 'install', 'source']: del os.environ['EASYBUILD_%sPATH' % cfg_opt.upper()] sourcepath = os.path.join(tmpdir, 'mysources') args = [ ec_file, '--prefix=%s' % tmpdir, '--sourcepath=%s' % ':'.join([sourcepath, '/bar']), # include senseless path which should be ignored '--debug', '--unittest-file=%s' % self.logfile, '--force', ] outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True) self.check_toy(tmpdir, outtxt) self.assertTrue(os.path.exists(os.path.join(sourcepath, 't', 'toy', 'toy-0.0.tar.gz'))) shutil.rmtree(tmpdir)
def test_toy_broken(self): """Test deliberately broken toy build.""" tmpdir = tempfile.mkdtemp() broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb") toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') broken_toy_ec_txt = read_file(toy_ec_file) broken_toy_ec_txt += "checksums = ['clearywrongchecksum']" write_file(broken_toy_ec, broken_toy_ec_txt) error_regex = "Checksum verification .* failed" self.assertErrorRegex(EasyBuildError, error_regex, self.test_toy_build, ec_file=broken_toy_ec, tmpdir=tmpdir, verify=False, fails=True, verbose=False, raise_error=True) # make sure log file is retained, also for failed build log_path_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*.log') self.assertTrue(len(glob.glob(log_path_pattern)) == 1, "Log file found at %s" % log_path_pattern) # make sure individual test report is retained, also for failed build test_report_fp_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*test_report.md') self.assertTrue(len(glob.glob(test_report_fp_pattern)) == 1, "Test report %s found" % test_report_fp_pattern) # test dumping full test report (doesn't raise an exception) test_report_fp = os.path.join(self.test_buildpath, 'full_test_report.md') self.test_toy_build(ec_file=broken_toy_ec, tmpdir=tmpdir, verify=False, fails=True, verbose=False, raise_error=True, test_report=test_report_fp) # cleanup shutil.rmtree(tmpdir)
def test_list_easyblocks(self): """Test listing easyblock hierarchy.""" # adjust PYTHONPATH such that test easyblocks are found orig_sys_path = sys.path import easybuild sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'easyblocks_sandbox'))) easybuild = reload(easybuild) import easybuild.easyblocks easybuild.easyblocks = reload(easybuild.easyblocks) # simple view for list_arg in ['--list-easyblocks', '--list-easyblocks=simple']: # clear log write_file(self.logfile, '') args = [ list_arg, '--unittest-file=%s' % self.logfile, ] try: main((args, None)) except (SystemExit, Exception), err: pass outtxt = read_file(self.logfile) for pat in [ r"EasyBlock\n", r"|--\s+EB_foo\n|\s+|--\s+EB_foofoo\n", r"|--\s+bar\n", ]: self.assertTrue(re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt))
def test_external_modules_metadata(self): """Test --external-modules-metadata.""" # empty list by default cfg = init_config() self.assertEqual(cfg.external_modules_metadata, []) testcfgtxt = EXTERNAL_MODULES_METADATA testcfg = os.path.join(self.test_prefix, 'test_external_modules_metadata.cfg') write_file(testcfg, testcfgtxt) cfg = init_config(args=['--external-modules-metadata=%s' % testcfg]) netcdf = { 'name': ['netCDF', 'netCDF-Fortran'], 'version': ['4.3.2', '4.3.2'], 'prefix': 'NETCDF_DIR', } self.assertEqual(cfg.external_modules_metadata['cray-netcdf/4.3.2'], netcdf) hdf5 = { 'name': ['HDF5'], 'version': ['1.8.13'], 'prefix': 'HDF5_DIR', } self.assertEqual(cfg.external_modules_metadata['cray-hdf5/1.8.13'], hdf5) # impartial metadata is fine self.assertEqual(cfg.external_modules_metadata['foo'], {'name': ['Foo'], 'prefix': '/foo'}) self.assertEqual(cfg.external_modules_metadata['bar/1.2.3'], {'name': ['bar'], 'version': ['1.2.3']}) # if both names and versions are specified, lists must have same lengths write_file(testcfg, '\n'.join(['[foo/1.2.3]', 'name = foo,bar', 'version = 1.2.3'])) args = ['--external-modules-metadata=%s' % testcfg] err_msg = "Different length for lists of names/versions in metadata for external module" self.assertErrorRegex(EasyBuildError, err_msg, init_config, args=args)
def test_remove_file(self): """Test remove_file""" testfile = os.path.join(self.test_prefix, 'foo') ft.write_file(testfile, 'bar') self.assertTrue(os.path.exists(testfile)) ft.remove_file(testfile) ft.write_file(testfile, 'bar') ft.adjust_permissions(self.test_prefix, stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH, add=False) self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile) # also test behaviour of remove_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) self.mock_stdout(True) ft.remove_file(testfile) txt = self.get_stdout() self.mock_stdout(False) regex = re.compile("^file [^ ]* removed$") self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def setUp(self): """Test setup.""" fd, self.logfile = tempfile.mkstemp(suffix=".log", prefix="eb-options-test-") os.close(fd) fd, self.dummylogfn = tempfile.mkstemp(prefix="easybuild-dummy", suffix=".log") os.close(fd) # adjust PYTHONPATH such that test easyblocks are found self.orig_sys_path = sys.path[:] sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "sandbox"))) import easybuild.easyblocks reload(easybuild.easyblocks) reload(easybuild.tools.module_naming_scheme) # clear log write_file(self.logfile, "") self.buildpath = tempfile.mkdtemp() self.installpath = tempfile.mkdtemp() self.sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox", "sources") # keep track of original environment to restore self.orig_environ = copy.deepcopy(os.environ)
def test_toy_tweaked(self): """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') ec_file = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') shutil.copy2(os.path.join(test_ecs_dir, 'toy-0.0.eb'), ec_file) # tweak easyconfig by appending to it ec_extra = '\n'.join([ "versionsuffix = '-tweaked'", "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz']}", "modextravars = {'FOO': 'bar'}", "modloadmsg = 'THANKS FOR LOADING ME, I AM %(name)s v%(version)s'", "modtclfooter = 'puts stderr \"oh hai!\"'", ]) write_file(ec_file, ec_extra, append=True) args = [ ec_file, '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--debug', '--force', ] outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.check_toy(self.test_installpath, outtxt, versionsuffix='-tweaked') toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked') toy_module_txt = read_file(toy_module) self.assertTrue(re.search('setenv\s*FOO\s*"bar"', toy_module_txt)) self.assertTrue(re.search('prepend-path\s*SOMEPATH\s*\$root/foo/bar', toy_module_txt)) self.assertTrue(re.search('prepend-path\s*SOMEPATH\s*\$root/baz', toy_module_txt)) self.assertTrue(re.search('module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0', toy_module_txt)) self.assertTrue(re.search('puts stderr "oh hai!"', toy_module_txt))
def test_expand_glob_paths(self): """Test expand_glob_paths function.""" for dirname in ['empty_dir', 'test_dir']: ft.mkdir(os.path.join(self.test_prefix, dirname), parents=True) for filename in ['file1.txt', 'test_dir/file2.txt', 'test_dir/file3.txt', 'test_dir2/file4.dat']: ft.write_file(os.path.join(self.test_prefix, filename), 'gibberish') globs = [os.path.join(self.test_prefix, '*.txt'), os.path.join(self.test_prefix, '*', '*')] expected = [ os.path.join(self.test_prefix, 'file1.txt'), os.path.join(self.test_prefix, 'test_dir', 'file2.txt'), os.path.join(self.test_prefix, 'test_dir', 'file3.txt'), os.path.join(self.test_prefix, 'test_dir2', 'file4.dat'), ] self.assertEqual(sorted(ft.expand_glob_paths(globs)), sorted(expected)) # passing non-glob patterns is fine too file2 = os.path.join(self.test_prefix, 'test_dir', 'file2.txt') self.assertEqual(ft.expand_glob_paths([file2]), [file2]) # test expanding of '~' into $HOME value # hard overwrite $HOME in environment (used by os.path.expanduser) so we can reliably test this new_home = os.path.join(self.test_prefix, 'home') ft.mkdir(new_home, parents=True) ft.write_file(os.path.join(new_home, 'test.txt'), 'test') os.environ['HOME'] = new_home self.assertEqual(ft.expand_glob_paths(['~/*.txt']), [os.path.join(new_home, 'test.txt')]) # check behaviour if glob that has no (file) matches is passed glob_pat = os.path.join(self.test_prefix, 'test_*') self.assertErrorRegex(EasyBuildError, "No files found using glob pattern", ft.expand_glob_paths, [glob_pat])
def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" # put fake 'gcc' command in place that just echos its arguments fake_gcc = os.path.join(self.test_prefix, 'fake', 'gcc') write_file(fake_gcc, '#!/bin/bash\necho "$@"') adjust_permissions(fake_gcc, stat.S_IXUSR) os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', '')) # enable --rpath and prepare toolchain init_config(build_options={'rpath': True, 'rpath_filter': ['/ba.*']}) tc = self.get_toolchain('gompi', version='1.3.12') # preparing RPATH wrappers requires --experimental, need to bypass that here tc.log.experimental = lambda x: x tc.prepare() # check whether fake gcc was wrapped and that arguments are what they should be # no -rpath for /bar because of rpath filter out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""') expected = ' '.join([ '-Wl,-rpath=$ORIGIN/../lib', '-Wl,-rpath=$ORIGIN/../lib64', '-Wl,--disable-new-dtags', '-Wl,-rpath=/foo', '%(user)s.c', '-L/foo', '-L/bar', '$FOO', '-DX=""', ]) self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
def install_step(self): """Custom install procedure for TensorFlow.""" # find .whl file that was built, and install it using 'pip install' if ("-rc" in self.version): whl_version = self.version.replace("-rc", "rc") else: whl_version = self.version whl_paths = glob.glob( os.path.join(self.builddir, 'tensorflow-%s-*.whl' % whl_version)) if not whl_paths: whl_paths = glob.glob( os.path.join(self.builddir, 'tensorflow-*.whl')) if len(whl_paths) == 1: # --ignore-installed is required to ensure *this* wheel is installed cmd = "pip install --ignore-installed --prefix=%s %s" % ( self.installdir, whl_paths[0]) # if extensions are listed, assume they will provide all required dependencies, # so use --no-deps to prevent pip from downloading & installing them if self.cfg['exts_list']: cmd += ' --no-deps' run_cmd(cmd, log_all=True, simple=True, log_ok=True) else: raise EasyBuildError("Failed to isolate built .whl in %s: %s", whl_paths, self.builddir) # Fix for https://github.com/tensorflow/tensorflow/issues/6341 on Python < 3.3 # If the site-packages/google/__init__.py file is missing, make it an empty file. # This fixes the "No module named google.protobuf" error that sometimes shows up during sanity_check # For Python >= 3.3 the logic is reversed: The __init__.py must not exist. # See e.g. http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html google_protobuf_dir = os.path.join(self.installdir, self.pylibdir, 'google', 'protobuf') google_init_file = os.path.join(self.installdir, self.pylibdir, 'google', '__init__.py') if LooseVersion(det_python_version( self.python_cmd)) < LooseVersion('3.3'): if os.path.isdir( google_protobuf_dir) and not is_readable(google_init_file): self.log.debug("Creating (empty) missing %s", google_init_file) write_file(google_init_file, '') else: if os.path.exists(google_init_file): self.log.debug("Removing %s for Python >= 3.3", google_init_file) remove_file(google_init_file) # Fix cuda header paths # This is needed for building custom TensorFlow ops if LooseVersion(self.version) < LooseVersion('1.14'): pyshortver = '.'.join( get_software_version('Python').split('.')[:2]) regex_subs = [(r'#include "cuda/include/', r'#include "')] base_path = os.path.join(self.installdir, 'lib', 'python%s' % pyshortver, 'site-packages', 'tensorflow', 'include', 'tensorflow') for header in glob.glob( os.path.join(base_path, 'stream_executor', 'cuda', 'cuda*.h')) + glob.glob( os.path.join(base_path, 'core', 'util', 'cuda*.h')): apply_regex_substitutions(header, regex_subs)
def test_get_log_filename(self): """Test for get_log_filename().""" tmpdir = tempfile.gettempdir() res = get_log_filename('foo', '1.2.3') regex = re.compile( os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-[0-9]{8}\.[0-9]{6}\.log$')) self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res)) res = get_log_filename('foo', '1.2.3', date='19700101') regex = re.compile( os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.[0-9]{6}\.log$')) self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res)) res = get_log_filename('foo', '1.2.3', timestamp='094651') regex = re.compile( os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-[0-9]{8}\.094651\.log$')) self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res)) res = get_log_filename('foo', '1.2.3', date='19700101', timestamp='094651') regex = re.compile( os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.log$')) self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res)) # if log file already exists, numbers are added to the filename to obtain a new file path write_file(res, '') res = get_log_filename('foo', '1.2.3', date='19700101', timestamp='094651') regex = re.compile( os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.log\.1$')) self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res)) # adding salt ensures a unique filename (pretty much) prev_log_filenames = [] for i in range(10): res = get_log_filename('foo', '1.2.3', date='19700101', timestamp='094651', add_salt=True) regex = re.compile( os.path.join( tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.[a-zA-Z]{5}\.log$' )) self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res)) self.assertTrue(res not in prev_log_filenames) prev_log_filenames.append(res)
def test_flex_robot_paths(self): """Test prepend/appending to default robot search path via --robot-paths.""" # unset $EASYBUILD_ROBOT_PATHS that was defined in setUp del os.environ['EASYBUILD_ROBOT_PATHS'] # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory # to check whether easyconfigs install path is auto-included in robot path tmpdir = tempfile.mkdtemp( prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) test_ecs_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') tmp_ecs_dir = os.path.join(tmpdir, 'easybuild', 'easyconfigs') copy_dir(test_ecs_path, tmp_ecs_dir) # prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default orig_sys_path = sys.path[:] sys.path = [tmpdir] + [ p for p in sys.path if not os.path.exists(os.path.join(p, 'easybuild', 'easyconfigs')) ] # default: only pick up installed easyconfigs via sys.path eb_go = eboptions.parse_options(args=[]) self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir]) # prepend to default robot path eb_go = eboptions.parse_options(args=['--robot-paths=/foo:']) self.assertEqual(eb_go.options.robot_paths, ['/foo', tmp_ecs_dir]) eb_go = eboptions.parse_options(args=['--robot-paths=/foo:/bar/baz/:']) self.assertEqual(eb_go.options.robot_paths, ['/foo', '/bar/baz/', tmp_ecs_dir]) # append to default robot path eb_go = eboptions.parse_options(args=['--robot-paths=:/bar/baz']) self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir, '/bar/baz']) # append to default robot path eb_go = eboptions.parse_options(args=['--robot-paths=:/bar/baz:/foo']) self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir, '/bar/baz', '/foo']) # prepend and append to default robot path eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar::/baz']) self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', tmp_ecs_dir, '/baz']) eb_go = eboptions.parse_options( args=['--robot-paths=/foo/bar::/baz:/trala']) self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', tmp_ecs_dir, '/baz', '/trala']) eb_go = eboptions.parse_options( args=['--robot-paths=/foo/bar:/trala::/baz']) self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', '/trala', tmp_ecs_dir, '/baz']) # also via $EASYBUILD_ROBOT_PATHS os.environ['EASYBUILD_ROBOT_PATHS'] = '/foo::/bar/baz' eb_go = eboptions.parse_options(args=[]) self.assertEqual(eb_go.options.robot_paths, ['/foo', tmp_ecs_dir, '/bar/baz']) # --robot-paths overrides $EASYBUILD_ROBOT_PATHS os.environ['EASYBUILD_ROBOT_PATHS'] = '/foobar::/barbar/baz/baz' eb_go = eboptions.parse_options(args=['--robot-paths=/one::/last']) self.assertEqual(eb_go.options.robot_paths, ['/one', tmp_ecs_dir, '/last']) del os.environ['EASYBUILD_ROBOT_PATHS'] # also works with a cfgfile in the mix config_file = os.path.join(self.tmpdir, 'testconfig.cfg') cfgtxt = '\n'.join([ '[config]', 'robot-paths=/cfgfirst::/cfglast', ]) write_file(config_file, cfgtxt) eb_go = eboptions.parse_options( args=['--configfiles=%s' % config_file]) self.assertEqual(eb_go.options.robot_paths, ['/cfgfirst', tmp_ecs_dir, '/cfglast']) # cfgfile entry is lost when env var and/or cmdline options are used os.environ['EASYBUILD_ROBOT_PATHS'] = '/envfirst::/envend' eb_go = eboptions.parse_options( args=['--configfiles=%s' % config_file]) self.assertEqual(eb_go.options.robot_paths, ['/envfirst', tmp_ecs_dir, '/envend']) del os.environ['EASYBUILD_ROBOT_PATHS'] eb_go = eboptions.parse_options(args=[ '--robot-paths=/veryfirst:', '--configfiles=%s' % config_file ]) self.assertEqual(eb_go.options.robot_paths, ['/veryfirst', tmp_ecs_dir]) os.environ['EASYBUILD_ROBOT_PATHS'] = ':/envend' eb_go = eboptions.parse_options(args=[ '--robot-paths=/veryfirst:', '--configfiles=%s' % config_file ]) self.assertEqual(eb_go.options.robot_paths, ['/veryfirst', tmp_ecs_dir]) del os.environ['EASYBUILD_ROBOT_PATHS'] # override default robot path eb_go = eboptions.parse_options(args=['--robot-paths=/foo:/bar/baz']) self.assertEqual(eb_go.options.robot_paths, ['/foo', '/bar/baz']) # paths specified via --robot still get preference first = os.path.join(self.test_prefix, 'first') mkdir(first) eb_go = eboptions.parse_options( args=['--robot-paths=/foo/bar::/baz', '--robot=%s' % first]) self.assertEqual(eb_go.options.robot_paths, [first, '/foo/bar', tmp_ecs_dir, '/baz']) sys.path[:] = orig_sys_path
def test_XDG_CONFIG_env_vars(self): """Test effect of XDG_CONFIG* environment variables on default configuration.""" self.purge_environment() xdg_config_home = os.environ.get('XDG_CONFIG_HOME') xdg_config_dirs = os.environ.get('XDG_CONFIG_DIRS') cfg_template = '\n'.join([ '[config]', 'prefix=%s', ]) homedir = os.path.join(self.test_prefix, 'homedir', '.config') mkdir(os.path.join(homedir, 'easybuild'), parents=True) write_file(os.path.join(homedir, 'easybuild', 'config.cfg'), cfg_template % '/home') dir1 = os.path.join(self.test_prefix, 'dir1') mkdir(os.path.join(dir1, 'easybuild.d'), parents=True) write_file(os.path.join(dir1, 'easybuild.d', 'foo.cfg'), cfg_template % '/foo') write_file(os.path.join(dir1, 'easybuild.d', 'bar.cfg'), cfg_template % '/bar') dir2 = os.path.join(self.test_prefix, 'dir2') # empty on purpose mkdir(os.path.join(dir2, 'easybuild.d'), parents=True) dir3 = os.path.join(self.test_prefix, 'dir3') mkdir(os.path.join(dir3, 'easybuild.d'), parents=True) write_file(os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'), cfg_template % '/foobarbaz') # set $XDG_CONFIG_DIRS to non-existing dir to isolate ourselves from possible system-wide config files os.environ[ 'XDG_CONFIG_DIRS'] = '/there/should/be/no/such/directory/we/hope' # only $XDG_CONFIG_HOME set (to existing path) os.environ['XDG_CONFIG_HOME'] = homedir cfg_files = [os.path.join(homedir, 'easybuild', 'config.cfg')] reload(eboptions) eb_go = eboptions.parse_options(args=[]) self.assertEqual(eb_go.options.configfiles, cfg_files) self.assertEqual(eb_go.options.prefix, '/home') # $XDG_CONFIG_HOME set, one directory listed in $XDG_CONFIG_DIRS os.environ['XDG_CONFIG_DIRS'] = dir1 cfg_files = [ os.path.join(dir1, 'easybuild.d', 'bar.cfg'), os.path.join(dir1, 'easybuild.d', 'foo.cfg'), os.path.join(homedir, 'easybuild', 'config.cfg'), # $XDG_CONFIG_HOME goes last ] reload(eboptions) eb_go = eboptions.parse_options(args=[]) self.assertEqual(eb_go.options.configfiles, cfg_files) self.assertEqual(eb_go.options.prefix, '/home') # last cfgfile wins # $XDG_CONFIG_HOME not set, multiple directories listed in $XDG_CONFIG_DIRS del os.environ['XDG_CONFIG_HOME'] # unset, so should become default os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join([dir1, dir2, dir3]) cfg_files = [ os.path.join(dir1, 'easybuild.d', 'bar.cfg'), os.path.join(dir1, 'easybuild.d', 'foo.cfg'), os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'), ] reload(eboptions) eb_go = eboptions.parse_options(args=[]) # note: there may be a config file in $HOME too, so don't use a strict comparison self.assertEqual(cfg_files, eb_go.options.configfiles[:3]) # $XDG_CONFIG_HOME set to non-existing directory, multiple directories listed in $XDG_CONFIG_DIRS os.environ['XDG_CONFIG_HOME'] = os.path.join(self.test_prefix, 'nosuchdir') cfg_files = [ os.path.join(dir1, 'easybuild.d', 'bar.cfg'), os.path.join(dir1, 'easybuild.d', 'foo.cfg'), os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'), ] reload(eboptions) eb_go = eboptions.parse_options(args=[]) self.assertEqual(eb_go.options.configfiles, cfg_files) self.assertEqual(eb_go.options.prefix, '/foobarbaz') # last cfgfile wins # restore $XDG_CONFIG env vars to original state if xdg_config_home is None: del os.environ['XDG_CONFIG_HOME'] else: os.environ['XDG_CONFIG_HOME'] = xdg_config_home if xdg_config_dirs is None: del os.environ['XDG_CONFIG_DIRS'] else: os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs reload(eboptions)
def test_generaloption_config_file(self): """Test use of new-style configuration file.""" self.purge_environment() config_file = os.path.join(self.tmpdir, 'testconfig.cfg') testpath1 = os.path.join(self.tmpdir, 'test1') testpath2 = os.path.join(self.tmpdir, 'testtwo') # test with config file passed via command line cfgtxt = '\n'.join([ '[config]', 'installpath = %s' % testpath2, ]) write_file(config_file, cfgtxt) installpath_software = tempfile.mkdtemp(prefix='installpath-software') args = [ '--configfiles', config_file, '--debug', '--buildpath', testpath1, '--installpath-software', installpath_software, ] options = init_config(args=args) self.assertEqual(build_path(), testpath1) # via command line self.assertEqual(source_paths(), [ os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources') ]) # default self.assertEqual(install_path(), installpath_software) # via cmdline arg self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory # to check whether easyconfigs install path is auto-included in robot path tmpdir = tempfile.mkdtemp( prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') copy_dir(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert( 0, tmpdir ) # prepend to give it preference over possible other installed easyconfigs pkgs # test with config file passed via environment variable # also test for existence of HOME and USER by adding paths to robot-paths installpath_modules = tempfile.mkdtemp(prefix='installpath-modules') cfgtxt = '\n'.join([ '[config]', 'buildpath = %s' % testpath1, 'sourcepath = %(DEFAULT_REPOSITORYPATH)s', 'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir', 'robot-paths=/tmp/foo:%(sourcepath)s:%(HOME)s:/tmp/%(USER)s:%(DEFAULT_ROBOT_PATHS)s', 'installpath-modules=%s' % installpath_modules, ]) write_file(config_file, cfgtxt) os.environ['EASYBUILD_CONFIGFILES'] = config_file args = [ '--debug', '--sourcepath', testpath2, ] options = init_config(args=args) topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild') self.assertEqual(install_path(), os.path.join(topdir, 'software')) # default self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir' ]) # via config file # hardcoded first entry self.assertEqual(options.robot_paths[0], '/tmp/foo') # resolved value for %(sourcepath)s template self.assertEqual( options.robot_paths[1], os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'ebfiles_repo')) # resolved value for HOME constant self.assertEqual(options.robot_paths[2], os.getenv('HOME')) # resolved value that uses USER constant self.assertEqual(options.robot_paths[3], os.path.join('/tmp', os.getenv('USER'))) # first path in DEFAULT_ROBOT_PATHS self.assertEqual(options.robot_paths[4], os.path.join(tmpdir, 'easybuild', 'easyconfigs')) testpath3 = os.path.join(self.tmpdir, 'testTHREE') os.environ['EASYBUILD_SOURCEPATH'] = testpath2 args = [ '--debug', '--installpath', testpath3, ] options = init_config(args=args) self.assertEqual( source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(build_path(), testpath1) # via config file del os.environ['EASYBUILD_CONFIGFILES'] sys.path[:] = orig_sys_path
def test_generaloption_config(self): """Test new-style configuration (based on generaloption).""" self.purge_environment() # check whether configuration via environment variables works as expected prefix = os.path.join(self.tmpdir, 'testprefix') buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build', 'path') os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var options = init_config(args=[]) self.assertEqual(build_path(), buildpath_env_var) self.assertEqual(install_path(), os.path.join(prefix, 'software')) self.assertEqual(get_repositorypath(), [os.path.join(prefix, 'ebfiles_repo')]) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_BUILDPATH'] # check whether configuration via command line arguments works prefix = os.path.join(self.tmpdir, 'test1') install = os.path.join(self.tmpdir, 'test2', 'install') repopath = os.path.join(self.tmpdir, 'test2', 'repo') config_file = os.path.join(self.tmpdir, 'nooldconfig.py') write_file(config_file, '') args = [ '--configfiles', config_file, # force empty config file '--prefix', prefix, '--installpath', install, '--repositorypath', repopath, '--subdir-software', 'APPS', ] options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, 'APPS')) self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) self.assertTrue(config_file in options.configfiles) # check mixed command line/env var configuration prefix = os.path.join(self.tmpdir, 'test3') install = os.path.join(self.tmpdir, 'test4', 'install') subdir_software = 'eb-soft' args = [ '--configfiles', config_file, # force empty config file '--installpath', install, ] os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software installpath_modules = tempfile.mkdtemp(prefix='installpath-modules') os.environ['EASYBUILD_INSTALLPATH_MODULES'] = installpath_modules options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, subdir_software)) self.assertEqual(install_path('mod'), installpath_modules) # subdir options *must* be relative (to --installpath) installpath_software = tempfile.mkdtemp(prefix='installpath-software') os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = installpath_software error_regex = r"Found problems validating the options.*'subdir_software' must specify a \*relative\* path" self.assertErrorRegex(EasyBuildError, error_regex, init_config) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def writeEC(self): """ create temporary easyconfig file """ write_file(self.eb_file, self.contents)
class EB_SuiteSparse(ConfigureMake): """Support for building SuiteSparse.""" def __init__(self, *args, **kwargs): """Custom constructor for SuiteSparse easyblock, initialize custom class parameters.""" super(EB_SuiteSparse, self).__init__(*args, **kwargs) self.config_name = 'UNKNOWN' def configure_step(self): """Configure build by patching UFconfig.mk or SuiteSparse_config.mk.""" if LooseVersion(self.version) < LooseVersion('4.0'): self.config_name = 'UFconfig' else: self.config_name = 'SuiteSparse_config' cfgvars = { 'CC': os.getenv('MPICC'), 'CFLAGS': os.getenv('CFLAGS'), 'CXX': os.getenv('MPICXX'), 'F77': os.getenv('MPIF77'), 'F77FLAGS': os.getenv('F77FLAGS'), } # Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt self.cfg.update('buildopts', 'BLAS="%s"' % os.getenv('LIBBLAS_MT')) self.cfg.update('buildopts', 'LAPACK="%s"' % os.getenv('LIBLAPACK_MT')) # Get METIS or ParMETIS settings metis = get_software_root('METIS') parmetis = get_software_root('ParMETIS') if parmetis: metis_path = parmetis metis_include = os.path.join(parmetis, 'include') metis_libs = os.path.join(parmetis, get_software_libdir('ParMETIS'), 'libmetis.a') elif metis: metis_path = metis metis_include = os.path.join(metis, 'include') metis_libs = os.path.join(metis, get_software_libdir('METIS'), 'libmetis.a') else: raise EasyBuildError("Neither METIS or ParMETIS module loaded.") if LooseVersion(self.version) >= LooseVersion('4.5.1'): cfgvars.update({ 'MY_METIS_LIB': metis_libs, 'MY_METIS_INC': metis_include, }) else: cfgvars.update({ 'METIS_PATH': metis_path, 'METIS': metis_libs, }) # patch file fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name) try: for line in fileinput.input(fp, inplace=1, backup='.orig'): for (var, val) in cfgvars.items(): orig_line = line # for variables in cfgvars, substiture lines assignment # in the file, whatever they are, by assignments to the # values in cfgvars line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var, r"\1 %s # patched by EasyBuild\n" % val, line) if line != orig_line: cfgvars.pop(var) sys.stdout.write(line) except IOError, err: raise EasyBuildError("Failed to patch %s in: %s", fp, err) # add remaining entries at the end if cfgvars: cfgtxt = '# lines below added automatically by EasyBuild\n' cfgtxt += '\n'.join( ["%s = %s" % (var, val) for (var, val) in cfgvars.items()]) write_file(fp, cfgtxt, append=True)
DEFAULT_EXAMPLE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'examples') DEFAULT_MODULE = 'easybuild.easyblocks.generic' options = { 'out-file': ('Path to output file', 'string', 'store', None, 'o'), 'examples': ('Path to dir that contains example files', 'string', 'store', DEFAULT_EXAMPLE_PATH, 'e'), 'module': ('Name of module to load the easyblocks from', 'string', 'store', DEFAULT_MODULE, 'm') } so = simple_option(options) config.init_build_options({'validate': False, 'external_modules_metadata': {}}) autogen_comment = [ ".. This file is automatically generated using the %s script, " % os.path.basename(__file__), ".. and information and docstrings from easyblocks and the EasyBuild framework.", ".. Doo not edit this file manually, but update the docstrings and regenerate it.", '', ] easyblocks_overview = gen_easyblocks_overview_rst(so.options.module, so.options.examples, COMMON_PARAMS, DOC_FUNCTIONS) txt = '\n'.join(autogen_comment + easyblocks_overview) if so.options.out_file: write_file(so.options.out_file, txt) print '%s updated' % so.options.out_file else: print txt
def retrieve_blocks_in_spec(spec, only_blocks, silent=False): """ Easyconfigs can contain blocks (headed by a [Title]-line) which contain commands specific to that block. Commands in the beginning of the file above any block headers are common and shared between each block. """ reg_block = re.compile(r"^\s*\[([\w.-]+)\]\s*$", re.M) reg_dep_block = re.compile(r"^\s*block\s*=(\s*.*?)\s*$", re.M) spec_fn = os.path.basename(spec) txt = read_file(spec) # split into blocks using regex pieces = reg_block.split(txt) # the first block contains common statements common = pieces.pop(0) # determine version of easyconfig format ec_format_version = get_format_version(txt) if ec_format_version is None: ec_format_version = FORMAT_DEFAULT_VERSION _log.debug( "retrieve_blocks_in_spec: derived easyconfig format version: %s" % ec_format_version) # blocks in easyconfigs are only supported in easyconfig format 1.0 if pieces and ec_format_version == EasyVersion('1.0'): # make a map of blocks blocks = [] while pieces: block_name = pieces.pop(0) block_contents = pieces.pop(0) if block_name in [b['name'] for b in blocks]: raise EasyBuildError("Found block %s twice in %s.", block_name, spec) block = {'name': block_name, 'contents': block_contents} # dependency block dep_block = reg_dep_block.search(block_contents) if dep_block: dependencies = eval(dep_block.group(1)) if isinstance(dependencies, list): block['dependencies'] = dependencies else: block['dependencies'] = [dependencies] blocks.append(block) # make a new easyconfig for each block # they will be processed in the same order as they are all described in the original file specs = [] for block in blocks: name = block['name'] if only_blocks and not (name in only_blocks): print_msg("Skipping block %s-%s" % (spec_fn, name), silent=silent) continue (fd, block_path) = tempfile.mkstemp(prefix='easybuild-', suffix='%s-%s' % (spec_fn, name)) os.close(fd) txt = common if 'dependencies' in block: for dep in block['dependencies']: if dep not in [b['name'] for b in blocks]: raise EasyBuildError( "Block %s depends on %s, but block was not found.", name, dep) dep = [b for b in blocks if b['name'] == dep][0] txt += "\n# Dependency block %s" % (dep['name']) txt += dep['contents'] txt += "\n# Main block %s" % name txt += block['contents'] write_file(block_path, txt) specs.append(block_path) _log.debug("Found %s block(s) in %s" % (len(specs), spec)) return specs else: # no blocks, one file return [spec]
def build_and_install_software(ecs, init_session_state, exit_on_failure=True, hooks=None): """ Build and install software for all provided parsed easyconfig files. :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports :param exit_on_failure: whether or not to exit on installation failure :param hooks: list of defined pre- and post-step hooks """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) run_hook(START, hooks) res = [] for ec in ecs: ec_res = {} try: (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, hooks=hooks) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) except Exception, err: # purposely catch all exceptions ec_res['success'] = False ec_res['err'] = err ec_res['traceback'] = traceback.format_exc() # keep track of success/total count if ec_res['success']: test_msg = "Successfully built %s" % ec['spec'] else: test_msg = "Build of %s failed" % ec['spec'] if 'err' in ec_res: test_msg += " (err: %s)" % ec_res['err'] # dump test report next to log file test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) if 'log_file' in ec_res and ec_res['log_file']: test_report_fp = "%s_test_report.md" % '.'.join( ec_res['log_file'].split('.')[:-1]) parent_dir = os.path.dirname(test_report_fp) # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used if os.stat(parent_dir).st_mode & 0200: write_file(test_report_fp, test_report_txt) else: adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) write_file(test_report_fp, test_report_txt) adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) if not ec_res['success'] and exit_on_failure: if 'traceback' in ec_res: raise EasyBuildError(ec_res['traceback']) else: raise EasyBuildError(test_msg) res.append((ec, ec_res))
def test_end2end_singularity_image(self): """End-to-end test for --containerize (recipe + image).""" topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) test_img = os.path.join(self.test_prefix, 'test123.img') write_file(test_img, '') args = [ toy_ec, '-C', # equivalent with --containerize '--experimental', '--container-config=bootstrap=localimage,from=%s' % test_img, '--container-build-image', ] if which('singularity') is None: error_pattern = "singularity with version 2.4 or higher not found on your system." self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) # install mocked versions of 'sudo' and 'singularity' commands singularity = os.path.join(self.test_prefix, 'bin', 'singularity') write_file(singularity, '') # placeholder adjust_permissions(singularity, stat.S_IXUSR, add=True) sudo = os.path.join(self.test_prefix, 'bin', 'sudo') write_file( sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n' ) adjust_permissions(sudo, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join( [os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) for (version, ext) in [('2.4.0', 'simg'), ('3.1.0', 'sif')]: write_file(singularity, MOCKED_SINGULARITY % {'version': version}) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs = [ r"^== singularity tool found at %s/bin/singularity" % self.test_prefix, r"^== singularity version '%s' is 2.4 or higher ... OK" % version, r"^== Singularity definition file created at %s/containers/Singularity\.toy-0.0" % self.test_prefix, r"^== Running 'sudo\s*\S*/singularity build\s*/.* /.*', you may need to enter your 'sudo' password...", r"^== Singularity image created at %s/containers/toy-0.0\.%s" % (self.test_prefix, ext), ] self.check_regexs(regexs, stdout) self.assertTrue( os.path.exists(os.path.join(containerpath, 'toy-0.0.%s' % ext))) remove_file(os.path.join(containerpath, 'Singularity.toy-0.0')) # check use of --container-image-format & --container-image-name write_file(singularity, MOCKED_SINGULARITY % {'version': '2.4.0'}) args.extend([ "--container-image-format=ext3", "--container-image-name=foo-bar", ]) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs = [ r"^== singularity tool found at %s/bin/singularity" % self.test_prefix, r"^== singularity version '2.4.0' is 2.4 or higher ... OK", r"^== Singularity definition file created at %s/containers/Singularity\.foo-bar" % self.test_prefix, r"^== Running 'sudo\s*\S*/singularity build --writable /.* /.*', you may need to enter .*", r"^== Singularity image created at %s/containers/foo-bar\.img$" % self.test_prefix, ] self.check_regexs(regexs, stdout) cont_img = os.path.join(containerpath, 'foo-bar.img') self.assertTrue(os.path.exists(cont_img)) remove_file(os.path.join(containerpath, 'Singularity.foo-bar')) # test again with container image already existing error_pattern = "Container image already exists at %s, not overwriting it without --force" % cont_img self.mock_stdout(True) self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True) self.mock_stdout(False) args.append('--force') stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs.extend([ "WARNING: overwriting existing container image at %s due to --force" % cont_img, ]) self.check_regexs(regexs, stdout) self.assertTrue(os.path.exists(cont_img)) # also check behaviour under --extended-dry-run args.append('--extended-dry-run') stdout, stderr = self.run_main(args) self.assertFalse(stderr) self.check_regexs(regexs, stdout) # test use of --container-tmpdir args.append('--container-tmpdir=%s' % self.test_prefix) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs[ -3] = r"^== Running 'sudo\s*SINGULARITY_TMPDIR=%s \S*/singularity build .*" % self.test_prefix self.check_regexs(regexs, stdout)
def test_find_flexlm_license(self): """Test find_flexlm_license function.""" lic_file1 = os.path.join(self.test_prefix, 'one.lic') ft.write_file(lic_file1, "This is a license file (no, really!)") lic_file2 = os.path.join(self.test_prefix, 'two.dat') ft.write_file(lic_file2, "This is another license file (sure it is!)") lic_server = '*****@*****.**' # make test robust against environment in which $LM_LICENSE_FILE is defined if 'LM_LICENSE_FILE' in os.environ: del os.environ['LM_LICENSE_FILE'] # default return value self.assertEqual(ft.find_flexlm_license(), ([], None)) # provided license spec self.assertEqual(ft.find_flexlm_license(lic_specs=[lic_file1]), ([lic_file1], None)) self.assertEqual( ft.find_flexlm_license(lic_specs=[lic_server, lic_file2]), ([lic_server, lic_file2], None)) # non-existing license file os.environ[ 'LM_LICENSE_FILE'] = '/no/such/file/unless/you/aim/to/break/this/check' self.assertEqual(ft.find_flexlm_license(), ([], None)) # existing license file os.environ['LM_LICENSE_FILE'] = lic_file2 self.assertEqual(ft.find_flexlm_license(), ([lic_file2], 'LM_LICENSE_FILE')) # directory with existing license files os.environ['LM_LICENSE_FILE'] = self.test_prefix self.assertEqual(ft.find_flexlm_license(), ([lic_file1, lic_file2], 'LM_LICENSE_FILE')) # server spec os.environ['LM_LICENSE_FILE'] = lic_server self.assertEqual(ft.find_flexlm_license(), ([lic_server], 'LM_LICENSE_FILE')) # duplicates are filtered out, order is maintained os.environ['LM_LICENSE_FILE'] = ':'.join( [lic_file1, lic_server, self.test_prefix, lic_file2, lic_file1]) self.assertEqual( ft.find_flexlm_license(), ([lic_file1, lic_server, lic_file2], 'LM_LICENSE_FILE')) # invalid server spec (missing port) os.environ['LM_LICENSE_FILE'] = 'test.license.server' self.assertEqual(ft.find_flexlm_license(), ([], None)) # env var wins of provided lic spec os.environ['LM_LICENSE_FILE'] = lic_file2 self.assertEqual(ft.find_flexlm_license(lic_specs=[lic_server]), ([lic_file2], 'LM_LICENSE_FILE')) # custom env var wins over $LM_LICENSE_FILE os.environ['INTEL_LICENSE_FILE'] = lic_file1 expected = ([lic_file1], 'INTEL_LICENSE_FILE') self.assertEqual( ft.find_flexlm_license(custom_env_vars='INTEL_LICENSE_FILE'), expected) self.assertEqual( ft.find_flexlm_license(custom_env_vars=['INTEL_LICENSE_FILE']), expected) self.assertEqual( ft.find_flexlm_license( custom_env_vars=['NOSUCHENVVAR', 'INTEL_LICENSE_FILE']), expected) # $LM_LICENSE_FILE is always considered os.environ['LM_LICENSE_FILE'] = lic_server os.environ[ 'INTEL_LICENSE_FILE'] = '/no/such/file/unless/you/aim/to/break/this/check' expected = ([lic_server], 'LM_LICENSE_FILE') self.assertEqual( ft.find_flexlm_license(custom_env_vars=['INTEL_LICENSE_FILE']), expected) # license server *and* file spec; order is preserved os.environ['LM_LICENSE_FILE'] = ':'.join( [lic_file2, lic_server, lic_file1]) self.assertEqual( ft.find_flexlm_license(), ([lic_file2, lic_server, lic_file1], 'LM_LICENSE_FILE')) # typical usage os.environ['LM_LICENSE_FILE'] = lic_server os.environ[ 'INTEL_LICENSE_FILE'] = '/not/a/valid/license/path:%s:/another/bogus/license/file' % lic_file2 expected = ([lic_file2], 'INTEL_LICENSE_FILE') self.assertEqual( ft.find_flexlm_license(custom_env_vars='INTEL_LICENSE_FILE'), expected) os.environ[ 'INTEL_LICENSE_FILE'] = '[email protected]:[email protected]:[email protected]' expected = (['*****@*****.**', '*****@*****.**', '*****@*****.**'], 'INTEL_LICENSE_FILE') self.assertEqual( ft.find_flexlm_license(custom_env_vars=['INTEL_LICENSE_FILE']), expected) # make sure find_flexlm_license is robust against None input; # this occurs if license_file is left unspecified del os.environ['INTEL_LICENSE_FILE'] del os.environ['LM_LICENSE_FILE'] self.assertEqual(ft.find_flexlm_license(lic_specs=[None]), ([], None))
def test_fix_broken_easyconfig(self): """Test fix_broken_easyconfigs.py script.""" testdir = os.path.dirname(__file__) topdir = os.path.dirname(os.path.dirname(testdir)) script = os.path.join(topdir, 'easybuild', 'scripts', 'fix_broken_easyconfigs.py') test_easyblocks = os.path.join(testdir, 'sandbox') broken_ec_txt_tmpl = '\n'.join([ "# licenseheader", "%sname = '%s'", "version = '1.2.3'", '', "description = 'foo'", "homepage = 'http://example.com'", '', "toolchain = {'name': 'GCC', 'version': '4.8.2'}", '', "premakeopts = 'FOO=libfoo.%%s' %% shared_lib_ext", "makeopts = 'CC=gcc'", '', "license = 'foo.lic'", ]) fixed_ec_txt_tmpl = '\n'.join([ "# licenseheader", "%sname = '%s'", "version = '1.2.3'", '', "description = 'foo'", "homepage = 'http://example.com'", '', "toolchain = {'name': 'GCC', 'version': '4.8.2'}", '', "prebuildopts = 'FOO=libfoo.%%s' %% SHLIB_EXT", "buildopts = 'CC=gcc'", '', "license_file = 'foo.lic'", ]) broken_ec_tmpl = os.path.join(self.test_prefix, '%s.eb') script_cmd_tmpl = "PYTHONPATH=%s:$PYTHONPATH:%s %s %%s" % ( topdir, test_easyblocks, script) # don't change it if it isn't broken broken_ec = broken_ec_tmpl % 'notbroken' script_cmd = script_cmd_tmpl % broken_ec fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'ConfigureMake'\n\n", 'foo') write_file(broken_ec, fixed_ec_txt) # (dummy) ConfigureMake easyblock is available in test sandbox script_cmd = script_cmd_tmpl % broken_ec new_ec_txt = read_file(broken_ec) self.assertEqual(new_ec_txt, fixed_ec_txt) self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) self.assertFalse(os.path.exists( '%s.bk' % broken_ec)) # no backup created if nothing was fixed broken_ec = broken_ec_tmpl % 'nosuchsoftware' script_cmd = script_cmd_tmpl % broken_ec broken_ec_txt = broken_ec_txt_tmpl % ('', 'nosuchsoftware') fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'ConfigureMake'\n\n", 'nosuchsoftware') # broken easyconfig is fixed in place, original file is backed up write_file(broken_ec, broken_ec_txt) run_cmd(script_cmd) new_ec_txt = read_file(broken_ec) self.assertEqual(new_ec_txt, fixed_ec_txt) self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt) self.assertFalse(os.path.exists('%s.bk1' % broken_ec)) # broken easyconfig is fixed in place, original file is backed up, existing backup is not overwritten write_file(broken_ec, broken_ec_txt) write_file('%s.bk' % broken_ec, 'thisshouldnot\nbechanged') run_cmd(script_cmd) new_ec_txt = read_file(broken_ec) self.assertEqual(new_ec_txt, fixed_ec_txt) self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) self.assertEqual(read_file('%s.bk' % broken_ec), 'thisshouldnot\nbechanged') self.assertEqual(read_file('%s.bk1' % broken_ec), broken_ec_txt) # if easyblock is specified, that part is left untouched broken_ec = broken_ec_tmpl % 'footoy' script_cmd = script_cmd_tmpl % broken_ec broken_ec_txt = broken_ec_txt_tmpl % ("easyblock = 'EB_toy'\n\n", 'foo') fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'EB_toy'\n\n", 'foo') write_file(broken_ec, broken_ec_txt) run_cmd(script_cmd) new_ec_txt = read_file(broken_ec) self.assertEqual(new_ec_txt, fixed_ec_txt) self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt) # for existing easyblocks, "easyblock = 'ConfigureMake'" should *not* be added # EB_toy easyblock is available in test sandbox test_easyblocks = os.path.join(testdir, 'sandbox') broken_ec = broken_ec_tmpl % 'toy' # path to test easyblocks must be *appended* to PYTHONPATH (due to flattening in easybuild-easyblocks repo) script_cmd = script_cmd_tmpl % broken_ec broken_ec_txt = broken_ec_txt_tmpl % ('', 'toy') fixed_ec_txt = fixed_ec_txt_tmpl % ('', 'toy') write_file(broken_ec, broken_ec_txt) run_cmd(script_cmd) new_ec_txt = read_file(broken_ec) self.assertEqual(new_ec_txt, fixed_ec_txt) self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt)
def test_move_logs(self): """Test move_logs function.""" fh, fp = tempfile.mkstemp() os.close(fh) ft.write_file(fp, 'foobar') ft.write_file(fp + '.1', 'moarfoobar') ft.move_logs(fp, os.path.join(self.test_prefix, 'foo.log')) self.assertEqual( ft.read_file(os.path.join(self.test_prefix, 'foo.log')), 'foobar') self.assertEqual( ft.read_file(os.path.join(self.test_prefix, 'foo.log.1')), 'moarfoobar') ft.write_file(os.path.join(self.test_prefix, 'bar.log'), 'bar') ft.write_file(os.path.join(self.test_prefix, 'bar.log_1'), 'barbar') fh, fp = tempfile.mkstemp() os.close(fh) ft.write_file(fp, 'moarbar') ft.write_file(fp + '.1', 'evenmoarbar') ft.move_logs(fp, os.path.join(self.test_prefix, 'bar.log')) logs = [ 'bar.log', 'bar.log.1', 'bar.log_0', 'bar.log_1', os.path.basename(self.logfile), 'foo.log', 'foo.log.1' ] self.assertEqual( sorted([f for f in os.listdir(self.test_prefix) if 'log' in f]), logs) self.assertEqual( ft.read_file(os.path.join(self.test_prefix, 'bar.log_0')), 'bar') self.assertEqual( ft.read_file(os.path.join(self.test_prefix, 'bar.log_1')), 'barbar') self.assertEqual( ft.read_file(os.path.join(self.test_prefix, 'bar.log')), 'moarbar') self.assertEqual( ft.read_file(os.path.join(self.test_prefix, 'bar.log.1')), 'evenmoarbar')
txt = '\n'.join([ "nwchem_basis_library %(path)s/data/libraries/", "nwchem_nwpw_library %(path)s/data/libraryps/", "ffield amber", "amber_1 %(path)s/data/amber_s/", "amber_2 %(path)s/data/amber_q/", "amber_3 %(path)s/data/amber_x/", "amber_4 %(path)s/data/amber_u/", "spce %(path)s/data/solvents/spce.rst", "charmm_s %(path)s/data/charmm_s/", "charmm_x %(path)s/data/charmm_x/", ]) % { 'path': self.installdir } write_file(default_nwchemrc, txt) # fix permissions in data directory datadir = os.path.join(self.installdir, 'data') adjust_permissions(datadir, stat.S_IROTH, add=True, recursive=True) adjust_permissions(datadir, stat.S_IXOTH, add=True, recursive=True, onlydirs=True) def sanity_check_step(self): """Custom sanity check for NWChem.""" custom_paths = { 'files': ['bin/nwchem'], 'dirs': [
def configure_step(self): """ Configure for GCC build: - prepare extra source dirs (GMP, MPFR, MPC, ...) - create obj dir to build in (GCC doesn't like to be built in source dir) - add configure and make options, according to .eb spec file - decide whether or not to do a staged build (which is required to enable PPL/CLooG support) - set platform_lib based on config.guess output """ sysroot = build_option('sysroot') if sysroot: # based on changes made to GCC in Gentoo Prefix # https://gitweb.gentoo.org/repo/gentoo.git/tree/profiles/features/prefix/standalone/profile.bashrc # add --with-sysroot configure option, to instruct GCC to consider # value set for EasyBuild's --sysroot configuration option as the root filesystem of the operating system # (see https://gcc.gnu.org/install/configure.html) self.cfg.update('configopts', '--with-sysroot=%s' % sysroot) # avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c apply_regex_substitutions(os.path.join('gcc', 'gcc.c'), [('--sysroot=%R', '')]) # prefix dynamic linkers with sysroot # this patches lines like: # #define GLIBC_DYNAMIC_LINKER64 "/lib64/ld-linux-x86-64.so.2" gcc_config_headers = glob.glob( os.path.join('gcc', 'config', '*', '*linux*.h')) regex_subs = [('(_DYNAMIC_LINKER.*[":])/lib', r'\1%s/lib' % sysroot)] apply_regex_substitutions(gcc_config_headers, regex_subs) # self.configopts will be reused in a 3-staged build, # configopts is only used in first configure self.configopts = self.cfg['configopts'] # I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them stage1_info = self.prep_extra_src_dirs("stage1") configopts = stage1_info['configopts'] # II) update config options # enable specified language support if self.cfg['languages']: self.configopts += " --enable-languages=%s" % ','.join( self.cfg['languages']) # enable building of libiberty, if desired if self.cfg['withlibiberty']: self.configopts += " --enable-install-libiberty" # enable link-time-optimization (LTO) support, if desired if self.cfg['withlto']: self.configopts += " --enable-lto" else: self.configopts += " --disable-lto" # configure for a release build self.configopts += " --enable-checking=release " # enable multilib: allow both 32 and 64 bit if self.cfg['multilib']: glibc_32bit = [ "glibc.i686", # Fedora, RedHat-based "glibc.ppc", # "" on Power "libc6-dev-i386", # Debian-based "gcc-c++-32bit", # OpenSuSE, SLES ] if not any([check_os_dependency(dep) for dep in glibc_32bit]): raise EasyBuildError( "Using multilib requires 32-bit glibc (install one of %s, depending on your OS)", ', '.join(glibc_32bit)) self.configopts += " --enable-multilib --with-multilib-list=m32,m64" else: self.configopts += " --disable-multilib" # build both static and dynamic libraries (???) self.configopts += " --enable-shared=yes --enable-static=yes " # use POSIX threads self.configopts += " --enable-threads=posix " # enable plugin support self.configopts += " --enable-plugins " # use GOLD as default linker if self.cfg['use_gold_linker']: self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold" else: self.configopts += " --enable-gold --enable-ld=default" # enable bootstrap build for self-containment (unless for staged build) if not self.stagedbuild: configopts += " --enable-bootstrap" else: configopts += " --disable-bootstrap" if self.stagedbuild: # # STAGE 1: configure GCC build that will be used to build PPL/CLooG # self.log.info( "Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support..." ) self.stage1installdir = os.path.join(self.builddir, 'GCC_stage1_eb') configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % { 'p': self.stage1installdir } else: # unstaged build, so just run standard configure/make/make install # set prefixes self.log.info("Performing regular GCC build...") configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % { 'p': self.installdir } # prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config # only do this when multilib is not enabled if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']: cfgfile = 'gcc/config/i386/t-linux64' multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32" self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'", cfgfile, multilib_osdirnames) write_file(cfgfile, multilib_osdirnames, append=True) elif self.cfg['multilib']: self.log.info( "Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled" ) # III) create obj dir to build in, and change to it # GCC doesn't like to be built in the source dir if self.stagedbuild: objdir = self.create_dir("stage1_obj") self.stage1prefix = objdir else: objdir = self.create_dir("obj") # note: this also triggers the use of an updated config.guess script # (unless both the 'build_type' and 'host_type' easyconfig parameters are specified) build_type, host_type = self.determine_build_and_host_type() if build_type: configopts += ' --build=' + build_type if host_type: configopts += ' --host=' + host_type # IV) actual configure, but not on default path cmd = "../configure %s %s" % (self.configopts, configopts) self.run_configure_cmd(cmd) self.disable_lto_mpfr_old_gcc(objdir)
def test_adjust_permissions(self): """Test adjust_permissions""" # set umask hard to run test reliably orig_umask = os.umask(0022) # prep files/dirs/(broken) symlinks is test dir # file: rw-r--r-- ft.write_file(os.path.join(self.test_prefix, 'foo'), 'foo') foo_perms = os.stat(os.path.join(self.test_prefix, 'foo'))[stat.ST_MODE] for bit in [stat.S_IRUSR, stat.S_IWUSR, stat.S_IRGRP, stat.S_IROTH]: self.assertTrue(foo_perms & bit) for bit in [ stat.S_IXUSR, stat.S_IWGRP, stat.S_IXGRP, stat.S_IWOTH, stat.S_IXOTH ]: self.assertFalse(foo_perms & bit) # dir: rwxr-xr-x ft.mkdir(os.path.join(self.test_prefix, 'bar')) bar_perms = os.stat(os.path.join(self.test_prefix, 'bar'))[stat.ST_MODE] for bit in [ stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IXOTH ]: self.assertTrue(bar_perms & bit) for bit in [stat.S_IWGRP, stat.S_IWOTH]: self.assertFalse(bar_perms & bit) # file in dir: rw-r--r-- foobar_path = os.path.join(self.test_prefix, 'bar', 'foobar') ft.write_file(foobar_path, 'foobar') foobar_perms = os.stat(foobar_path)[stat.ST_MODE] for bit in [stat.S_IRUSR, stat.S_IWUSR, stat.S_IRGRP, stat.S_IROTH]: self.assertTrue(foobar_perms & bit) for bit in [ stat.S_IXUSR, stat.S_IWGRP, stat.S_IXGRP, stat.S_IWOTH, stat.S_IXOTH ]: self.assertFalse(foobar_perms & bit) # include symlink os.symlink(foobar_path, os.path.join(self.test_prefix, 'foobar_symlink')) # include broken symlink (symlinks are skipped, so this shouldn't cause problems) tmpfile = os.path.join(self.test_prefix, 'thiswontbetherelong') ft.write_file(tmpfile, 'poof!') os.symlink(tmpfile, os.path.join(self.test_prefix, 'broken_symlink')) os.remove(tmpfile) # test default behaviour: # recursive, add permissions, relative to existing permissions, both files and dirs, skip symlinks # add user execution, group write permissions ft.adjust_permissions(self.test_prefix, stat.S_IXUSR | stat.S_IWGRP) # foo file: rwxrw-r-- foo_perms = os.stat(os.path.join(self.test_prefix, 'foo'))[stat.ST_MODE] for bit in [ stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRGRP, stat.S_IWGRP, stat.S_IROTH ]: self.assertTrue(foo_perms & bit) for bit in [stat.S_IXGRP, stat.S_IWOTH, stat.S_IXOTH]: self.assertFalse(foo_perms & bit) # bar dir: rwxrwxr-x bar_perms = os.stat(os.path.join(self.test_prefix, 'bar'))[stat.ST_MODE] for bit in [ stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IXOTH ]: self.assertTrue(bar_perms & bit) self.assertFalse(bar_perms & stat.S_IWOTH) # foo/foobar file: rwxrw-r-- for path in [ os.path.join(self.test_prefix, 'bar', 'foobar'), os.path.join(self.test_prefix, 'foobar_symlink') ]: perms = os.stat(path)[stat.ST_MODE] for bit in [ stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRGRP, stat.S_IWGRP, stat.S_IROTH ]: self.assertTrue(perms & bit) for bit in [stat.S_IXGRP, stat.S_IWOTH, stat.S_IXOTH]: self.assertFalse(perms & bit) # broken symlinks are trouble if symlinks are not skipped self.assertErrorRegex(EasyBuildError, "No such file or directory", ft.adjust_permissions, self.test_prefix, stat.S_IXUSR, skip_symlinks=False) # restore original umask os.umask(orig_umask)
def post_install_step(self): """ Install group libraries and interfaces (if desired). """ super(EB_imkl, self).post_install_step() # extract examples examples_subdir = os.path.join(self.installdir, self.mkl_basedir, 'examples') if os.path.exists(examples_subdir): cwd = change_dir(examples_subdir) for examples_tarball in glob.glob('examples_*.tgz'): run_cmd("tar xvzf %s -C ." % examples_tarball) change_dir(cwd) # reload the dependencies self.load_dependency_modules() shlib_ext = get_shared_lib_ext() if self.cfg['m32']: extra = { 'libmkl.%s' % shlib_ext: 'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } else: extra = { 'libmkl.%s' % shlib_ext: 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } loosever = LooseVersion(self.version) if loosever >= LooseVersion('10.3'): libsubdir = os.path.join(self.mkl_basedir, 'lib', 'intel64') else: if self.cfg['m32']: libsubdir = os.path.join('lib', '32') else: libsubdir = os.path.join('lib', 'em64t') for fil, txt in extra.items(): dest = os.path.join(self.installdir, libsubdir, fil) if not os.path.exists(dest): write_file(dest, txt) # build the mkl interfaces, if desired if self.cfg['interfaces']: if loosever >= LooseVersion('10.3'): intsubdir = os.path.join(self.mkl_basedir, 'interfaces') inttarget = 'libintel64' else: intsubdir = 'interfaces' if self.cfg['m32']: inttarget = 'lib32' else: inttarget = 'libem64t' cmd = "make -f makefile %s" % inttarget # blas95 and lapack95 need more work, ignore for now # blas95 and lapack also need include/.mod to be processed fftw2libs = ['fftw2xc', 'fftw2xf'] fftw3libs = ['fftw3xc', 'fftw3xf'] interfacedir = os.path.join(self.installdir, intsubdir) change_dir(interfacedir) self.log.info("Changed to interfaces directory %s", interfacedir) compopt = None # determine whether we're using a non-Intel GCC-based or PGI-based toolchain # can't use toolchain.comp_family, because of system toolchain used when installing imkl if get_software_root('icc') or get_software_root( 'intel-compilers'): compopt = 'compiler=intel' else: # check for PGI first, since there's a GCC underneath PGI too... if get_software_root('PGI'): compopt = 'compiler=pgi' elif get_software_root('GCC'): compopt = 'compiler=gnu' else: raise EasyBuildError( "Not using Intel/GCC/PGI compilers, don't know how to build wrapper libs" ) # patch makefiles for cdft wrappers when PGI is used as compiler if get_software_root('PGI'): regex_subs = [ # pgi should be considered as a valid compiler ("intel gnu", "intel gnu pgi"), # transform 'gnu' case to 'pgi' case (r"ifeq \(\$\(compiler\),gnu\)", "ifeq ($(compiler),pgi)"), ('=gcc', '=pgcc'), # correct flag to use C99 standard ('-std=c99', '-c99'), # -Wall and -Werror are not valid options for pgcc, no close equivalent ('-Wall', ''), ('-Werror', ''), ] for lib in self.cdftlibs: apply_regex_substitutions( os.path.join(interfacedir, lib, 'makefile'), regex_subs) for lib in fftw2libs + fftw3libs + self.cdftlibs: buildopts = [compopt] if lib in fftw3libs: buildopts.append('install_to=$INSTALL_DIR') elif lib in self.cdftlibs: if self.mpi_spec is not None: buildopts.append('mpi=%s' % self.mpi_spec) precflags = [''] if lib.startswith('fftw2x') and not self.cfg['m32']: # build both single and double precision variants precflags = [ 'PRECISION=MKL_DOUBLE', 'PRECISION=MKL_SINGLE' ] intflags = [''] if lib in self.cdftlibs and not self.cfg['m32']: # build both 32-bit and 64-bit interfaces intflags = ['interface=lp64', 'interface=ilp64'] allopts = [ list(opts) for opts in itertools.product(intflags, precflags) ] for flags, extraopts in itertools.product(['', '-fPIC'], allopts): tup = (lib, flags, buildopts, extraopts) self.log.debug( "Building lib %s with: flags %s, buildopts %s, extraopts %s" % tup) tmpbuild = tempfile.mkdtemp(dir=self.builddir) self.log.debug("Created temporary directory %s" % tmpbuild) # always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS # fftw2x(c|f): use $INSTALL_DIR, $CFLAGS and $COPTS # fftw3x(c|f): use $CFLAGS # fftw*cdft: use $INSTALL_DIR and $SPEC_OPT env.setvar('INSTALL_DIR', tmpbuild) env.setvar('SPEC_OPT', flags) env.setvar('COPTS', flags) env.setvar('CFLAGS', flags) try: intdir = os.path.join(interfacedir, lib) os.chdir(intdir) self.log.info("Changed to interface %s directory %s" % (lib, intdir)) except OSError as err: raise EasyBuildError( "Can't change to interface %s directory %s: %s", lib, intdir, err) fullcmd = "%s %s" % (cmd, ' '.join(buildopts + extraopts)) res = run_cmd(fullcmd, log_all=True, simple=True) if not res: raise EasyBuildError( "Building %s (flags: %s, fullcmd: %s) failed", lib, flags, fullcmd) for fn in os.listdir(tmpbuild): src = os.path.join(tmpbuild, fn) if flags == '-fPIC': # add _pic to filename ff = fn.split('.') fn = '.'.join(ff[:-1]) + '_pic.' + ff[-1] dest = os.path.join(self.installdir, libsubdir, fn) try: if os.path.isfile(src): shutil.move(src, dest) self.log.info("Moved %s to %s" % (src, dest)) except OSError as err: raise EasyBuildError("Failed to move %s to %s: %s", src, dest, err) remove_dir(tmpbuild)
def post_install_step(self, *args, **kwargs): """ Post-processing after installation: add symlinks for cc, c++, f77, f95 """ super(EB_GCC, self).post_install_step(*args, **kwargs) # Add symlinks for cc/c++/f77/f95. bindir = os.path.join(self.installdir, 'bin') for key in COMP_CMD_SYMLINKS: src = COMP_CMD_SYMLINKS[key] target = os.path.join(bindir, key) if os.path.exists(target): self.log.info( "'%s' already exists in %s, not replacing it with symlink to '%s'", key, bindir, src) elif os.path.exists(os.path.join(bindir, src)): symlink(src, target, use_abspath_source=False) else: raise EasyBuildError( "Can't link '%s' to non-existing location %s", target, os.path.join(bindir, src)) # Rename include-fixed directory which includes system header files that were processed by fixincludes, # since these may cause problems when upgrading to newer OS version. # (see https://github.com/easybuilders/easybuild-easyconfigs/issues/10666) glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc', '*-linux-gnu', self.version, 'include-fixed') matches = glob.glob(glob_pattern) if matches: if len(matches) == 1: include_fixed_path = matches[0] msg = "Found include-fixed subdirectory at %s, " msg += "renaming it to avoid using system header files patched by fixincludes..." self.log.info(msg, include_fixed_path) # limits.h and syslimits.h need to be copied to include/ first, # these are strictly required (by /usr/include/limits.h for example) include_path = os.path.join( os.path.dirname(include_fixed_path), 'include') retained_header_files = ['limits.h', 'syslimits.h'] for fn in retained_header_files: from_path = os.path.join(include_fixed_path, fn) to_path = os.path.join(include_path, fn) if os.path.exists(from_path): if os.path.exists(to_path): raise EasyBuildError( "%s already exists, not overwriting it with %s!", to_path, from_path) else: copy_file(from_path, to_path) self.log.info("%s copied to %s before renaming %s", from_path, to_path, include_fixed_path) else: self.log.warning( "Can't copy non-existing file %s to %s, since it doesn't exist!", from_path, to_path) readme = os.path.join(include_fixed_path, 'README.easybuild') readme_txt = '\n'.join([ "This directory was renamed by EasyBuild to avoid that the header files in it are picked up,", "since they may cause problems when the OS is upgraded to a new (minor) version.", '', "These files were copied to %s first: %s" % (include_path, ', '.join(retained_header_files)), '', "See https://github.com/easybuilders/easybuild-easyconfigs/issues/10666 for more information.", '', ]) write_file(readme, readme_txt) include_fixed_renamed = include_fixed_path + '.renamed-by-easybuild' move_file(include_fixed_path, include_fixed_renamed) self.log.info( "%s renamed to %s to avoid using the header files in it", include_fixed_path, include_fixed_renamed) else: raise EasyBuildError( "Exactly one 'include-fixed' directory expected, found %d: %s", len(matches), matches) else: self.log.info("No include-fixed subdirectory found at %s", glob_pattern)
def test_step(self): """Run GAMESS-US tests (if 'runtest' easyconfig parameter is set to True).""" # don't use provided 'runall' script for tests, since that only runs the tests single-core if self.cfg['runtest']: if not build_option('mpi_tests'): self.log.info( "Skipping testing of GAMESS-US since MPI testing is disabled" ) return try: cwd = os.getcwd() os.chdir(self.testdir) except OSError as err: raise EasyBuildError( "Failed to move to temporary directory for running tests: %s", err) # copy input files for exam<id> standard tests for test_input in glob.glob( os.path.join(self.installdir, 'tests', 'standard', 'exam*.inp')): try: shutil.copy2(test_input, os.getcwd()) except OSError as err: raise EasyBuildError("Failed to copy %s to %s: %s", test_input, os.getcwd(), err) rungms = os.path.join(self.installdir, 'rungms') test_env_vars = ['TMPDIR=%s' % self.testdir] if self.toolchain.mpi_family() == toolchain.INTELMPI: test_env_vars.extend([ 'I_MPI_FALLBACK=enable', # enable fallback in case first fabric fails (see $I_MPI_FABRICS_LIST) 'I_MPI_HYDRA_BOOTSTRAP=fork', # tests are only run locally (2 processes), so no SSH required ]) # run all exam<id> tests, dump output to exam<id>.log n_tests = 47 for i in range(1, n_tests + 1): test_cmd = ' '.join( test_env_vars + [rungms, 'exam%02d' % i, self.version, '1', '2']) (out, _) = run_cmd(test_cmd, log_all=True, simple=False) write_file('exam%02d.log' % i, out) # verify output of tests check_cmd = os.path.join(self.installdir, 'tests', 'standard', 'checktst') (out, _) = run_cmd(check_cmd, log_all=True, simple=False) success_regex = re.compile( "^All %d test results are correct" % n_tests, re.M) if success_regex.search(out): self.log.info("All tests ran successfully!") else: raise EasyBuildError("Not all tests ran successfully...") # cleanup os.chdir(cwd) try: shutil.rmtree(self.testdir) except OSError as err: raise EasyBuildError("Failed to remove test directory %s: %s", self.testdir, err)
def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. :param easyconfigs: list of easyconfigs to generate scripts for """ ecs_and_script_paths = [] for easyconfig in easyconfigs: script_path = '%s.env' % os.path.splitext( os.path.basename(easyconfig['spec']))[0] ecs_and_script_paths.append((easyconfig['ec'], script_path)) # don't just overwrite existing scripts existing_scripts = [ s for (_, s) in ecs_and_script_paths if os.path.exists(s) ] if existing_scripts: if build_option('force'): _log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts)) else: raise EasyBuildError( "Script(s) already exists, not overwriting them (unless --force is used): %s", ' '.join(existing_scripts)) orig_env = copy.deepcopy(os.environ) for ec, script_path in ecs_and_script_paths: # obtain EasyBlock instance app_class = get_easyblock_class(ec['easyblock'], name=ec['name']) app = app_class(ec) # mimic dry run, and keep quiet app.dry_run = app.silent = app.toolchain.dry_run = True # prepare build environment (in dry run mode) app.check_readiness_step() app.prepare_step(start_dir=False) # compose script ecfile = os.path.basename(ec.path) script_lines = [ "#!/bin/bash", "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), "# usage: source %s" % os.path.basename(script_path), ] script_lines.extend(['', "# toolchain & dependency modules"]) if app.toolchain.modules: script_lines.extend( ["module load %s" % mod for mod in app.toolchain.modules]) else: script_lines.append("# (no modules loaded)") script_lines.extend(['', "# build environment"]) if app.toolchain.vars: env_vars = sorted(app.toolchain.vars.items()) script_lines.extend([ "export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars ]) else: script_lines.append("# (no build environment defined)") write_file(script_path, '\n'.join(script_lines)) print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False) restore_env(orig_env)
def test_fetch_files_from_pr_cache(self): """Test caching for fetch_files_from_pr.""" if self.skip_github_tests: print( "Skipping test_fetch_files_from_pr_cache, no GitHub token available?" ) return init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, }) # clear cache first, to make sure we start with a clean slate gh.fetch_files_from_pr.clear_cache() self.assertFalse(gh.fetch_files_from_pr._cache) pr7159_filenames = [ 'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb', 'OpenFOAM-5.0-20180108-foss-2018a.eb', 'OpenFOAM-5.0-20180108-intel-2018a.eb', 'OpenFOAM-6-foss-2018b.eb', 'OpenFOAM-6-intel-2018a.eb', 'OpenFOAM-v1806-foss-2018b.eb', 'PETSc-3.9.3-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018b.eb', 'SCOTCH-6.0.6-intel-2018a.eb', 'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb' ] pr7159_files = gh.fetch_easyconfigs_from_pr( 7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(sorted(pr7159_filenames), sorted(os.path.basename(f) for f in pr7159_files)) # check that cache has been populated for PR 7159 self.assertEqual(len(gh.fetch_files_from_pr._cache.keys()), 1) # github_account value is None (results in using default 'easybuilders') cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix) self.assertTrue(cache_key in gh.fetch_files_from_pr._cache.keys()) cache_entry = gh.fetch_files_from_pr._cache[cache_key] self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]), sorted(pr7159_filenames)) # same query should return result from cache entry res = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(res, pr7159_files) # inject entry in cache and check result of matching query pr_id = 12345 tmpdir = os.path.join(self.test_prefix, 'easyblocks-pr-12345') pr12345_files = [ os.path.join(tmpdir, 'foo.py'), os.path.join(tmpdir, 'bar.py'), ] for fp in pr12345_files: write_file(fp, '') # github_account value is None (results in using default 'easybuilders') cache_key = (pr_id, None, 'easybuild-easyblocks', tmpdir) gh.fetch_files_from_pr.update_cache({cache_key: pr12345_files}) res = gh.fetch_easyblocks_from_pr(12345, tmpdir) self.assertEqual(sorted(pr12345_files), sorted(res))
def test_easybuildlog(self): """Tests for EasyBuildLog.""" fd, tmplog = tempfile.mkstemp() os.close(fd) # compose versions older/newer than current version depr_ver = int(os.environ['EASYBUILD_DEPRECATED']) older_ver = str(depr_ver - 1) newer_ver = str(depr_ver + 1) # set log format, for each regex searching setLogFormat("%(name)s [%(levelname)s] :: %(message)s") # test basic log methods logToFile(tmplog, enable=True) log = getLogger('test_easybuildlog') self.mock_stderr(True) log.setLevelName('DEBUG') log.debug("123 debug") log.info("foobar info") log.warning("justawarning") log.deprecated("anotherwarning", newer_ver) log.deprecated("onemorewarning", '1.0', '2.0') log.deprecated("lastwarning", '1.0', max_ver='2.0') log.deprecated("thisisnotprinted", '1.0', max_ver='2.0', silent=True) log.error("kaput") log.error("err: %s", 'msg: %s') stderr = self.get_stderr() self.mock_stderr(False) more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information" expected_stderr = '\n\n'.join([ "\nWARNING: Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info, "\nWARNING: Deprecated functionality, will no longer work in v2.0: onemorewarning", "\nWARNING: Deprecated functionality, will no longer work in v2.0: lastwarning", ]) + '\n\n' self.assertEqual(stderr, expected_stderr) try: log.exception("oops") except EasyBuildError: pass logToFile(tmplog, enable=False) logtxt = read_file(tmplog) expected_logtxt = '\n'.join([ r"root.test_easybuildlog \[DEBUG\] :: 123 debug", r"root.test_easybuildlog \[INFO\] :: foobar info", r"root.test_easybuildlog \[WARNING\] :: justawarning", r"root.test_easybuildlog \[WARNING\] :: Deprecated functionality.*anotherwarning.*", r"root.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*", r"root.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*", r"root.test_easybuildlog \[WARNING\] :: Deprecated functionality.*thisisnotprinted.*", r"root.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput", r"root.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s", r"root.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops", '', ]) logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt)) self.assertErrorRegex(EasyBuildError, r"DEPRECATED \(since .*: kaput", log.deprecated, "kaput", older_ver) self.assertErrorRegex(EasyBuildError, r"DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0', '1.0') self.assertErrorRegex(EasyBuildError, r"DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0', max_ver='1.0') # wipe log so we can reuse it write_file(tmplog, '') # test formatting log messages by providing extra arguments logToFile(tmplog, enable=True) log.warning("%s", "bleh") log.info("%s+%s = %d", '4', '2', 42) args = ['this', 'is', 'just', 'a', 'test'] log.debug("%s %s %s %s %s", *args) log.error("foo %s baz", 'baz') logToFile(tmplog, enable=False) logtxt = read_file(tmplog) expected_logtxt = '\n'.join([ r"root.test_easybuildlog \[WARNING\] :: bleh", r"root.test_easybuildlog \[INFO\] :: 4\+2 = 42", r"root.test_easybuildlog \[DEBUG\] :: this is just a test", r"root.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz", '', ]) logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt)) write_file(tmplog, '') logToFile(tmplog, enable=True) # also test use of 'more_info' named argument for log.deprecated self.mock_stderr(True) log.deprecated("\nthis is just a test\n", newer_ver, more_info="(see URLGOESHERE for more information)") self.mock_stderr(False) logtxt = read_file(tmplog) expected_logtxt = '\n'.join([ "[WARNING] :: Deprecated functionality, will no longer work in v10000001: ", "this is just a test", "(see URLGOESHERE for more information)", ]) self.assertTrue(logtxt.strip().endswith(expected_logtxt))
def test_generaloption_config(self): """Test new-style configuration (based on generaloption).""" self.purge_environment() # check whether configuration via environment variables works as expected prefix = os.path.join(self.tmpdir, 'testprefix') buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build', 'path') os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var options = init_config(args=[]) self.assertEqual(build_path(), buildpath_env_var) self.assertEqual(install_path(), os.path.join(prefix, 'software')) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_BUILDPATH'] # check whether configuration via command line arguments works prefix = os.path.join(self.tmpdir, 'test1') install = os.path.join(self.tmpdir, 'test2', 'install') repopath = os.path.join(self.tmpdir, 'test2', 'repo') config_file = os.path.join(self.tmpdir, 'nooldconfig.py') write_file(config_file, '') args = [ '--config', config_file, # force empty oldstyle config file '--prefix', prefix, '--installpath', install, '--repositorypath', repopath, ] options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, 'software')) self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) self.assertEqual(options.config, config_file) # check mixed command line/env var configuration prefix = os.path.join(self.tmpdir, 'test3') install = os.path.join(self.tmpdir, 'test4', 'install') subdir_software = 'eb-soft' args = [ '--config', config_file, # force empty oldstyle config file '--installpath', install, ] os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, subdir_software)) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def configure_step(self): """Configure build by patching UFconfig.mk or SuiteSparse_config.mk.""" if LooseVersion(self.version) < LooseVersion('4.0'): self.config_name = 'UFconfig' else: self.config_name = 'SuiteSparse_config' cfgvars = { 'CC': os.getenv('MPICC'), 'CFLAGS': os.getenv('CFLAGS'), 'CXX': os.getenv('MPICXX'), 'F77': os.getenv('MPIF77'), 'F77FLAGS': os.getenv('F77FLAGS'), } # avoid that (system) Intel compilers are always considered self.cfg.update('buildopts', 'AUTOCC=no') # Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt self.cfg.update('buildopts', 'BLAS="%s"' % os.getenv('LIBBLAS_MT')) self.cfg.update('buildopts', 'LAPACK="%s"' % os.getenv('LIBLAPACK_MT')) # Get CUDA and set it up appropriately cuda = get_software_root('CUDA') if cuda: cuda_cc_space_sep = self.cfg.get_cuda_cc_template_value( 'cuda_cc_space_sep').replace('.', '').split() nvcc_gencode = ' '.join([ '-gencode=arch=compute_' + x + ',code=sm_' + x for x in cuda_cc_space_sep ]) cfgvars.update({ 'NVCCFLAGS': ' '.join(['-Xcompiler', '-fPIC', '-O3', nvcc_gencode]), }) # Get METIS or ParMETIS settings metis = get_software_root('METIS') parmetis = get_software_root('ParMETIS') if parmetis: metis_path = parmetis metis_include = os.path.join(parmetis, 'include') metis_libs = os.path.join(parmetis, get_software_libdir('ParMETIS'), 'libmetis.a') elif metis: metis_path = metis metis_include = os.path.join(metis, 'include') metis_libs = os.path.join(metis, get_software_libdir('METIS'), 'libmetis.a') else: raise EasyBuildError("Neither METIS or ParMETIS module loaded.") if LooseVersion(self.version) >= LooseVersion('4.5.1'): cfgvars.update({ 'MY_METIS_LIB': metis_libs, 'MY_METIS_INC': metis_include, }) else: cfgvars.update({ 'METIS_PATH': metis_path, 'METIS': metis_libs, }) # patch file fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name) try: for line in fileinput.input(fp, inplace=1, backup='.orig'): for (var, val) in list(cfgvars.items()): # Let's overwrite NVCCFLAGS at the end, since the line breaks and the fact that it appears multiple # times makes it tricky to handle it properly if var != 'NVCCFLAGS': orig_line = line # for variables in cfgvars, substiture lines assignment # in the file, whatever they are, by assignments to the # values in cfgvars line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var, r"\1 %s # patched by EasyBuild\n" % val, line) if line != orig_line: cfgvars.pop(var) sys.stdout.write(line) except IOError as err: raise EasyBuildError("Failed to patch %s in: %s", fp, err) # add remaining entries at the end if cfgvars: cfgtxt = '# lines below added automatically by EasyBuild\n' cfgtxt += '\n'.join( ["%s = %s" % (var, val) for (var, val) in cfgvars.items()]) write_file(fp, cfgtxt, append=True)
def mpi_cmd_for(self, cmd, nr_ranks): """Construct an MPI command for the given command and number of ranks.""" # parameter values for mpirun command params = { 'nr_ranks': nr_ranks, 'cmd': cmd, } # different known mpirun commands mpirun_n_cmd = "mpirun -n %(nr_ranks)d %(cmd)s" mpi_cmds = { toolchain.OPENMPI: mpirun_n_cmd, # @UndefinedVariable toolchain.QLOGICMPI: "mpirun -H localhost -np %(nr_ranks)d %(cmd)s", # @UndefinedVariable toolchain.INTELMPI: "mpirun %(mpdbf)s %(nodesfile)s -np %(nr_ranks)d %(cmd)s", # @UndefinedVariable toolchain.MVAPICH2: mpirun_n_cmd, # @UndefinedVariable toolchain.MPICH: mpirun_n_cmd, # @UndefinedVariable toolchain.MPICH2: mpirun_n_cmd, # @UndefinedVariable } mpi_family = self.mpi_family() # Intel MPI mpirun needs more work if mpi_family == toolchain.INTELMPI: # @UndefinedVariable # set temporary dir for mdp # note: this needs to be kept *short*, to avoid mpirun failing with "socket.error: AF_UNIX path too long" # exact limit is unknown, but ~20 characters seems to be OK env.setvar('I_MPI_MPD_TMPDIR', tempfile.gettempdir()) mpd_tmpdir = os.environ['I_MPI_MPD_TMPDIR'] if len(mpd_tmpdir) > 20: self.log.warning("$I_MPI_MPD_TMPDIR should be (very) short to avoid problems: %s" % mpd_tmpdir) # temporary location for mpdboot and nodes files tmpdir = tempfile.mkdtemp(prefix='mpi_cmd_for-') # set PBS_ENVIRONMENT, so that --file option for mpdboot isn't stripped away env.setvar('PBS_ENVIRONMENT', "PBS_BATCH_MPI") # make sure we're always using mpd as process manager # only required for/picked up by Intel MPI v4.1 or higher, no harm done for others env.setvar('I_MPI_PROCESS_MANAGER', 'mpd') # create mpdboot file fn = os.path.join(tmpdir, 'mpdboot') try: if os.path.exists(fn): os.remove(fn) write_file(fn, "localhost ifhn=localhost") except OSError, err: raise EasyBuildError("Failed to create file %s: %s", fn, err) params.update({'mpdbf': "--file=%s" % fn}) # create nodes file fn = os.path.join(tmpdir, 'nodes') try: if os.path.exists(fn): os.remove(fn) write_file(fn, "localhost\n" * nr_ranks) except OSError, err: raise EasyBuildError("Failed to create file %s: %s", fn, err)
def test_generaloption_config_file(self): """Test use of new-style configuration file.""" self.purge_environment() oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py') config_file = os.path.join(self.tmpdir, 'testconfig.cfg') testpath1 = os.path.join(self.tmpdir, 'test1') testpath2 = os.path.join(self.tmpdir, 'testtwo') write_file(oldstyle_config_file, '') # test with config file passed via command line cfgtxt = '\n'.join([ '[config]', 'installpath = %s' % testpath2, ]) write_file(config_file, cfgtxt) args = [ '--configfiles', config_file, '--debug', '--buildpath', testpath1, ] options = init_config(args=args) self.assertEqual(build_path(), testpath1) # via command line self.assertEqual(source_paths(), [ os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources') ]) # default self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file # test with config file passed via environment variable cfgtxt = '\n'.join([ '[config]', 'buildpath = %s' % testpath1, ]) write_file(config_file, cfgtxt) os.environ['EASYBUILD_CONFIGFILES'] = config_file args = [ '--debug', '--sourcepath', testpath2, ] options = init_config(args=args) self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file testpath3 = os.path.join(self.tmpdir, 'testTHREE') os.environ['EASYBUILD_SOURCEPATH'] = testpath2 args = [ '--debug', '--installpath', testpath3, ] options = init_config(args=args) self.assertEqual( source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line self.assertEqual(build_path(), testpath1) # via config file del os.environ['EASYBUILD_CONFIGFILES']
class EasyConfigTest(TestCase): """Baseclass for easyconfig testcases.""" # initialize configuration (required for e.g. default modules_tool setting) eb_go = eboptions.parse_options() config.init(eb_go.options, eb_go.get_options_by_section('config')) build_options = { 'check_osdeps': False, 'external_modules_metadata': {}, 'force': True, 'optarch': 'test', 'robot_path': get_paths_for("easyconfigs")[0], 'silent': True, 'suffix_modules_path': GENERAL_CLASS, 'valid_module_classes': config.module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } config.init_build_options(build_options=build_options) set_tmpdir() del eb_go # put dummy 'craype-test' module in place, which is required for parsing easyconfigs using Cray* toolchains TMPDIR = tempfile.mkdtemp() os.environ['MODULEPATH'] = TMPDIR write_file(os.path.join(TMPDIR, 'craype-test'), '#%Module\n') log = fancylogger.getLogger("EasyConfigTest", fname=False) # make sure a logger is present for main main._log = log ordered_specs = None parsed_easyconfigs = [] def process_all_easyconfigs(self): """Process all easyconfigs and resolve inter-easyconfig dependencies.""" # all available easyconfig files easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path) # parse all easyconfigs if they haven't been already if not self.parsed_easyconfigs: for spec in specs: self.parsed_easyconfigs.extend(process_easyconfig(spec)) # filter out external modules for ec in self.parsed_easyconfigs: for dep in ec['dependencies'][:]: if dep.get('external_module', False): ec['dependencies'].remove(dep) self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs, modules_tool(), retain_all_deps=True) def test_dep_graph(self): """Unit test that builds a full dependency graph.""" # pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6 if LooseVersion( sys.version) >= LooseVersion('2.6') and single_tests_ok: # temporary file for dep graph (hn, fn) = tempfile.mkstemp(suffix='.dot') os.close(hn) if self.ordered_specs is None: self.process_all_easyconfigs() dep_graph(fn, self.ordered_specs) try: os.remove(fn) except OSError, err: log.error("Failed to remove %s: %s" % (fn, err)) else:
def test_legacy_config_file(self): """Test finding/using legacy configuration files.""" self.purge_environment() cfg_fn = self.configure(args=[]) self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py')) configtxt = """ build_path = '%(buildpath)s' source_path = '%(sourcepath)s' install_path = '%(installpath)s' repository_path = '%(repopath)s' repository = FileRepository(repository_path) log_format = ('%(logdir)s', '%(logtmpl)s') log_dir = '%(tmplogdir)s' software_install_suffix = '%(softsuffix)s' modules_install_suffix = '%(modsuffix)s' """ buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path') sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path') installpath = os.path.join(self.tmpdir, 'my', 'test', 'install', 'path') repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path') logdir = 'somedir' logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log' tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir') softsuffix = 'myfavoritesoftware' modsuffix = 'modulesgohere' configdict = { 'buildpath': buildpath, 'sourcepath': sourcepath, 'installpath': installpath, 'repopath': repopath, 'logdir': logdir, 'logtmpl': logtmpl, 'tmplogdir': tmplogdir, 'softsuffix': softsuffix, 'modsuffix': modsuffix } # create user config file on default location myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py') if not os.path.exists(os.path.dirname(myconfigfile)): os.makedirs(os.path.dirname(myconfigfile)) write_file(myconfigfile, configtxt % configdict) # redefine home so we can test user config file on default location home = os.environ.get('HOME', None) os.environ['HOME'] = self.tmpdir init_config() cfg_fn = self.configure(args=[]) if home is not None: os.environ['HOME'] = home # check finding and use of config file self.assertEqual(cfg_fn, myconfigfile) self.assertEqual(build_path(), buildpath) self.assertEqual(source_paths()[0], sourcepath) self.assertEqual(install_path(), os.path.join(installpath, softsuffix)) self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix)) repo = init_repository(get_repository(), get_repositorypath()) self.assertTrue(isinstance(repo, FileRepository)) self.assertEqual(repo.repo, repopath) self.assertEqual(log_file_format(return_directory=True), logdir) self.assertEqual(log_file_format(), logtmpl) self.assertEqual(get_build_log_path(), tmplogdir) # redefine config file entries for proper testing below buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build', 'path') sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'source', 'path') installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'install', 'path') repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo', 'path') logdir = 'somedir_custom' logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log' tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'tmplogdir') softsuffix = 'myfavoritesoftware_custom' modsuffix = 'modulesgohere_custom' configdict = { 'buildpath': buildpath, 'sourcepath': sourcepath, 'installpath': installpath, 'repopath': repopath, 'logdir': logdir, 'logtmpl': logtmpl, 'tmplogdir': tmplogdir, 'softsuffix': softsuffix, 'modsuffix': modsuffix } # create custom config file, and point to it mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py') if not os.path.exists(os.path.dirname(mycustomconfigfile)): os.makedirs(os.path.dirname(mycustomconfigfile)) write_file(mycustomconfigfile, configtxt % configdict) os.environ['EASYBUILDCONFIG'] = mycustomconfigfile # reconfigure init_config() cfg_fn = self.configure(args=[]) # verify configuration self.assertEqual(cfg_fn, mycustomconfigfile) self.assertEqual(build_path(), buildpath) self.assertEqual(source_paths()[0], sourcepath) self.assertEqual(install_path(), os.path.join(installpath, softsuffix)) self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix)) repo = init_repository(get_repository(), get_repositorypath()) self.assertTrue(isinstance(repo, FileRepository)) self.assertEqual(repo.repo, repopath) self.assertEqual(log_file_format(return_directory=True), logdir) self.assertEqual(log_file_format(), logtmpl) self.assertEqual(get_build_log_path(), tmplogdir)