def configure_step(self):
"""Configure MATLAB installation: create license file."""
# create license file
licserv = self.cfg['license_server']
licport = self.cfg['license_server_port']
lictxt = '\n'.join([
"SERVER %s 000000000000 %s" % (licserv, licport),
"USE_SERVER",
])
licfile = os.path.join(self.builddir, 'matlab.lic')
write_file(licfile, lictxt)
try:
shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile)
config = read_file(self.configfile)
regdest = re.compile(r"^# destinationFolder=.*", re.M)
regkey = re.compile(r"^# fileInstallationKey=.*", re.M)
regagree = re.compile(r"^# agreeToLicense=.*", re.M)
regmode = re.compile(r"^# mode=.*", re.M)
reglicpath = re.compile(r"^# licensePath=.*", re.M)
config = regdest.sub("destinationFolder=%s" % self.installdir, config)
key = self.cfg['key']
config = regkey.sub("fileInstallationKey=%s" % key, config)
config = regagree.sub("agreeToLicense=Yes", config)
config = regmode.sub("mode=silent", config)
config = reglicpath.sub("licensePath=%s" % licfile, config)
write_file(self.configfile, config)
except IOError, err:
raise EasyBuildError("Failed to create installation config file %s: %s", self.configfile, err)
def test_toy_build(self):
"""Perform a toy build."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
# adjust PYTHONPATH such that test easyblocks are found
orig_sys_path = sys.path[:]
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'sandbox')))
import easybuild.easyblocks
reload(easybuild.easyblocks)
reload(easybuild.tools.module_naming_scheme)
# clear log
write_file(self.logfile, '')
buildpath = tempfile.mkdtemp()
installpath = tempfile.mkdtemp()
sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources')
args = [
os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'),
'--sourcepath=%s' % sourcepath,
'--buildpath=%s' % buildpath,
'--installpath=%s' % installpath,
'--debug',
'--unittest-file=%s' % self.logfile,
'--force',
]
try:
main((args, dummylogfn, True))
except (SystemExit, Exception), err:
print err
def extensions_step(self):
"""Build & Install both Python and R extension"""
# we start with the python bindings
self.py_ext.src = os.path.join(self.mxnet_src_dir, "python")
change_dir(self.py_ext.src)
self.py_ext.prerun()
self.py_ext.run(unpack_src=False)
self.py_ext.postrun()
# next up, the R bindings
self.r_ext.src = os.path.join(self.mxnet_src_dir, "R-package")
change_dir(self.r_ext.src)
mkdir("inst")
symlink(os.path.join(self.installdir, "lib"), os.path.join("inst", "libs"))
symlink(os.path.join(self.installdir, "include"), os.path.join("inst", "include"))
# MXNet doesn't provide a list of its R dependencies by default
write_file("NAMESPACE", R_NAMESPACE)
change_dir(self.mxnet_src_dir)
self.r_ext.prerun()
# MXNet is just weird. To install the R extension, we have to:
# - First install the extension like it is
# - Let R export the extension again. By doing this, all the dependencies get
# correctly filled and some mappings are done
# - Reinstal the exported version
self.r_ext.run()
run_cmd("R_LIBS=%s Rscript -e \"require(mxnet); mxnet:::mxnet.export(\\\"R-package\\\")\"" % self.installdir)
change_dir(self.r_ext.src)
self.r_ext.run()
self.r_ext.postrun()
def configure_step(self):
"""Custom configuration procedure for pplacer."""
# install dir has to be non-existing when we start (it may be there from a previous (failed) install
try:
if os.path.exists(self.installdir):
shutil.rmtree(self.installdir)
self.log.warning("Existing install directory %s removed", self.installdir)
except OSError as err:
raise EasyBuildError("Failed to remove %s: %s", self.installdir, err)
# configure OPAM to install pplacer dependencies
env.setvar('OPAMROOT', self.installdir)
opam_init_cmd = mk_opam_init_cmd()
run_cmd(opam_init_cmd)
run_cmd("opam repo add pplacer-deps %s/pplacer-opam-repository*/" % self.builddir)
run_cmd("opam update pplacer-deps")
env.setvar('OCAML_BACKEND', 'gcc')
run_cmd("eval `opam config env` && cat opam-requirements.txt | xargs -t opam install -y")
txt = "let version = \"v%s\"\n" % self.version
write_file(os.path.join(self.builddir, 'pplacer-%s' % self.version, 'common_src', 'version.ml'), txt)
def test_tweak_one_version(self):
"""Test tweak_one function"""
test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')
# test tweaking of software version (--try-software-version)
tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
# checksums should be reset to empty list, only version should be changed, nothing else
self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]:
val = toy_ec_parsed[key]
self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key)
tweaked_val = tweaked_toy_ec_parsed.get(key)
self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val))
# check behaviour if target file already exists
error_pattern = "File exists, not overwriting it without --force"
self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
# existing file does get overwritten when --force is used
init_config(build_options={'force': True, 'silent': True})
write_file(tweaked_toy_ec, '')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
def test_include_mns(self):
"""Test include_module_naming_schemes()."""
testdir = os.path.dirname(os.path.abspath(__file__))
test_mns = os.path.join(testdir, 'sandbox', 'easybuild', 'module_naming_scheme')
my_mns = os.path.join(self.test_prefix, 'my_mns')
mkdir(my_mns)
# include __init__.py file that should be ignored, and shouldn't cause trouble (bug #1697)
write_file(os.path.join(my_mns, '__init__.py'), "# dummy init, should not get included")
my_mns_txt = '\n'.join([
"from easybuild.tools.module_naming_scheme import ModuleNamingScheme",
"class MyMNS(ModuleNamingScheme):",
" pass",
])
write_file(os.path.join(my_mns, 'my_mns.py'), my_mns_txt)
# include custom MNS
included_mns_path = include_module_naming_schemes(self.test_prefix, [os.path.join(my_mns, '*.py')])
expected_paths = ['__init__.py', 'tools/__init__.py', 'tools/module_naming_scheme/__init__.py',
'tools/module_naming_scheme/my_mns.py']
for filepath in expected_paths:
fullpath = os.path.join(included_mns_path, 'easybuild', filepath)
self.assertTrue(os.path.exists(fullpath), "%s exists" % fullpath)
# path to included MNSs should be prepended to Python search path
self.assertEqual(sys.path[0], included_mns_path)
# importing custom MNS should work
import easybuild.tools.module_naming_scheme.my_mns
my_mns_pyc_path = easybuild.tools.module_naming_scheme.my_mns.__file__
my_mns_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_mns_pyc_path), 'my_mns.py'))
self.assertTrue(os.path.samefile(up(my_mns_real_py_path, 1), my_mns))
def test_download_repo(self):
"""Test download_repo function."""
if self.github_token is None:
print "Skipping test_download_repo, no GitHub token available?"
return
# default: download tarball for master branch of hpcugent/easybuild-easyconfigs repo
path = gh.download_repo(path=self.test_prefix)
repodir = os.path.join(self.test_prefix, 'hpcugent', 'easybuild-easyconfigs-master')
self.assertTrue(os.path.samefile(path, repodir))
self.assertTrue(os.path.exists(repodir))
shafile = os.path.join(repodir, 'latest-sha')
self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile)))
self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2015a.eb')))
# existing downloaded repo is not reperformed, except if SHA is different
account, repo, branch = 'boegel', 'easybuild-easyblocks', 'develop'
repodir = os.path.join(self.test_prefix, account, '%s-%s' % (repo, branch))
latest_sha = gh.fetch_latest_commit_sha(repo, account, branch=branch)
# put 'latest-sha' fail in place, check whether repo was (re)downloaded (should not)
shafile = os.path.join(repodir, 'latest-sha')
write_file(shafile, latest_sha)
path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix)
self.assertTrue(os.path.samefile(path, repodir))
self.assertEqual(os.listdir(repodir), ['latest-sha'])
# remove 'latest-sha' file and verify that download was performed
os.remove(shafile)
path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix)
self.assertTrue(os.path.samefile(path, repodir))
self.assertTrue('easybuild' in os.listdir(repodir))
self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile)))
self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')))
def configure_step(self):
"""Configure MCR installation: create license file."""
configfile = os.path.join(self.builddir, self.configfilename)
if LooseVersion(self.version) < LooseVersion('R2015a'):
shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), configfile)
config = read_file(configfile)
# compile regex first since re.sub doesn't accept re.M flag for multiline regex in Python 2.6
regdest = re.compile(r"^# destinationFolder=.*", re.M)
regagree = re.compile(r"^# agreeToLicense=.*", re.M)
regmode = re.compile(r"^# mode=.*", re.M)
config = regdest.sub("destinationFolder=%s" % self.installdir, config)
config = regagree.sub("agreeToLicense=Yes", config)
config = regmode.sub("mode=silent", config)
else:
config = '\n'.join([
"destinationFolder=%s" % self.installdir,
"agreeToLicense=Yes",
"mode=silent",
])
write_file(configfile, config)
self.log.debug("configuration file written to %s:\n %s", configfile, config)
def test_include_easyblocks_priority(self):
"""Test whether easyblocks included via include_easyblocks() get prioroity over others."""
test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks')
# make sure that test 'foo' easyblocks is there
import easybuild.easyblocks.foo
foo_path = os.path.dirname(os.path.dirname(easybuild.easyblocks.foo.__file__))
self.assertTrue(os.path.samefile(foo_path, test_easyblocks))
# inject custom 'foo' easyblocks
myeasyblocks = os.path.join(self.test_prefix, 'myeasyblocks')
mkdir(myeasyblocks)
# include __init__.py file that should be ignored, and shouldn't cause trouble (bug #1697)
write_file(os.path.join(myeasyblocks, '__init__.py'), "# dummy init, should not get included")
# 'undo' import of foo easyblock
del sys.modules['easybuild.easyblocks.foo']
foo_easyblock_txt = '\n'.join([
"from easybuild.framework.easyblock import EasyBlock",
"class EB_Foo(EasyBlock):",
" pass",
])
write_file(os.path.join(myeasyblocks, 'foo.py'), foo_easyblock_txt)
include_easyblocks(self.test_prefix, [os.path.join(myeasyblocks, 'foo.py')])
foo_pyc_path = easybuild.easyblocks.foo.__file__
foo_real_py_path = os.path.realpath(os.path.join(os.path.dirname(foo_pyc_path), 'foo.py'))
self.assertFalse(os.path.samefile(os.path.dirname(foo_pyc_path), test_easyblocks))
self.assertTrue(os.path.samefile(foo_real_py_path, os.path.join(myeasyblocks, 'foo.py')))
# 'undo' import of foo easyblock
del sys.modules['easybuild.easyblocks.foo']
def test_robot_archived_easyconfigs(self):
"""Test whether robot can pick up archived easyconfigs when asked."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-ictce-4.1.13.eb')
gzip_ectxt = read_file(gzip_ec)
test_ec = os.path.join(self.test_prefix, 'test.eb')
tc_spec = "toolchain = {'name': 'ictce', 'version': '3.2.2.u3'}"
regex = re.compile("^toolchain = .*", re.M)
test_ectxt = regex.sub(tc_spec, gzip_ectxt)
write_file(test_ec, test_ectxt)
ecs, _ = parse_easyconfigs([(test_ec, False)])
self.assertErrorRegex(EasyBuildError, "Irresolvable dependencies encountered", resolve_dependencies,
ecs, self.modtool, retain_all_deps=True)
# --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs
init_config(build_options={
'consider_archived_easyconfigs': True,
'robot_path': [test_ecs],
})
res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True)
self.assertEqual([ec['full_mod_name'] for ec in res], ['ictce/3.2.2.u3', 'gzip/1.5-ictce-3.2.2.u3'])
expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb')
self.assertTrue(os.path.samefile(res[0]['spec'], expected))
def test_checksums(self):
"""Test checksum functionality."""
fh, fp = tempfile.mkstemp()
os.close(fh)
ft.write_file(fp, "easybuild\n")
known_checksums = {
'adler32': '0x379257805',
'crc32': '0x1457143216',
'md5': '7167b64b1ca062b9674ffef46f9325db',
'sha1': 'db05b79e09a4cc67e9dd30b313b5488813db3190',
}
# make sure checksums computation/verification is correct
for checksum_type, checksum in known_checksums.items():
self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum)
self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum)))
# md5 is default
self.assertEqual(ft.compute_checksum(fp), known_checksums['md5'])
self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
# make sure faulty checksums are reported
broken_checksums = dict([(typ, val + 'foo') for (typ, val) in known_checksums.items()])
for checksum_type, checksum in broken_checksums.items():
self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum)
self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum)))
# md5 is default
self.assertFalse(ft.compute_checksum(fp) == broken_checksums['md5'])
self.assertFalse(ft.verify_checksum(fp, broken_checksums['md5']))
# cleanup
os.remove(fp)
def test_job(self):
"""Test submitting build as a job."""
# use gzip-1.4.eb easyconfig file that comes with the tests
eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4.eb')
# check log message with --job
for job_args in [ # options passed are reordered, so order here matters to make tests pass
['--debug'],
['--debug', '--stop=configure', '--try-software-name=foo'],
]:
# clear log file
write_file(self.logfile, '')
args = [
eb_file,
'--job',
] + job_args
outtxt = self.eb_main(args)
job_msg = "INFO.* Command template for jobs: .* && eb %%\(spec\)s.* %s.*\n" % ' .*'.join(job_args)
assertmsg = "Info log message with job command template when using --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt)
self.assertTrue(re.search(job_msg, outtxt), assertmsg)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
def generate_recipe(self):
"""
This method will make use of resolve_template and resolve_template_data methods
in order to generate the container recipe.
"""
template = self.resolve_template()
data = self.resolve_template_data()
if self.img_name:
file_label = os.path.splitext(self.img_name)[0]
else:
file_label = data['mod_names'].split(' ')[0].replace('/', '-')
recipe_path = os.path.join(self.container_path, "%s.%s" % (self.RECIPE_FILE_NAME, file_label))
if os.path.exists(recipe_path):
if build_option('force'):
print_msg("WARNING: overwriting existing container recipe at %s due to --force" % recipe_path)
else:
raise EasyBuildError("Container recipe at %s already exists, not overwriting it without --force",
recipe_path)
recipe_content = template % data
write_file(recipe_path, recipe_content)
print_msg("%s definition file created at %s" % (self.RECIPE_FILE_NAME, recipe_path), log=self.log)
return recipe_path
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--config', config_file, # force empty oldstyle config file
'--prefix', prefix,
'--installpath', install,
'--repositorypath', repopath,
]
options = self.configure_options(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'software'))
self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertEqual(options.config, config_file)
def test_force(self):
"""Test forcing installation even if the module is already available."""
# use GCC-4.6.3.eb easyconfig file that comes with the tests
eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'GCC-4.6.3.eb')
# check log message without --force
args = [
eb_file,
'--debug',
]
outtxt, error_thrown = self.eb_main(args, return_error=True)
self.assertTrue(not error_thrown, "No error is thrown if software is already installed (error_thrown: %s)" % error_thrown)
already_msg = "GCC/4.6.3 is already installed"
self.assertTrue(re.search(already_msg, outtxt), "Already installed message without --force, outtxt: %s" % outtxt)
# clear log file, clean up environment
write_file(self.logfile, '')
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
# check that --force works
args = [
eb_file,
'--force',
'--debug',
]
outtxt = self.eb_main(args)
self.assertTrue(not re.search(already_msg, outtxt), "Already installed message not there with --force")
def setup_sandbox_for_intel_fftw(self, moddir, imklver='10.3.12.361'):
"""Set up sandbox for Intel FFTW"""
# hack to make Intel FFTW lib check pass
# create dummy imkl module and put required lib*.a files in place
imkl_module_path = os.path.join(moddir, 'imkl', imklver)
imkl_dir = os.path.join(self.test_prefix, 'software', 'imkl', imklver)
imkl_mod_txt = '\n'.join([
"#%Module",
"setenv EBROOTIMKL %s" % imkl_dir,
"setenv EBVERSIONIMKL %s" % imklver,
])
write_file(imkl_module_path, imkl_mod_txt)
fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64']
fftw_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64']
if LooseVersion(imklver) >= LooseVersion('11'):
fftw_libs.extend(['fftw3x_cdft_ilp64', 'fftw3x_cdft_lp64'])
else:
fftw_libs.append('fftw3x_cdft')
for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']:
os.makedirs(os.path.join(imkl_dir, subdir))
for fftlib in fftw_libs:
write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo')
def add_easyconfig(self, cfg, name, version, stats, previous):
"""
Add the eb-file for software name and version to the repository.
stats should be a dict containing statistics.
if previous is true -> append the statistics to the file
This will return the path to the created file (for use in subclasses)
"""
# create directory for eb file
full_path = os.path.join(self.wc, self.subdir, name)
mkdir(full_path, parents=True)
# destination
dest = os.path.join(full_path, "%s-%s.eb" % (name, version))
txt = "# Built with EasyBuild version %s on %s\n" % (VERBOSE_VERSION, time.strftime("%Y-%m-%d_%H-%M-%S"))
# copy file
txt += read_file(cfg)
# append a line to the eb file so that we don't have git merge conflicts
if not previous:
statsprefix = "\n# Build statistics\nbuildstats = ["
statssuffix = "]\n"
else:
# statstemplate = "\nbuildstats.append(%s)\n"
statsprefix = "\nbuildstats.append("
statssuffix = ")\n"
txt += statsprefix + stats_to_str(stats) + statssuffix
write_file(dest, txt)
return dest
def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state):
"""
Upload/dump overall test report
@param ecs_with_res: processed easyconfigs with build result (success/failure)
@param orig_cnt: number of original easyconfig paths
@param success: boolean indicating whether all builds were successful
@param msg: message to be included in test report
@param init_session_state: initial session state info to include in test report
"""
dump_path = build_option('dump_test_report')
pr_nr = build_option('from_pr')
upload = build_option('upload_test_report')
if upload:
msg = msg + " (%d easyconfigs in this PR)" % orig_cnt
test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True)
if pr_nr:
# upload test report to gist and issue a comment in the PR to notify
txt = post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, success)
else:
# only upload test report as a gist
gist_url = upload_test_report_as_gist(test_report)
txt = "Test report uploaded to %s" % gist_url
else:
test_report = create_test_report(msg, ecs_with_res, init_session_state)
txt = None
_log.debug("Test report: %s" % test_report)
if dump_path is not None:
write_file(dump_path, test_report)
_log.info("Test report dumped to %s" % dump_path)
return txt
def test_toy_download_sources(self):
"""Test toy build with sources that still need to be 'downloaded'."""
tmpdir = tempfile.mkdtemp()
# copy toy easyconfig file, and append source_urls to it
shutil.copy2(os.path.join(os.path.dirname(__file__), "easyconfigs", "toy-0.0.eb"), tmpdir)
source_url = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sandbox", "sources", "toy")
ec_file = os.path.join(tmpdir, "toy-0.0.eb")
write_file(ec_file, '\nsource_urls = ["file://%s"]\n' % source_url, append=True)
# unset $EASYBUILD_XPATH env vars, to make sure --prefix is picked up
for cfg_opt in ["build", "install", "source"]:
del os.environ["EASYBUILD_%sPATH" % cfg_opt.upper()]
sourcepath = os.path.join(tmpdir, "mysources")
args = [
ec_file,
"--prefix=%s" % tmpdir,
"--sourcepath=%s" % ":".join([sourcepath, "/bar"]), # include senseless path which should be ignored
"--debug",
"--unittest-file=%s" % self.logfile,
"--force",
]
outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True)
self.check_toy(tmpdir, outtxt)
self.assertTrue(os.path.exists(os.path.join(sourcepath, "t", "toy", "toy-0.0.tar.gz")))
shutil.rmtree(tmpdir)
def process_easyconfig_file(ec_file):
"""Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested)."""
ectxt = read_file(ec_file)
name, easyblock = fetch_parameters_from_easyconfig(ectxt, ['name', 'easyblock'])
derived_easyblock_class = get_easyblock_class(easyblock, name=name, default_fallback=False)
fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class)
if ectxt != fixed_ectxt:
if go.options.backup:
try:
backup_ec_file = '%s.bk' % ec_file
i = 1
while os.path.exists(backup_ec_file):
backup_ec_file = '%s.bk%d' % (ec_file, i)
i += 1
os.rename(ec_file, backup_ec_file)
log.info("Backed up %s to %s" % (ec_file, backup_ec_file))
except OSError, err:
raise EasyBuildError("Failed to backup %s before rewriting it: %s", ec_file, err)
write_file(ec_file, fixed_ectxt)
log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt)
log.info("%s: fixed" % ec_file)
def test_apply_regex_substitutions(self):
"""Test apply_regex_substitutions function."""
testfile = os.path.join(self.test_prefix, 'test.txt')
testtxt = '\n'.join([
"CC = gcc",
"CFLAGS = -O3 -g",
"FC = gfortran",
"FFLAGS = -O3 -g -ffixed-form",
])
ft.write_file(testfile, testtxt)
regex_subs = [
(r"^(CC)\s*=\s*.*$", r"\1 = ${CC}"),
(r"^(FC\s*=\s*).*$", r"\1${FC}"),
(r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"),
]
ft.apply_regex_substitutions(testfile, regex_subs)
expected_testtxt = '\n'.join([
"CC = ${CC}",
"CFLAGS = -O2",
"FC = ${FC}",
"FFLAGS = -O2 -ffixed-form",
])
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, expected_testtxt)
def test_toy_download_sources(self):
"""Test toy build with sources that still need to be 'downloaded'."""
tmpdir = tempfile.mkdtemp()
# copy toy easyconfig file, and append source_urls to it
shutil.copy2(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'), tmpdir)
source_url = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sandbox', 'sources', 'toy')
ec_file = os.path.join(tmpdir, 'toy-0.0.eb')
write_file(ec_file, '\nsource_urls = ["file://%s"]\n' % source_url, append=True)
# unset $EASYBUILD_XPATH env vars, to make sure --prefix is picked up
for cfg_opt in ['build', 'install', 'source']:
del os.environ['EASYBUILD_%sPATH' % cfg_opt.upper()]
sourcepath = os.path.join(tmpdir, 'mysources')
args = [
ec_file,
'--prefix=%s' % tmpdir,
'--sourcepath=%s' % ':'.join([sourcepath, '/bar']), # include senseless path which should be ignored
'--debug',
'--unittest-file=%s' % self.logfile,
'--force',
]
outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True)
self.check_toy(tmpdir, outtxt)
self.assertTrue(os.path.exists(os.path.join(sourcepath, 't', 'toy', 'toy-0.0.tar.gz')))
shutil.rmtree(tmpdir)
def test_toy_broken(self):
"""Test deliberately broken toy build."""
tmpdir = tempfile.mkdtemp()
broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb")
toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb')
broken_toy_ec_txt = read_file(toy_ec_file)
broken_toy_ec_txt += "checksums = ['clearywrongchecksum']"
write_file(broken_toy_ec, broken_toy_ec_txt)
error_regex = "Checksum verification .* failed"
self.assertErrorRegex(EasyBuildError, error_regex, self.test_toy_build, ec_file=broken_toy_ec, tmpdir=tmpdir,
verify=False, fails=True, verbose=False, raise_error=True)
# make sure log file is retained, also for failed build
log_path_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*.log')
self.assertTrue(len(glob.glob(log_path_pattern)) == 1, "Log file found at %s" % log_path_pattern)
# make sure individual test report is retained, also for failed build
test_report_fp_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*test_report.md')
self.assertTrue(len(glob.glob(test_report_fp_pattern)) == 1, "Test report %s found" % test_report_fp_pattern)
# test dumping full test report (doesn't raise an exception)
test_report_fp = os.path.join(self.test_buildpath, 'full_test_report.md')
self.test_toy_build(ec_file=broken_toy_ec, tmpdir=tmpdir, verify=False, fails=True, verbose=False,
raise_error=True, test_report=test_report_fp)
# cleanup
shutil.rmtree(tmpdir)
def test_list_easyblocks(self):
"""Test listing easyblock hierarchy."""
# adjust PYTHONPATH such that test easyblocks are found
orig_sys_path = sys.path
import easybuild
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'easyblocks_sandbox')))
easybuild = reload(easybuild)
import easybuild.easyblocks
easybuild.easyblocks = reload(easybuild.easyblocks)
# simple view
for list_arg in ['--list-easyblocks', '--list-easyblocks=simple']:
# clear log
write_file(self.logfile, '')
args = [
list_arg,
'--unittest-file=%s' % self.logfile,
]
try:
main((args, None))
except (SystemExit, Exception), err:
pass
outtxt = read_file(self.logfile)
for pat in [
r"EasyBlock\n",
r"|--\s+EB_foo\n|\s+|--\s+EB_foofoo\n",
r"|--\s+bar\n",
]:
self.assertTrue(re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt))
def test_external_modules_metadata(self):
"""Test --external-modules-metadata."""
# empty list by default
cfg = init_config()
self.assertEqual(cfg.external_modules_metadata, [])
testcfgtxt = EXTERNAL_MODULES_METADATA
testcfg = os.path.join(self.test_prefix, 'test_external_modules_metadata.cfg')
write_file(testcfg, testcfgtxt)
cfg = init_config(args=['--external-modules-metadata=%s' % testcfg])
netcdf = {
'name': ['netCDF', 'netCDF-Fortran'],
'version': ['4.3.2', '4.3.2'],
'prefix': 'NETCDF_DIR',
}
self.assertEqual(cfg.external_modules_metadata['cray-netcdf/4.3.2'], netcdf)
hdf5 = {
'name': ['HDF5'],
'version': ['1.8.13'],
'prefix': 'HDF5_DIR',
}
self.assertEqual(cfg.external_modules_metadata['cray-hdf5/1.8.13'], hdf5)
# impartial metadata is fine
self.assertEqual(cfg.external_modules_metadata['foo'], {'name': ['Foo'], 'prefix': '/foo'})
self.assertEqual(cfg.external_modules_metadata['bar/1.2.3'], {'name': ['bar'], 'version': ['1.2.3']})
# if both names and versions are specified, lists must have same lengths
write_file(testcfg, '\n'.join(['[foo/1.2.3]', 'name = foo,bar', 'version = 1.2.3']))
args = ['--external-modules-metadata=%s' % testcfg]
err_msg = "Different length for lists of names/versions in metadata for external module"
self.assertErrorRegex(EasyBuildError, err_msg, init_config, args=args)
def test_remove_file(self):
"""Test remove_file"""
testfile = os.path.join(self.test_prefix, 'foo')
ft.write_file(testfile, 'bar')
self.assertTrue(os.path.exists(testfile))
ft.remove_file(testfile)
ft.write_file(testfile, 'bar')
ft.adjust_permissions(self.test_prefix, stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH, add=False)
self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile)
# also test behaviour of remove_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
self.mock_stdout(True)
ft.remove_file(testfile)
txt = self.get_stdout()
self.mock_stdout(False)
regex = re.compile("^file [^ ]* removed$")
self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def setUp(self):
"""Test setup."""
fd, self.logfile = tempfile.mkstemp(suffix=".log", prefix="eb-options-test-")
os.close(fd)
fd, self.dummylogfn = tempfile.mkstemp(prefix="easybuild-dummy", suffix=".log")
os.close(fd)
# adjust PYTHONPATH such that test easyblocks are found
self.orig_sys_path = sys.path[:]
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "sandbox")))
import easybuild.easyblocks
reload(easybuild.easyblocks)
reload(easybuild.tools.module_naming_scheme)
# clear log
write_file(self.logfile, "")
self.buildpath = tempfile.mkdtemp()
self.installpath = tempfile.mkdtemp()
self.sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox", "sources")
# keep track of original environment to restore
self.orig_environ = copy.deepcopy(os.environ)
def test_toy_tweaked(self):
"""Test toy build with tweaked easyconfig, for testing extra easyconfig parameters."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
ec_file = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb')
shutil.copy2(os.path.join(test_ecs_dir, 'toy-0.0.eb'), ec_file)
# tweak easyconfig by appending to it
ec_extra = '\n'.join([
"versionsuffix = '-tweaked'",
"modextrapaths = {'SOMEPATH': ['foo/bar', 'baz']}",
"modextravars = {'FOO': 'bar'}",
"modloadmsg = 'THANKS FOR LOADING ME, I AM %(name)s v%(version)s'",
"modtclfooter = 'puts stderr \"oh hai!\"'",
])
write_file(ec_file, ec_extra, append=True)
args = [
ec_file,
'--sourcepath=%s' % self.test_sourcepath,
'--buildpath=%s' % self.test_buildpath,
'--installpath=%s' % self.test_installpath,
'--debug',
'--force',
]
outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.check_toy(self.test_installpath, outtxt, versionsuffix='-tweaked')
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked')
toy_module_txt = read_file(toy_module)
self.assertTrue(re.search('setenv\s*FOO\s*"bar"', toy_module_txt))
self.assertTrue(re.search('prepend-path\s*SOMEPATH\s*\$root/foo/bar', toy_module_txt))
self.assertTrue(re.search('prepend-path\s*SOMEPATH\s*\$root/baz', toy_module_txt))
self.assertTrue(re.search('module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0', toy_module_txt))
self.assertTrue(re.search('puts stderr "oh hai!"', toy_module_txt))
def test_expand_glob_paths(self):
"""Test expand_glob_paths function."""
for dirname in ['empty_dir', 'test_dir']:
ft.mkdir(os.path.join(self.test_prefix, dirname), parents=True)
for filename in ['file1.txt', 'test_dir/file2.txt', 'test_dir/file3.txt', 'test_dir2/file4.dat']:
ft.write_file(os.path.join(self.test_prefix, filename), 'gibberish')
globs = [os.path.join(self.test_prefix, '*.txt'), os.path.join(self.test_prefix, '*', '*')]
expected = [
os.path.join(self.test_prefix, 'file1.txt'),
os.path.join(self.test_prefix, 'test_dir', 'file2.txt'),
os.path.join(self.test_prefix, 'test_dir', 'file3.txt'),
os.path.join(self.test_prefix, 'test_dir2', 'file4.dat'),
]
self.assertEqual(sorted(ft.expand_glob_paths(globs)), sorted(expected))
# passing non-glob patterns is fine too
file2 = os.path.join(self.test_prefix, 'test_dir', 'file2.txt')
self.assertEqual(ft.expand_glob_paths([file2]), [file2])
# test expanding of '~' into $HOME value
# hard overwrite $HOME in environment (used by os.path.expanduser) so we can reliably test this
new_home = os.path.join(self.test_prefix, 'home')
ft.mkdir(new_home, parents=True)
ft.write_file(os.path.join(new_home, 'test.txt'), 'test')
os.environ['HOME'] = new_home
self.assertEqual(ft.expand_glob_paths(['~/*.txt']), [os.path.join(new_home, 'test.txt')])
# check behaviour if glob that has no (file) matches is passed
glob_pat = os.path.join(self.test_prefix, 'test_*')
self.assertErrorRegex(EasyBuildError, "No files found using glob pattern", ft.expand_glob_paths, [glob_pat])
def test_toolchain_prepare_rpath(self):
"""Test toolchain.prepare under --rpath"""
# put fake 'gcc' command in place that just echos its arguments
fake_gcc = os.path.join(self.test_prefix, 'fake', 'gcc')
write_file(fake_gcc, '#!/bin/bash\necho "$@"')
adjust_permissions(fake_gcc, stat.S_IXUSR)
os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', ''))
# enable --rpath and prepare toolchain
init_config(build_options={'rpath': True, 'rpath_filter': ['/ba.*']})
tc = self.get_toolchain('gompi', version='1.3.12')
# preparing RPATH wrappers requires --experimental, need to bypass that here
tc.log.experimental = lambda x: x
tc.prepare()
# check whether fake gcc was wrapped and that arguments are what they should be
# no -rpath for /bar because of rpath filter
out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""')
expected = ' '.join([
'-Wl,-rpath=$ORIGIN/../lib',
'-Wl,-rpath=$ORIGIN/../lib64',
'-Wl,--disable-new-dtags',
'-Wl,-rpath=/foo',
'%(user)s.c',
'-L/foo',
'-L/bar',
'$FOO',
'-DX=""',
])
self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
def install_step(self):
"""Custom install procedure for TensorFlow."""
# find .whl file that was built, and install it using 'pip install'
if ("-rc" in self.version):
whl_version = self.version.replace("-rc", "rc")
else:
whl_version = self.version
whl_paths = glob.glob(
os.path.join(self.builddir, 'tensorflow-%s-*.whl' % whl_version))
if not whl_paths:
whl_paths = glob.glob(
os.path.join(self.builddir, 'tensorflow-*.whl'))
if len(whl_paths) == 1:
# --ignore-installed is required to ensure *this* wheel is installed
cmd = "pip install --ignore-installed --prefix=%s %s" % (
self.installdir, whl_paths[0])
# if extensions are listed, assume they will provide all required dependencies,
# so use --no-deps to prevent pip from downloading & installing them
if self.cfg['exts_list']:
cmd += ' --no-deps'
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
else:
raise EasyBuildError("Failed to isolate built .whl in %s: %s",
whl_paths, self.builddir)
# Fix for https://github.com/tensorflow/tensorflow/issues/6341 on Python < 3.3
# If the site-packages/google/__init__.py file is missing, make it an empty file.
# This fixes the "No module named google.protobuf" error that sometimes shows up during sanity_check
# For Python >= 3.3 the logic is reversed: The __init__.py must not exist.
# See e.g. http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html
google_protobuf_dir = os.path.join(self.installdir, self.pylibdir,
'google', 'protobuf')
google_init_file = os.path.join(self.installdir, self.pylibdir,
'google', '__init__.py')
if LooseVersion(det_python_version(
self.python_cmd)) < LooseVersion('3.3'):
if os.path.isdir(
google_protobuf_dir) and not is_readable(google_init_file):
self.log.debug("Creating (empty) missing %s", google_init_file)
write_file(google_init_file, '')
else:
if os.path.exists(google_init_file):
self.log.debug("Removing %s for Python >= 3.3",
google_init_file)
remove_file(google_init_file)
# Fix cuda header paths
# This is needed for building custom TensorFlow ops
if LooseVersion(self.version) < LooseVersion('1.14'):
pyshortver = '.'.join(
get_software_version('Python').split('.')[:2])
regex_subs = [(r'#include "cuda/include/', r'#include "')]
base_path = os.path.join(self.installdir, 'lib',
'python%s' % pyshortver, 'site-packages',
'tensorflow', 'include', 'tensorflow')
for header in glob.glob(
os.path.join(base_path, 'stream_executor', 'cuda',
'cuda*.h')) + glob.glob(
os.path.join(base_path, 'core', 'util',
'cuda*.h')):
apply_regex_substitutions(header, regex_subs)
def test_get_log_filename(self):
"""Test for get_log_filename()."""
tmpdir = tempfile.gettempdir()
res = get_log_filename('foo', '1.2.3')
regex = re.compile(
os.path.join(tmpdir,
r'easybuild-foo-1\.2\.3-[0-9]{8}\.[0-9]{6}\.log$'))
self.assertTrue(regex.match(res),
"Pattern '%s' matches '%s'" % (regex.pattern, res))
res = get_log_filename('foo', '1.2.3', date='19700101')
regex = re.compile(
os.path.join(tmpdir,
r'easybuild-foo-1\.2\.3-19700101\.[0-9]{6}\.log$'))
self.assertTrue(regex.match(res),
"Pattern '%s' matches '%s'" % (regex.pattern, res))
res = get_log_filename('foo', '1.2.3', timestamp='094651')
regex = re.compile(
os.path.join(tmpdir,
r'easybuild-foo-1\.2\.3-[0-9]{8}\.094651\.log$'))
self.assertTrue(regex.match(res),
"Pattern '%s' matches '%s'" % (regex.pattern, res))
res = get_log_filename('foo',
'1.2.3',
date='19700101',
timestamp='094651')
regex = re.compile(
os.path.join(tmpdir,
r'easybuild-foo-1\.2\.3-19700101\.094651\.log$'))
self.assertTrue(regex.match(res),
"Pattern '%s' matches '%s'" % (regex.pattern, res))
# if log file already exists, numbers are added to the filename to obtain a new file path
write_file(res, '')
res = get_log_filename('foo',
'1.2.3',
date='19700101',
timestamp='094651')
regex = re.compile(
os.path.join(tmpdir,
r'easybuild-foo-1\.2\.3-19700101\.094651\.log\.1$'))
self.assertTrue(regex.match(res),
"Pattern '%s' matches '%s'" % (regex.pattern, res))
# adding salt ensures a unique filename (pretty much)
prev_log_filenames = []
for i in range(10):
res = get_log_filename('foo',
'1.2.3',
date='19700101',
timestamp='094651',
add_salt=True)
regex = re.compile(
os.path.join(
tmpdir,
r'easybuild-foo-1\.2\.3-19700101\.094651\.[a-zA-Z]{5}\.log$'
))
self.assertTrue(regex.match(res),
"Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertTrue(res not in prev_log_filenames)
prev_log_filenames.append(res)
def test_flex_robot_paths(self):
"""Test prepend/appending to default robot search path via --robot-paths."""
# unset $EASYBUILD_ROBOT_PATHS that was defined in setUp
del os.environ['EASYBUILD_ROBOT_PATHS']
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
tmpdir = tempfile.mkdtemp(
prefix='easybuild-easyconfigs-pkg-install-path')
mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
test_ecs_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
tmp_ecs_dir = os.path.join(tmpdir, 'easybuild', 'easyconfigs')
copy_dir(test_ecs_path, tmp_ecs_dir)
# prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default
orig_sys_path = sys.path[:]
sys.path = [tmpdir] + [
p for p in sys.path
if not os.path.exists(os.path.join(p, 'easybuild', 'easyconfigs'))
]
# default: only pick up installed easyconfigs via sys.path
eb_go = eboptions.parse_options(args=[])
self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir])
# prepend to default robot path
eb_go = eboptions.parse_options(args=['--robot-paths=/foo:'])
self.assertEqual(eb_go.options.robot_paths, ['/foo', tmp_ecs_dir])
eb_go = eboptions.parse_options(args=['--robot-paths=/foo:/bar/baz/:'])
self.assertEqual(eb_go.options.robot_paths,
['/foo', '/bar/baz/', tmp_ecs_dir])
# append to default robot path
eb_go = eboptions.parse_options(args=['--robot-paths=:/bar/baz'])
self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir, '/bar/baz'])
# append to default robot path
eb_go = eboptions.parse_options(args=['--robot-paths=:/bar/baz:/foo'])
self.assertEqual(eb_go.options.robot_paths,
[tmp_ecs_dir, '/bar/baz', '/foo'])
# prepend and append to default robot path
eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar::/baz'])
self.assertEqual(eb_go.options.robot_paths,
['/foo/bar', tmp_ecs_dir, '/baz'])
eb_go = eboptions.parse_options(
args=['--robot-paths=/foo/bar::/baz:/trala'])
self.assertEqual(eb_go.options.robot_paths,
['/foo/bar', tmp_ecs_dir, '/baz', '/trala'])
eb_go = eboptions.parse_options(
args=['--robot-paths=/foo/bar:/trala::/baz'])
self.assertEqual(eb_go.options.robot_paths,
['/foo/bar', '/trala', tmp_ecs_dir, '/baz'])
# also via $EASYBUILD_ROBOT_PATHS
os.environ['EASYBUILD_ROBOT_PATHS'] = '/foo::/bar/baz'
eb_go = eboptions.parse_options(args=[])
self.assertEqual(eb_go.options.robot_paths,
['/foo', tmp_ecs_dir, '/bar/baz'])
# --robot-paths overrides $EASYBUILD_ROBOT_PATHS
os.environ['EASYBUILD_ROBOT_PATHS'] = '/foobar::/barbar/baz/baz'
eb_go = eboptions.parse_options(args=['--robot-paths=/one::/last'])
self.assertEqual(eb_go.options.robot_paths,
['/one', tmp_ecs_dir, '/last'])
del os.environ['EASYBUILD_ROBOT_PATHS']
# also works with a cfgfile in the mix
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
cfgtxt = '\n'.join([
'[config]',
'robot-paths=/cfgfirst::/cfglast',
])
write_file(config_file, cfgtxt)
eb_go = eboptions.parse_options(
args=['--configfiles=%s' % config_file])
self.assertEqual(eb_go.options.robot_paths,
['/cfgfirst', tmp_ecs_dir, '/cfglast'])
# cfgfile entry is lost when env var and/or cmdline options are used
os.environ['EASYBUILD_ROBOT_PATHS'] = '/envfirst::/envend'
eb_go = eboptions.parse_options(
args=['--configfiles=%s' % config_file])
self.assertEqual(eb_go.options.robot_paths,
['/envfirst', tmp_ecs_dir, '/envend'])
del os.environ['EASYBUILD_ROBOT_PATHS']
eb_go = eboptions.parse_options(args=[
'--robot-paths=/veryfirst:',
'--configfiles=%s' % config_file
])
self.assertEqual(eb_go.options.robot_paths,
['/veryfirst', tmp_ecs_dir])
os.environ['EASYBUILD_ROBOT_PATHS'] = ':/envend'
eb_go = eboptions.parse_options(args=[
'--robot-paths=/veryfirst:',
'--configfiles=%s' % config_file
])
self.assertEqual(eb_go.options.robot_paths,
['/veryfirst', tmp_ecs_dir])
del os.environ['EASYBUILD_ROBOT_PATHS']
# override default robot path
eb_go = eboptions.parse_options(args=['--robot-paths=/foo:/bar/baz'])
self.assertEqual(eb_go.options.robot_paths, ['/foo', '/bar/baz'])
# paths specified via --robot still get preference
first = os.path.join(self.test_prefix, 'first')
mkdir(first)
eb_go = eboptions.parse_options(
args=['--robot-paths=/foo/bar::/baz',
'--robot=%s' % first])
self.assertEqual(eb_go.options.robot_paths,
[first, '/foo/bar', tmp_ecs_dir, '/baz'])
sys.path[:] = orig_sys_path
def test_XDG_CONFIG_env_vars(self):
"""Test effect of XDG_CONFIG* environment variables on default configuration."""
self.purge_environment()
xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
xdg_config_dirs = os.environ.get('XDG_CONFIG_DIRS')
cfg_template = '\n'.join([
'[config]',
'prefix=%s',
])
homedir = os.path.join(self.test_prefix, 'homedir', '.config')
mkdir(os.path.join(homedir, 'easybuild'), parents=True)
write_file(os.path.join(homedir, 'easybuild', 'config.cfg'),
cfg_template % '/home')
dir1 = os.path.join(self.test_prefix, 'dir1')
mkdir(os.path.join(dir1, 'easybuild.d'), parents=True)
write_file(os.path.join(dir1, 'easybuild.d', 'foo.cfg'),
cfg_template % '/foo')
write_file(os.path.join(dir1, 'easybuild.d', 'bar.cfg'),
cfg_template % '/bar')
dir2 = os.path.join(self.test_prefix, 'dir2') # empty on purpose
mkdir(os.path.join(dir2, 'easybuild.d'), parents=True)
dir3 = os.path.join(self.test_prefix, 'dir3')
mkdir(os.path.join(dir3, 'easybuild.d'), parents=True)
write_file(os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'),
cfg_template % '/foobarbaz')
# set $XDG_CONFIG_DIRS to non-existing dir to isolate ourselves from possible system-wide config files
os.environ[
'XDG_CONFIG_DIRS'] = '/there/should/be/no/such/directory/we/hope'
# only $XDG_CONFIG_HOME set (to existing path)
os.environ['XDG_CONFIG_HOME'] = homedir
cfg_files = [os.path.join(homedir, 'easybuild', 'config.cfg')]
reload(eboptions)
eb_go = eboptions.parse_options(args=[])
self.assertEqual(eb_go.options.configfiles, cfg_files)
self.assertEqual(eb_go.options.prefix, '/home')
# $XDG_CONFIG_HOME set, one directory listed in $XDG_CONFIG_DIRS
os.environ['XDG_CONFIG_DIRS'] = dir1
cfg_files = [
os.path.join(dir1, 'easybuild.d', 'bar.cfg'),
os.path.join(dir1, 'easybuild.d', 'foo.cfg'),
os.path.join(homedir, 'easybuild',
'config.cfg'), # $XDG_CONFIG_HOME goes last
]
reload(eboptions)
eb_go = eboptions.parse_options(args=[])
self.assertEqual(eb_go.options.configfiles, cfg_files)
self.assertEqual(eb_go.options.prefix, '/home') # last cfgfile wins
# $XDG_CONFIG_HOME not set, multiple directories listed in $XDG_CONFIG_DIRS
del os.environ['XDG_CONFIG_HOME'] # unset, so should become default
os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join([dir1, dir2, dir3])
cfg_files = [
os.path.join(dir1, 'easybuild.d', 'bar.cfg'),
os.path.join(dir1, 'easybuild.d', 'foo.cfg'),
os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'),
]
reload(eboptions)
eb_go = eboptions.parse_options(args=[])
# note: there may be a config file in $HOME too, so don't use a strict comparison
self.assertEqual(cfg_files, eb_go.options.configfiles[:3])
# $XDG_CONFIG_HOME set to non-existing directory, multiple directories listed in $XDG_CONFIG_DIRS
os.environ['XDG_CONFIG_HOME'] = os.path.join(self.test_prefix,
'nosuchdir')
cfg_files = [
os.path.join(dir1, 'easybuild.d', 'bar.cfg'),
os.path.join(dir1, 'easybuild.d', 'foo.cfg'),
os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'),
]
reload(eboptions)
eb_go = eboptions.parse_options(args=[])
self.assertEqual(eb_go.options.configfiles, cfg_files)
self.assertEqual(eb_go.options.prefix,
'/foobarbaz') # last cfgfile wins
# restore $XDG_CONFIG env vars to original state
if xdg_config_home is None:
del os.environ['XDG_CONFIG_HOME']
else:
os.environ['XDG_CONFIG_HOME'] = xdg_config_home
if xdg_config_dirs is None:
del os.environ['XDG_CONFIG_DIRS']
else:
os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs
reload(eboptions)
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
args = [
'--configfiles',
config_file,
'--debug',
'--buildpath',
testpath1,
'--installpath-software',
installpath_software,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [
os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')
]) # default
self.assertEqual(install_path(),
installpath_software) # via cmdline arg
self.assertEqual(install_path('mod'),
os.path.join(testpath2, 'modules')) # via config file
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
tmpdir = tempfile.mkdtemp(
prefix='easybuild-easyconfigs-pkg-install-path')
mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'easyconfigs')
copy_dir(test_ecs_dir, os.path.join(tmpdir, 'easybuild',
'easyconfigs'))
orig_sys_path = sys.path[:]
sys.path.insert(
0, tmpdir
) # prepend to give it preference over possible other installed easyconfigs pkgs
# test with config file passed via environment variable
# also test for existence of HOME and USER by adding paths to robot-paths
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
'sourcepath = %(DEFAULT_REPOSITORYPATH)s',
'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir',
'robot-paths=/tmp/foo:%(sourcepath)s:%(HOME)s:/tmp/%(USER)s:%(DEFAULT_ROBOT_PATHS)s',
'installpath-modules=%s' % installpath_modules,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath',
testpath2,
]
options = init_config(args=args)
topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild')
self.assertEqual(install_path(), os.path.join(topdir,
'software')) # default
self.assertEqual(install_path('mod'),
installpath_modules), # via config file
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
self.assertEqual(get_repositorypath(),
[os.path.join(topdir, 'ebfiles_repo'), 'somesubdir'
]) # via config file
# hardcoded first entry
self.assertEqual(options.robot_paths[0], '/tmp/foo')
# resolved value for %(sourcepath)s template
self.assertEqual(
options.robot_paths[1],
os.path.join(os.getenv('HOME'), '.local', 'easybuild',
'ebfiles_repo'))
# resolved value for HOME constant
self.assertEqual(options.robot_paths[2], os.getenv('HOME'))
# resolved value that uses USER constant
self.assertEqual(options.robot_paths[3],
os.path.join('/tmp', os.getenv('USER')))
# first path in DEFAULT_ROBOT_PATHS
self.assertEqual(options.robot_paths[4],
os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath',
testpath3,
]
options = init_config(args=args)
self.assertEqual(
source_paths(),
[testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(),
os.path.join(testpath3,
'software')) # via command line
self.assertEqual(install_path('mod'),
installpath_modules), # via config file
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
sys.path[:] = orig_sys_path
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
self.purge_environment()
# check whether configuration via environment variables works as expected
prefix = os.path.join(self.tmpdir, 'testprefix')
buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build',
'path')
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var
options = init_config(args=[])
self.assertEqual(build_path(), buildpath_env_var)
self.assertEqual(install_path(), os.path.join(prefix, 'software'))
self.assertEqual(get_repositorypath(),
[os.path.join(prefix, 'ebfiles_repo')])
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_BUILDPATH']
# check whether configuration via command line arguments works
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--configfiles',
config_file, # force empty config file
'--prefix',
prefix,
'--installpath',
install,
'--repositorypath',
repopath,
'--subdir-software',
'APPS',
]
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'APPS'))
self.assertEqual(install_path(typ='mod'),
os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertTrue(config_file in options.configfiles)
# check mixed command line/env var configuration
prefix = os.path.join(self.tmpdir, 'test3')
install = os.path.join(self.tmpdir, 'test4', 'install')
subdir_software = 'eb-soft'
args = [
'--configfiles',
config_file, # force empty config file
'--installpath',
install,
]
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
os.environ['EASYBUILD_INSTALLPATH_MODULES'] = installpath_modules
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install,
subdir_software))
self.assertEqual(install_path('mod'), installpath_modules)
# subdir options *must* be relative (to --installpath)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = installpath_software
error_regex = r"Found problems validating the options.*'subdir_software' must specify a \*relative\* path"
self.assertErrorRegex(EasyBuildError, error_regex, init_config)
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def writeEC(self):
""" create temporary easyconfig file """
write_file(self.eb_file, self.contents)
class EB_SuiteSparse(ConfigureMake):
"""Support for building SuiteSparse."""
def __init__(self, *args, **kwargs):
"""Custom constructor for SuiteSparse easyblock, initialize custom class parameters."""
super(EB_SuiteSparse, self).__init__(*args, **kwargs)
self.config_name = 'UNKNOWN'
def configure_step(self):
"""Configure build by patching UFconfig.mk or SuiteSparse_config.mk."""
if LooseVersion(self.version) < LooseVersion('4.0'):
self.config_name = 'UFconfig'
else:
self.config_name = 'SuiteSparse_config'
cfgvars = {
'CC': os.getenv('MPICC'),
'CFLAGS': os.getenv('CFLAGS'),
'CXX': os.getenv('MPICXX'),
'F77': os.getenv('MPIF77'),
'F77FLAGS': os.getenv('F77FLAGS'),
}
# Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt
self.cfg.update('buildopts', 'BLAS="%s"' % os.getenv('LIBBLAS_MT'))
self.cfg.update('buildopts', 'LAPACK="%s"' % os.getenv('LIBLAPACK_MT'))
# Get METIS or ParMETIS settings
metis = get_software_root('METIS')
parmetis = get_software_root('ParMETIS')
if parmetis:
metis_path = parmetis
metis_include = os.path.join(parmetis, 'include')
metis_libs = os.path.join(parmetis,
get_software_libdir('ParMETIS'),
'libmetis.a')
elif metis:
metis_path = metis
metis_include = os.path.join(metis, 'include')
metis_libs = os.path.join(metis, get_software_libdir('METIS'),
'libmetis.a')
else:
raise EasyBuildError("Neither METIS or ParMETIS module loaded.")
if LooseVersion(self.version) >= LooseVersion('4.5.1'):
cfgvars.update({
'MY_METIS_LIB': metis_libs,
'MY_METIS_INC': metis_include,
})
else:
cfgvars.update({
'METIS_PATH': metis_path,
'METIS': metis_libs,
})
# patch file
fp = os.path.join(self.cfg['start_dir'], self.config_name,
'%s.mk' % self.config_name)
try:
for line in fileinput.input(fp, inplace=1, backup='.orig'):
for (var, val) in cfgvars.items():
orig_line = line
# for variables in cfgvars, substiture lines assignment
# in the file, whatever they are, by assignments to the
# values in cfgvars
line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var,
r"\1 %s # patched by EasyBuild\n" % val,
line)
if line != orig_line:
cfgvars.pop(var)
sys.stdout.write(line)
except IOError, err:
raise EasyBuildError("Failed to patch %s in: %s", fp, err)
# add remaining entries at the end
if cfgvars:
cfgtxt = '# lines below added automatically by EasyBuild\n'
cfgtxt += '\n'.join(
["%s = %s" % (var, val) for (var, val) in cfgvars.items()])
write_file(fp, cfgtxt, append=True)
DEFAULT_EXAMPLE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'examples')
DEFAULT_MODULE = 'easybuild.easyblocks.generic'
options = {
'out-file': ('Path to output file', 'string', 'store', None, 'o'),
'examples': ('Path to dir that contains example files', 'string', 'store', DEFAULT_EXAMPLE_PATH, 'e'),
'module': ('Name of module to load the easyblocks from', 'string', 'store', DEFAULT_MODULE, 'm')
}
so = simple_option(options)
config.init_build_options({'validate': False, 'external_modules_metadata': {}})
autogen_comment = [
".. This file is automatically generated using the %s script, " % os.path.basename(__file__),
".. and information and docstrings from easyblocks and the EasyBuild framework.",
".. Doo not edit this file manually, but update the docstrings and regenerate it.",
'',
]
easyblocks_overview = gen_easyblocks_overview_rst(so.options.module, so.options.examples, COMMON_PARAMS, DOC_FUNCTIONS)
txt = '\n'.join(autogen_comment + easyblocks_overview)
if so.options.out_file:
write_file(so.options.out_file, txt)
print '%s updated' % so.options.out_file
else:
print txt
def retrieve_blocks_in_spec(spec, only_blocks, silent=False):
"""
Easyconfigs can contain blocks (headed by a [Title]-line)
which contain commands specific to that block. Commands in the beginning of the file
above any block headers are common and shared between each block.
"""
reg_block = re.compile(r"^\s*\[([\w.-]+)\]\s*$", re.M)
reg_dep_block = re.compile(r"^\s*block\s*=(\s*.*?)\s*$", re.M)
spec_fn = os.path.basename(spec)
txt = read_file(spec)
# split into blocks using regex
pieces = reg_block.split(txt)
# the first block contains common statements
common = pieces.pop(0)
# determine version of easyconfig format
ec_format_version = get_format_version(txt)
if ec_format_version is None:
ec_format_version = FORMAT_DEFAULT_VERSION
_log.debug(
"retrieve_blocks_in_spec: derived easyconfig format version: %s" %
ec_format_version)
# blocks in easyconfigs are only supported in easyconfig format 1.0
if pieces and ec_format_version == EasyVersion('1.0'):
# make a map of blocks
blocks = []
while pieces:
block_name = pieces.pop(0)
block_contents = pieces.pop(0)
if block_name in [b['name'] for b in blocks]:
raise EasyBuildError("Found block %s twice in %s.", block_name,
spec)
block = {'name': block_name, 'contents': block_contents}
# dependency block
dep_block = reg_dep_block.search(block_contents)
if dep_block:
dependencies = eval(dep_block.group(1))
if isinstance(dependencies, list):
block['dependencies'] = dependencies
else:
block['dependencies'] = [dependencies]
blocks.append(block)
# make a new easyconfig for each block
# they will be processed in the same order as they are all described in the original file
specs = []
for block in blocks:
name = block['name']
if only_blocks and not (name in only_blocks):
print_msg("Skipping block %s-%s" % (spec_fn, name),
silent=silent)
continue
(fd,
block_path) = tempfile.mkstemp(prefix='easybuild-',
suffix='%s-%s' % (spec_fn, name))
os.close(fd)
txt = common
if 'dependencies' in block:
for dep in block['dependencies']:
if dep not in [b['name'] for b in blocks]:
raise EasyBuildError(
"Block %s depends on %s, but block was not found.",
name, dep)
dep = [b for b in blocks if b['name'] == dep][0]
txt += "\n# Dependency block %s" % (dep['name'])
txt += dep['contents']
txt += "\n# Main block %s" % name
txt += block['contents']
write_file(block_path, txt)
specs.append(block_path)
_log.debug("Found %s block(s) in %s" % (len(specs), spec))
return specs
else:
# no blocks, one file
return [spec]
def build_and_install_software(ecs,
init_session_state,
exit_on_failure=True,
hooks=None):
"""
Build and install software for all provided parsed easyconfig files.
:param ecs: easyconfig files to install software with
:param init_session_state: initial session state, to use in test reports
:param exit_on_failure: whether or not to exit on installation failure
:param hooks: list of defined pre- and post-step hooks
"""
# obtain a copy of the starting environment so each build can start afresh
# we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
# e.g. via easyconfig.handle_allowed_system_deps
init_env = copy.deepcopy(os.environ)
run_hook(START, hooks)
res = []
for ec in ecs:
ec_res = {}
try:
(ec_res['success'], app_log,
err) = build_and_install_one(ec, init_env, hooks=hooks)
ec_res['log_file'] = app_log
if not ec_res['success']:
ec_res['err'] = EasyBuildError(err)
except Exception, err:
# purposely catch all exceptions
ec_res['success'] = False
ec_res['err'] = err
ec_res['traceback'] = traceback.format_exc()
# keep track of success/total count
if ec_res['success']:
test_msg = "Successfully built %s" % ec['spec']
else:
test_msg = "Build of %s failed" % ec['spec']
if 'err' in ec_res:
test_msg += " (err: %s)" % ec_res['err']
# dump test report next to log file
test_report_txt = create_test_report(test_msg, [(ec, ec_res)],
init_session_state)
if 'log_file' in ec_res and ec_res['log_file']:
test_report_fp = "%s_test_report.md" % '.'.join(
ec_res['log_file'].split('.')[:-1])
parent_dir = os.path.dirname(test_report_fp)
# parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used
if os.stat(parent_dir).st_mode & 0200:
write_file(test_report_fp, test_report_txt)
else:
adjust_permissions(parent_dir,
stat.S_IWUSR,
add=True,
recursive=False)
write_file(test_report_fp, test_report_txt)
adjust_permissions(parent_dir,
stat.S_IWUSR,
add=False,
recursive=False)
if not ec_res['success'] and exit_on_failure:
if 'traceback' in ec_res:
raise EasyBuildError(ec_res['traceback'])
else:
raise EasyBuildError(test_msg)
res.append((ec, ec_res))
def test_end2end_singularity_image(self):
"""End-to-end test for --containerize (recipe + image)."""
topdir = os.path.dirname(os.path.abspath(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
containerpath = os.path.join(self.test_prefix, 'containers')
os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
# --containerpath must be an existing directory (this is done to avoid misconfiguration)
mkdir(containerpath)
test_img = os.path.join(self.test_prefix, 'test123.img')
write_file(test_img, '')
args = [
toy_ec,
'-C', # equivalent with --containerize
'--experimental',
'--container-config=bootstrap=localimage,from=%s' % test_img,
'--container-build-image',
]
if which('singularity') is None:
error_pattern = "singularity with version 2.4 or higher not found on your system."
self.assertErrorRegex(EasyBuildError,
error_pattern,
self.eb_main,
args,
raise_error=True)
# install mocked versions of 'sudo' and 'singularity' commands
singularity = os.path.join(self.test_prefix, 'bin', 'singularity')
write_file(singularity, '') # placeholder
adjust_permissions(singularity, stat.S_IXUSR, add=True)
sudo = os.path.join(self.test_prefix, 'bin', 'sudo')
write_file(
sudo,
'#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n'
)
adjust_permissions(sudo, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join(
[os.path.join(self.test_prefix, 'bin'),
os.getenv('PATH')])
for (version, ext) in [('2.4.0', 'simg'), ('3.1.0', 'sif')]:
write_file(singularity, MOCKED_SINGULARITY % {'version': version})
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs = [
r"^== singularity tool found at %s/bin/singularity" %
self.test_prefix,
r"^== singularity version '%s' is 2.4 or higher ... OK" %
version,
r"^== Singularity definition file created at %s/containers/Singularity\.toy-0.0"
% self.test_prefix,
r"^== Running 'sudo\s*\S*/singularity build\s*/.* /.*', you may need to enter your 'sudo' password...",
r"^== Singularity image created at %s/containers/toy-0.0\.%s" %
(self.test_prefix, ext),
]
self.check_regexs(regexs, stdout)
self.assertTrue(
os.path.exists(os.path.join(containerpath,
'toy-0.0.%s' % ext)))
remove_file(os.path.join(containerpath, 'Singularity.toy-0.0'))
# check use of --container-image-format & --container-image-name
write_file(singularity, MOCKED_SINGULARITY % {'version': '2.4.0'})
args.extend([
"--container-image-format=ext3",
"--container-image-name=foo-bar",
])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs = [
r"^== singularity tool found at %s/bin/singularity" %
self.test_prefix,
r"^== singularity version '2.4.0' is 2.4 or higher ... OK",
r"^== Singularity definition file created at %s/containers/Singularity\.foo-bar"
% self.test_prefix,
r"^== Running 'sudo\s*\S*/singularity build --writable /.* /.*', you may need to enter .*",
r"^== Singularity image created at %s/containers/foo-bar\.img$" %
self.test_prefix,
]
self.check_regexs(regexs, stdout)
cont_img = os.path.join(containerpath, 'foo-bar.img')
self.assertTrue(os.path.exists(cont_img))
remove_file(os.path.join(containerpath, 'Singularity.foo-bar'))
# test again with container image already existing
error_pattern = "Container image already exists at %s, not overwriting it without --force" % cont_img
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError,
error_pattern,
self.run_main,
args,
raise_error=True)
self.mock_stdout(False)
args.append('--force')
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs.extend([
"WARNING: overwriting existing container image at %s due to --force"
% cont_img,
])
self.check_regexs(regexs, stdout)
self.assertTrue(os.path.exists(cont_img))
# also check behaviour under --extended-dry-run
args.append('--extended-dry-run')
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
self.check_regexs(regexs, stdout)
# test use of --container-tmpdir
args.append('--container-tmpdir=%s' % self.test_prefix)
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs[
-3] = r"^== Running 'sudo\s*SINGULARITY_TMPDIR=%s \S*/singularity build .*" % self.test_prefix
self.check_regexs(regexs, stdout)
def test_find_flexlm_license(self):
"""Test find_flexlm_license function."""
lic_file1 = os.path.join(self.test_prefix, 'one.lic')
ft.write_file(lic_file1, "This is a license file (no, really!)")
lic_file2 = os.path.join(self.test_prefix, 'two.dat')
ft.write_file(lic_file2, "This is another license file (sure it is!)")
lic_server = '*****@*****.**'
# make test robust against environment in which $LM_LICENSE_FILE is defined
if 'LM_LICENSE_FILE' in os.environ:
del os.environ['LM_LICENSE_FILE']
# default return value
self.assertEqual(ft.find_flexlm_license(), ([], None))
# provided license spec
self.assertEqual(ft.find_flexlm_license(lic_specs=[lic_file1]),
([lic_file1], None))
self.assertEqual(
ft.find_flexlm_license(lic_specs=[lic_server, lic_file2]),
([lic_server, lic_file2], None))
# non-existing license file
os.environ[
'LM_LICENSE_FILE'] = '/no/such/file/unless/you/aim/to/break/this/check'
self.assertEqual(ft.find_flexlm_license(), ([], None))
# existing license file
os.environ['LM_LICENSE_FILE'] = lic_file2
self.assertEqual(ft.find_flexlm_license(),
([lic_file2], 'LM_LICENSE_FILE'))
# directory with existing license files
os.environ['LM_LICENSE_FILE'] = self.test_prefix
self.assertEqual(ft.find_flexlm_license(),
([lic_file1, lic_file2], 'LM_LICENSE_FILE'))
# server spec
os.environ['LM_LICENSE_FILE'] = lic_server
self.assertEqual(ft.find_flexlm_license(),
([lic_server], 'LM_LICENSE_FILE'))
# duplicates are filtered out, order is maintained
os.environ['LM_LICENSE_FILE'] = ':'.join(
[lic_file1, lic_server, self.test_prefix, lic_file2, lic_file1])
self.assertEqual(
ft.find_flexlm_license(),
([lic_file1, lic_server, lic_file2], 'LM_LICENSE_FILE'))
# invalid server spec (missing port)
os.environ['LM_LICENSE_FILE'] = 'test.license.server'
self.assertEqual(ft.find_flexlm_license(), ([], None))
# env var wins of provided lic spec
os.environ['LM_LICENSE_FILE'] = lic_file2
self.assertEqual(ft.find_flexlm_license(lic_specs=[lic_server]),
([lic_file2], 'LM_LICENSE_FILE'))
# custom env var wins over $LM_LICENSE_FILE
os.environ['INTEL_LICENSE_FILE'] = lic_file1
expected = ([lic_file1], 'INTEL_LICENSE_FILE')
self.assertEqual(
ft.find_flexlm_license(custom_env_vars='INTEL_LICENSE_FILE'),
expected)
self.assertEqual(
ft.find_flexlm_license(custom_env_vars=['INTEL_LICENSE_FILE']),
expected)
self.assertEqual(
ft.find_flexlm_license(
custom_env_vars=['NOSUCHENVVAR', 'INTEL_LICENSE_FILE']),
expected)
# $LM_LICENSE_FILE is always considered
os.environ['LM_LICENSE_FILE'] = lic_server
os.environ[
'INTEL_LICENSE_FILE'] = '/no/such/file/unless/you/aim/to/break/this/check'
expected = ([lic_server], 'LM_LICENSE_FILE')
self.assertEqual(
ft.find_flexlm_license(custom_env_vars=['INTEL_LICENSE_FILE']),
expected)
# license server *and* file spec; order is preserved
os.environ['LM_LICENSE_FILE'] = ':'.join(
[lic_file2, lic_server, lic_file1])
self.assertEqual(
ft.find_flexlm_license(),
([lic_file2, lic_server, lic_file1], 'LM_LICENSE_FILE'))
# typical usage
os.environ['LM_LICENSE_FILE'] = lic_server
os.environ[
'INTEL_LICENSE_FILE'] = '/not/a/valid/license/path:%s:/another/bogus/license/file' % lic_file2
expected = ([lic_file2], 'INTEL_LICENSE_FILE')
self.assertEqual(
ft.find_flexlm_license(custom_env_vars='INTEL_LICENSE_FILE'),
expected)
os.environ[
'INTEL_LICENSE_FILE'] = '[email protected]:[email protected]:[email protected]'
expected = (['*****@*****.**', '*****@*****.**',
'*****@*****.**'], 'INTEL_LICENSE_FILE')
self.assertEqual(
ft.find_flexlm_license(custom_env_vars=['INTEL_LICENSE_FILE']),
expected)
# make sure find_flexlm_license is robust against None input;
# this occurs if license_file is left unspecified
del os.environ['INTEL_LICENSE_FILE']
del os.environ['LM_LICENSE_FILE']
self.assertEqual(ft.find_flexlm_license(lic_specs=[None]), ([], None))
def test_fix_broken_easyconfig(self):
"""Test fix_broken_easyconfigs.py script."""
testdir = os.path.dirname(__file__)
topdir = os.path.dirname(os.path.dirname(testdir))
script = os.path.join(topdir, 'easybuild', 'scripts',
'fix_broken_easyconfigs.py')
test_easyblocks = os.path.join(testdir, 'sandbox')
broken_ec_txt_tmpl = '\n'.join([
"# licenseheader",
"%sname = '%s'",
"version = '1.2.3'",
'',
"description = 'foo'",
"homepage = 'http://example.com'",
'',
"toolchain = {'name': 'GCC', 'version': '4.8.2'}",
'',
"premakeopts = 'FOO=libfoo.%%s' %% shared_lib_ext",
"makeopts = 'CC=gcc'",
'',
"license = 'foo.lic'",
])
fixed_ec_txt_tmpl = '\n'.join([
"# licenseheader",
"%sname = '%s'",
"version = '1.2.3'",
'',
"description = 'foo'",
"homepage = 'http://example.com'",
'',
"toolchain = {'name': 'GCC', 'version': '4.8.2'}",
'',
"prebuildopts = 'FOO=libfoo.%%s' %% SHLIB_EXT",
"buildopts = 'CC=gcc'",
'',
"license_file = 'foo.lic'",
])
broken_ec_tmpl = os.path.join(self.test_prefix, '%s.eb')
script_cmd_tmpl = "PYTHONPATH=%s:$PYTHONPATH:%s %s %%s" % (
topdir, test_easyblocks, script)
# don't change it if it isn't broken
broken_ec = broken_ec_tmpl % 'notbroken'
script_cmd = script_cmd_tmpl % broken_ec
fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'ConfigureMake'\n\n",
'foo')
write_file(broken_ec, fixed_ec_txt)
# (dummy) ConfigureMake easyblock is available in test sandbox
script_cmd = script_cmd_tmpl % broken_ec
new_ec_txt = read_file(broken_ec)
self.assertEqual(new_ec_txt, fixed_ec_txt)
self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt))
self.assertFalse(os.path.exists(
'%s.bk' % broken_ec)) # no backup created if nothing was fixed
broken_ec = broken_ec_tmpl % 'nosuchsoftware'
script_cmd = script_cmd_tmpl % broken_ec
broken_ec_txt = broken_ec_txt_tmpl % ('', 'nosuchsoftware')
fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'ConfigureMake'\n\n",
'nosuchsoftware')
# broken easyconfig is fixed in place, original file is backed up
write_file(broken_ec, broken_ec_txt)
run_cmd(script_cmd)
new_ec_txt = read_file(broken_ec)
self.assertEqual(new_ec_txt, fixed_ec_txt)
self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt))
self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt)
self.assertFalse(os.path.exists('%s.bk1' % broken_ec))
# broken easyconfig is fixed in place, original file is backed up, existing backup is not overwritten
write_file(broken_ec, broken_ec_txt)
write_file('%s.bk' % broken_ec, 'thisshouldnot\nbechanged')
run_cmd(script_cmd)
new_ec_txt = read_file(broken_ec)
self.assertEqual(new_ec_txt, fixed_ec_txt)
self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt))
self.assertEqual(read_file('%s.bk' % broken_ec),
'thisshouldnot\nbechanged')
self.assertEqual(read_file('%s.bk1' % broken_ec), broken_ec_txt)
# if easyblock is specified, that part is left untouched
broken_ec = broken_ec_tmpl % 'footoy'
script_cmd = script_cmd_tmpl % broken_ec
broken_ec_txt = broken_ec_txt_tmpl % ("easyblock = 'EB_toy'\n\n",
'foo')
fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'EB_toy'\n\n", 'foo')
write_file(broken_ec, broken_ec_txt)
run_cmd(script_cmd)
new_ec_txt = read_file(broken_ec)
self.assertEqual(new_ec_txt, fixed_ec_txt)
self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt))
self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt)
# for existing easyblocks, "easyblock = 'ConfigureMake'" should *not* be added
# EB_toy easyblock is available in test sandbox
test_easyblocks = os.path.join(testdir, 'sandbox')
broken_ec = broken_ec_tmpl % 'toy'
# path to test easyblocks must be *appended* to PYTHONPATH (due to flattening in easybuild-easyblocks repo)
script_cmd = script_cmd_tmpl % broken_ec
broken_ec_txt = broken_ec_txt_tmpl % ('', 'toy')
fixed_ec_txt = fixed_ec_txt_tmpl % ('', 'toy')
write_file(broken_ec, broken_ec_txt)
run_cmd(script_cmd)
new_ec_txt = read_file(broken_ec)
self.assertEqual(new_ec_txt, fixed_ec_txt)
self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt))
self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt)
def test_move_logs(self):
"""Test move_logs function."""
fh, fp = tempfile.mkstemp()
os.close(fh)
ft.write_file(fp, 'foobar')
ft.write_file(fp + '.1', 'moarfoobar')
ft.move_logs(fp, os.path.join(self.test_prefix, 'foo.log'))
self.assertEqual(
ft.read_file(os.path.join(self.test_prefix, 'foo.log')), 'foobar')
self.assertEqual(
ft.read_file(os.path.join(self.test_prefix, 'foo.log.1')),
'moarfoobar')
ft.write_file(os.path.join(self.test_prefix, 'bar.log'), 'bar')
ft.write_file(os.path.join(self.test_prefix, 'bar.log_1'), 'barbar')
fh, fp = tempfile.mkstemp()
os.close(fh)
ft.write_file(fp, 'moarbar')
ft.write_file(fp + '.1', 'evenmoarbar')
ft.move_logs(fp, os.path.join(self.test_prefix, 'bar.log'))
logs = [
'bar.log', 'bar.log.1', 'bar.log_0', 'bar.log_1',
os.path.basename(self.logfile), 'foo.log', 'foo.log.1'
]
self.assertEqual(
sorted([f for f in os.listdir(self.test_prefix) if 'log' in f]),
logs)
self.assertEqual(
ft.read_file(os.path.join(self.test_prefix, 'bar.log_0')), 'bar')
self.assertEqual(
ft.read_file(os.path.join(self.test_prefix, 'bar.log_1')),
'barbar')
self.assertEqual(
ft.read_file(os.path.join(self.test_prefix, 'bar.log')), 'moarbar')
self.assertEqual(
ft.read_file(os.path.join(self.test_prefix, 'bar.log.1')),
'evenmoarbar')
txt = '\n'.join([
"nwchem_basis_library %(path)s/data/libraries/",
"nwchem_nwpw_library %(path)s/data/libraryps/",
"ffield amber",
"amber_1 %(path)s/data/amber_s/",
"amber_2 %(path)s/data/amber_q/",
"amber_3 %(path)s/data/amber_x/",
"amber_4 %(path)s/data/amber_u/",
"spce %(path)s/data/solvents/spce.rst",
"charmm_s %(path)s/data/charmm_s/",
"charmm_x %(path)s/data/charmm_x/",
]) % {
'path': self.installdir
}
write_file(default_nwchemrc, txt)
# fix permissions in data directory
datadir = os.path.join(self.installdir, 'data')
adjust_permissions(datadir, stat.S_IROTH, add=True, recursive=True)
adjust_permissions(datadir,
stat.S_IXOTH,
add=True,
recursive=True,
onlydirs=True)
def sanity_check_step(self):
"""Custom sanity check for NWChem."""
custom_paths = {
'files': ['bin/nwchem'],
'dirs': [
def configure_step(self):
"""
Configure for GCC build:
- prepare extra source dirs (GMP, MPFR, MPC, ...)
- create obj dir to build in (GCC doesn't like to be built in source dir)
- add configure and make options, according to .eb spec file
- decide whether or not to do a staged build (which is required to enable PPL/CLooG support)
- set platform_lib based on config.guess output
"""
sysroot = build_option('sysroot')
if sysroot:
# based on changes made to GCC in Gentoo Prefix
# https://gitweb.gentoo.org/repo/gentoo.git/tree/profiles/features/prefix/standalone/profile.bashrc
# add --with-sysroot configure option, to instruct GCC to consider
# value set for EasyBuild's --sysroot configuration option as the root filesystem of the operating system
# (see https://gcc.gnu.org/install/configure.html)
self.cfg.update('configopts', '--with-sysroot=%s' % sysroot)
# avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c
apply_regex_substitutions(os.path.join('gcc', 'gcc.c'),
[('--sysroot=%R', '')])
# prefix dynamic linkers with sysroot
# this patches lines like:
# #define GLIBC_DYNAMIC_LINKER64 "/lib64/ld-linux-x86-64.so.2"
gcc_config_headers = glob.glob(
os.path.join('gcc', 'config', '*', '*linux*.h'))
regex_subs = [('(_DYNAMIC_LINKER.*[":])/lib',
r'\1%s/lib' % sysroot)]
apply_regex_substitutions(gcc_config_headers, regex_subs)
# self.configopts will be reused in a 3-staged build,
# configopts is only used in first configure
self.configopts = self.cfg['configopts']
# I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them
stage1_info = self.prep_extra_src_dirs("stage1")
configopts = stage1_info['configopts']
# II) update config options
# enable specified language support
if self.cfg['languages']:
self.configopts += " --enable-languages=%s" % ','.join(
self.cfg['languages'])
# enable building of libiberty, if desired
if self.cfg['withlibiberty']:
self.configopts += " --enable-install-libiberty"
# enable link-time-optimization (LTO) support, if desired
if self.cfg['withlto']:
self.configopts += " --enable-lto"
else:
self.configopts += " --disable-lto"
# configure for a release build
self.configopts += " --enable-checking=release "
# enable multilib: allow both 32 and 64 bit
if self.cfg['multilib']:
glibc_32bit = [
"glibc.i686", # Fedora, RedHat-based
"glibc.ppc", # "" on Power
"libc6-dev-i386", # Debian-based
"gcc-c++-32bit", # OpenSuSE, SLES
]
if not any([check_os_dependency(dep) for dep in glibc_32bit]):
raise EasyBuildError(
"Using multilib requires 32-bit glibc (install one of %s, depending on your OS)",
', '.join(glibc_32bit))
self.configopts += " --enable-multilib --with-multilib-list=m32,m64"
else:
self.configopts += " --disable-multilib"
# build both static and dynamic libraries (???)
self.configopts += " --enable-shared=yes --enable-static=yes "
# use POSIX threads
self.configopts += " --enable-threads=posix "
# enable plugin support
self.configopts += " --enable-plugins "
# use GOLD as default linker
if self.cfg['use_gold_linker']:
self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold"
else:
self.configopts += " --enable-gold --enable-ld=default"
# enable bootstrap build for self-containment (unless for staged build)
if not self.stagedbuild:
configopts += " --enable-bootstrap"
else:
configopts += " --disable-bootstrap"
if self.stagedbuild:
#
# STAGE 1: configure GCC build that will be used to build PPL/CLooG
#
self.log.info(
"Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support..."
)
self.stage1installdir = os.path.join(self.builddir,
'GCC_stage1_eb')
configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {
'p': self.stage1installdir
}
else:
# unstaged build, so just run standard configure/make/make install
# set prefixes
self.log.info("Performing regular GCC build...")
configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {
'p': self.installdir
}
# prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config
# only do this when multilib is not enabled
if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']:
cfgfile = 'gcc/config/i386/t-linux64'
multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32"
self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'",
cfgfile, multilib_osdirnames)
write_file(cfgfile, multilib_osdirnames, append=True)
elif self.cfg['multilib']:
self.log.info(
"Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled"
)
# III) create obj dir to build in, and change to it
# GCC doesn't like to be built in the source dir
if self.stagedbuild:
objdir = self.create_dir("stage1_obj")
self.stage1prefix = objdir
else:
objdir = self.create_dir("obj")
# note: this also triggers the use of an updated config.guess script
# (unless both the 'build_type' and 'host_type' easyconfig parameters are specified)
build_type, host_type = self.determine_build_and_host_type()
if build_type:
configopts += ' --build=' + build_type
if host_type:
configopts += ' --host=' + host_type
# IV) actual configure, but not on default path
cmd = "../configure %s %s" % (self.configopts, configopts)
self.run_configure_cmd(cmd)
self.disable_lto_mpfr_old_gcc(objdir)
def test_adjust_permissions(self):
"""Test adjust_permissions"""
# set umask hard to run test reliably
orig_umask = os.umask(0022)
# prep files/dirs/(broken) symlinks is test dir
# file: rw-r--r--
ft.write_file(os.path.join(self.test_prefix, 'foo'), 'foo')
foo_perms = os.stat(os.path.join(self.test_prefix,
'foo'))[stat.ST_MODE]
for bit in [stat.S_IRUSR, stat.S_IWUSR, stat.S_IRGRP, stat.S_IROTH]:
self.assertTrue(foo_perms & bit)
for bit in [
stat.S_IXUSR, stat.S_IWGRP, stat.S_IXGRP, stat.S_IWOTH,
stat.S_IXOTH
]:
self.assertFalse(foo_perms & bit)
# dir: rwxr-xr-x
ft.mkdir(os.path.join(self.test_prefix, 'bar'))
bar_perms = os.stat(os.path.join(self.test_prefix,
'bar'))[stat.ST_MODE]
for bit in [
stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRGRP,
stat.S_IXGRP, stat.S_IROTH, stat.S_IXOTH
]:
self.assertTrue(bar_perms & bit)
for bit in [stat.S_IWGRP, stat.S_IWOTH]:
self.assertFalse(bar_perms & bit)
# file in dir: rw-r--r--
foobar_path = os.path.join(self.test_prefix, 'bar', 'foobar')
ft.write_file(foobar_path, 'foobar')
foobar_perms = os.stat(foobar_path)[stat.ST_MODE]
for bit in [stat.S_IRUSR, stat.S_IWUSR, stat.S_IRGRP, stat.S_IROTH]:
self.assertTrue(foobar_perms & bit)
for bit in [
stat.S_IXUSR, stat.S_IWGRP, stat.S_IXGRP, stat.S_IWOTH,
stat.S_IXOTH
]:
self.assertFalse(foobar_perms & bit)
# include symlink
os.symlink(foobar_path, os.path.join(self.test_prefix,
'foobar_symlink'))
# include broken symlink (symlinks are skipped, so this shouldn't cause problems)
tmpfile = os.path.join(self.test_prefix, 'thiswontbetherelong')
ft.write_file(tmpfile, 'poof!')
os.symlink(tmpfile, os.path.join(self.test_prefix, 'broken_symlink'))
os.remove(tmpfile)
# test default behaviour:
# recursive, add permissions, relative to existing permissions, both files and dirs, skip symlinks
# add user execution, group write permissions
ft.adjust_permissions(self.test_prefix, stat.S_IXUSR | stat.S_IWGRP)
# foo file: rwxrw-r--
foo_perms = os.stat(os.path.join(self.test_prefix,
'foo'))[stat.ST_MODE]
for bit in [
stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRGRP,
stat.S_IWGRP, stat.S_IROTH
]:
self.assertTrue(foo_perms & bit)
for bit in [stat.S_IXGRP, stat.S_IWOTH, stat.S_IXOTH]:
self.assertFalse(foo_perms & bit)
# bar dir: rwxrwxr-x
bar_perms = os.stat(os.path.join(self.test_prefix,
'bar'))[stat.ST_MODE]
for bit in [
stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRGRP,
stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IXOTH
]:
self.assertTrue(bar_perms & bit)
self.assertFalse(bar_perms & stat.S_IWOTH)
# foo/foobar file: rwxrw-r--
for path in [
os.path.join(self.test_prefix, 'bar', 'foobar'),
os.path.join(self.test_prefix, 'foobar_symlink')
]:
perms = os.stat(path)[stat.ST_MODE]
for bit in [
stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRGRP,
stat.S_IWGRP, stat.S_IROTH
]:
self.assertTrue(perms & bit)
for bit in [stat.S_IXGRP, stat.S_IWOTH, stat.S_IXOTH]:
self.assertFalse(perms & bit)
# broken symlinks are trouble if symlinks are not skipped
self.assertErrorRegex(EasyBuildError,
"No such file or directory",
ft.adjust_permissions,
self.test_prefix,
stat.S_IXUSR,
skip_symlinks=False)
# restore original umask
os.umask(orig_umask)
def post_install_step(self):
"""
Install group libraries and interfaces (if desired).
"""
super(EB_imkl, self).post_install_step()
# extract examples
examples_subdir = os.path.join(self.installdir, self.mkl_basedir,
'examples')
if os.path.exists(examples_subdir):
cwd = change_dir(examples_subdir)
for examples_tarball in glob.glob('examples_*.tgz'):
run_cmd("tar xvzf %s -C ." % examples_tarball)
change_dir(cwd)
# reload the dependencies
self.load_dependency_modules()
shlib_ext = get_shared_lib_ext()
if self.cfg['m32']:
extra = {
'libmkl.%s' % shlib_ext:
'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)',
'libmkl_em64t.a':
'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_solver.a': 'GROUP (libmkl_solver.a)',
'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)',
'libmkl_lapack.a':
'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)'
}
else:
extra = {
'libmkl.%s' % shlib_ext:
'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)',
'libmkl_em64t.a':
'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)',
'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)',
'libmkl_lapack.a':
'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)'
}
loosever = LooseVersion(self.version)
if loosever >= LooseVersion('10.3'):
libsubdir = os.path.join(self.mkl_basedir, 'lib', 'intel64')
else:
if self.cfg['m32']:
libsubdir = os.path.join('lib', '32')
else:
libsubdir = os.path.join('lib', 'em64t')
for fil, txt in extra.items():
dest = os.path.join(self.installdir, libsubdir, fil)
if not os.path.exists(dest):
write_file(dest, txt)
# build the mkl interfaces, if desired
if self.cfg['interfaces']:
if loosever >= LooseVersion('10.3'):
intsubdir = os.path.join(self.mkl_basedir, 'interfaces')
inttarget = 'libintel64'
else:
intsubdir = 'interfaces'
if self.cfg['m32']:
inttarget = 'lib32'
else:
inttarget = 'libem64t'
cmd = "make -f makefile %s" % inttarget
# blas95 and lapack95 need more work, ignore for now
# blas95 and lapack also need include/.mod to be processed
fftw2libs = ['fftw2xc', 'fftw2xf']
fftw3libs = ['fftw3xc', 'fftw3xf']
interfacedir = os.path.join(self.installdir, intsubdir)
change_dir(interfacedir)
self.log.info("Changed to interfaces directory %s", interfacedir)
compopt = None
# determine whether we're using a non-Intel GCC-based or PGI-based toolchain
# can't use toolchain.comp_family, because of system toolchain used when installing imkl
if get_software_root('icc') or get_software_root(
'intel-compilers'):
compopt = 'compiler=intel'
else:
# check for PGI first, since there's a GCC underneath PGI too...
if get_software_root('PGI'):
compopt = 'compiler=pgi'
elif get_software_root('GCC'):
compopt = 'compiler=gnu'
else:
raise EasyBuildError(
"Not using Intel/GCC/PGI compilers, don't know how to build wrapper libs"
)
# patch makefiles for cdft wrappers when PGI is used as compiler
if get_software_root('PGI'):
regex_subs = [
# pgi should be considered as a valid compiler
("intel gnu", "intel gnu pgi"),
# transform 'gnu' case to 'pgi' case
(r"ifeq \(\$\(compiler\),gnu\)", "ifeq ($(compiler),pgi)"),
('=gcc', '=pgcc'),
# correct flag to use C99 standard
('-std=c99', '-c99'),
# -Wall and -Werror are not valid options for pgcc, no close equivalent
('-Wall', ''),
('-Werror', ''),
]
for lib in self.cdftlibs:
apply_regex_substitutions(
os.path.join(interfacedir, lib, 'makefile'),
regex_subs)
for lib in fftw2libs + fftw3libs + self.cdftlibs:
buildopts = [compopt]
if lib in fftw3libs:
buildopts.append('install_to=$INSTALL_DIR')
elif lib in self.cdftlibs:
if self.mpi_spec is not None:
buildopts.append('mpi=%s' % self.mpi_spec)
precflags = ['']
if lib.startswith('fftw2x') and not self.cfg['m32']:
# build both single and double precision variants
precflags = [
'PRECISION=MKL_DOUBLE', 'PRECISION=MKL_SINGLE'
]
intflags = ['']
if lib in self.cdftlibs and not self.cfg['m32']:
# build both 32-bit and 64-bit interfaces
intflags = ['interface=lp64', 'interface=ilp64']
allopts = [
list(opts)
for opts in itertools.product(intflags, precflags)
]
for flags, extraopts in itertools.product(['', '-fPIC'],
allopts):
tup = (lib, flags, buildopts, extraopts)
self.log.debug(
"Building lib %s with: flags %s, buildopts %s, extraopts %s"
% tup)
tmpbuild = tempfile.mkdtemp(dir=self.builddir)
self.log.debug("Created temporary directory %s" % tmpbuild)
# always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS
# fftw2x(c|f): use $INSTALL_DIR, $CFLAGS and $COPTS
# fftw3x(c|f): use $CFLAGS
# fftw*cdft: use $INSTALL_DIR and $SPEC_OPT
env.setvar('INSTALL_DIR', tmpbuild)
env.setvar('SPEC_OPT', flags)
env.setvar('COPTS', flags)
env.setvar('CFLAGS', flags)
try:
intdir = os.path.join(interfacedir, lib)
os.chdir(intdir)
self.log.info("Changed to interface %s directory %s" %
(lib, intdir))
except OSError as err:
raise EasyBuildError(
"Can't change to interface %s directory %s: %s",
lib, intdir, err)
fullcmd = "%s %s" % (cmd, ' '.join(buildopts + extraopts))
res = run_cmd(fullcmd, log_all=True, simple=True)
if not res:
raise EasyBuildError(
"Building %s (flags: %s, fullcmd: %s) failed", lib,
flags, fullcmd)
for fn in os.listdir(tmpbuild):
src = os.path.join(tmpbuild, fn)
if flags == '-fPIC':
# add _pic to filename
ff = fn.split('.')
fn = '.'.join(ff[:-1]) + '_pic.' + ff[-1]
dest = os.path.join(self.installdir, libsubdir, fn)
try:
if os.path.isfile(src):
shutil.move(src, dest)
self.log.info("Moved %s to %s" % (src, dest))
except OSError as err:
raise EasyBuildError("Failed to move %s to %s: %s",
src, dest, err)
remove_dir(tmpbuild)
def post_install_step(self, *args, **kwargs):
"""
Post-processing after installation: add symlinks for cc, c++, f77, f95
"""
super(EB_GCC, self).post_install_step(*args, **kwargs)
# Add symlinks for cc/c++/f77/f95.
bindir = os.path.join(self.installdir, 'bin')
for key in COMP_CMD_SYMLINKS:
src = COMP_CMD_SYMLINKS[key]
target = os.path.join(bindir, key)
if os.path.exists(target):
self.log.info(
"'%s' already exists in %s, not replacing it with symlink to '%s'",
key, bindir, src)
elif os.path.exists(os.path.join(bindir, src)):
symlink(src, target, use_abspath_source=False)
else:
raise EasyBuildError(
"Can't link '%s' to non-existing location %s", target,
os.path.join(bindir, src))
# Rename include-fixed directory which includes system header files that were processed by fixincludes,
# since these may cause problems when upgrading to newer OS version.
# (see https://github.com/easybuilders/easybuild-easyconfigs/issues/10666)
glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc',
'*-linux-gnu', self.version,
'include-fixed')
matches = glob.glob(glob_pattern)
if matches:
if len(matches) == 1:
include_fixed_path = matches[0]
msg = "Found include-fixed subdirectory at %s, "
msg += "renaming it to avoid using system header files patched by fixincludes..."
self.log.info(msg, include_fixed_path)
# limits.h and syslimits.h need to be copied to include/ first,
# these are strictly required (by /usr/include/limits.h for example)
include_path = os.path.join(
os.path.dirname(include_fixed_path), 'include')
retained_header_files = ['limits.h', 'syslimits.h']
for fn in retained_header_files:
from_path = os.path.join(include_fixed_path, fn)
to_path = os.path.join(include_path, fn)
if os.path.exists(from_path):
if os.path.exists(to_path):
raise EasyBuildError(
"%s already exists, not overwriting it with %s!",
to_path, from_path)
else:
copy_file(from_path, to_path)
self.log.info("%s copied to %s before renaming %s",
from_path, to_path,
include_fixed_path)
else:
self.log.warning(
"Can't copy non-existing file %s to %s, since it doesn't exist!",
from_path, to_path)
readme = os.path.join(include_fixed_path, 'README.easybuild')
readme_txt = '\n'.join([
"This directory was renamed by EasyBuild to avoid that the header files in it are picked up,",
"since they may cause problems when the OS is upgraded to a new (minor) version.",
'',
"These files were copied to %s first: %s" %
(include_path, ', '.join(retained_header_files)),
'',
"See https://github.com/easybuilders/easybuild-easyconfigs/issues/10666 for more information.",
'',
])
write_file(readme, readme_txt)
include_fixed_renamed = include_fixed_path + '.renamed-by-easybuild'
move_file(include_fixed_path, include_fixed_renamed)
self.log.info(
"%s renamed to %s to avoid using the header files in it",
include_fixed_path, include_fixed_renamed)
else:
raise EasyBuildError(
"Exactly one 'include-fixed' directory expected, found %d: %s",
len(matches), matches)
else:
self.log.info("No include-fixed subdirectory found at %s",
glob_pattern)
def test_step(self):
"""Run GAMESS-US tests (if 'runtest' easyconfig parameter is set to True)."""
# don't use provided 'runall' script for tests, since that only runs the tests single-core
if self.cfg['runtest']:
if not build_option('mpi_tests'):
self.log.info(
"Skipping testing of GAMESS-US since MPI testing is disabled"
)
return
try:
cwd = os.getcwd()
os.chdir(self.testdir)
except OSError as err:
raise EasyBuildError(
"Failed to move to temporary directory for running tests: %s",
err)
# copy input files for exam<id> standard tests
for test_input in glob.glob(
os.path.join(self.installdir, 'tests', 'standard',
'exam*.inp')):
try:
shutil.copy2(test_input, os.getcwd())
except OSError as err:
raise EasyBuildError("Failed to copy %s to %s: %s",
test_input, os.getcwd(), err)
rungms = os.path.join(self.installdir, 'rungms')
test_env_vars = ['TMPDIR=%s' % self.testdir]
if self.toolchain.mpi_family() == toolchain.INTELMPI:
test_env_vars.extend([
'I_MPI_FALLBACK=enable', # enable fallback in case first fabric fails (see $I_MPI_FABRICS_LIST)
'I_MPI_HYDRA_BOOTSTRAP=fork', # tests are only run locally (2 processes), so no SSH required
])
# run all exam<id> tests, dump output to exam<id>.log
n_tests = 47
for i in range(1, n_tests + 1):
test_cmd = ' '.join(
test_env_vars +
[rungms, 'exam%02d' % i, self.version, '1', '2'])
(out, _) = run_cmd(test_cmd, log_all=True, simple=False)
write_file('exam%02d.log' % i, out)
# verify output of tests
check_cmd = os.path.join(self.installdir, 'tests', 'standard',
'checktst')
(out, _) = run_cmd(check_cmd, log_all=True, simple=False)
success_regex = re.compile(
"^All %d test results are correct" % n_tests, re.M)
if success_regex.search(out):
self.log.info("All tests ran successfully!")
else:
raise EasyBuildError("Not all tests ran successfully...")
# cleanup
os.chdir(cwd)
try:
shutil.rmtree(self.testdir)
except OSError as err:
raise EasyBuildError("Failed to remove test directory %s: %s",
self.testdir, err)
def dump_env_script(easyconfigs):
"""
Dump source scripts that set up build environment for specified easyconfigs.
:param easyconfigs: list of easyconfigs to generate scripts for
"""
ecs_and_script_paths = []
for easyconfig in easyconfigs:
script_path = '%s.env' % os.path.splitext(
os.path.basename(easyconfig['spec']))[0]
ecs_and_script_paths.append((easyconfig['ec'], script_path))
# don't just overwrite existing scripts
existing_scripts = [
s for (_, s) in ecs_and_script_paths if os.path.exists(s)
]
if existing_scripts:
if build_option('force'):
_log.info("Found existing scripts, overwriting them: %s",
' '.join(existing_scripts))
else:
raise EasyBuildError(
"Script(s) already exists, not overwriting them (unless --force is used): %s",
' '.join(existing_scripts))
orig_env = copy.deepcopy(os.environ)
for ec, script_path in ecs_and_script_paths:
# obtain EasyBlock instance
app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
app = app_class(ec)
# mimic dry run, and keep quiet
app.dry_run = app.silent = app.toolchain.dry_run = True
# prepare build environment (in dry run mode)
app.check_readiness_step()
app.prepare_step(start_dir=False)
# compose script
ecfile = os.path.basename(ec.path)
script_lines = [
"#!/bin/bash",
"# script to set up build environment as defined by EasyBuild v%s for %s"
% (EASYBUILD_VERSION, ecfile),
"# usage: source %s" % os.path.basename(script_path),
]
script_lines.extend(['', "# toolchain & dependency modules"])
if app.toolchain.modules:
script_lines.extend(
["module load %s" % mod for mod in app.toolchain.modules])
else:
script_lines.append("# (no modules loaded)")
script_lines.extend(['', "# build environment"])
if app.toolchain.vars:
env_vars = sorted(app.toolchain.vars.items())
script_lines.extend([
"export %s='%s'" % (var, val.replace("'", "\\'"))
for (var, val) in env_vars
])
else:
script_lines.append("# (no build environment defined)")
write_file(script_path, '\n'.join(script_lines))
print_msg("Script to set up build environment for %s dumped to %s" %
(ecfile, script_path),
prefix=False)
restore_env(orig_env)
def test_fetch_files_from_pr_cache(self):
"""Test caching for fetch_files_from_pr."""
if self.skip_github_tests:
print(
"Skipping test_fetch_files_from_pr_cache, no GitHub token available?"
)
return
init_config(build_options={
'pr_target_account': gh.GITHUB_EB_MAIN,
})
# clear cache first, to make sure we start with a clean slate
gh.fetch_files_from_pr.clear_cache()
self.assertFalse(gh.fetch_files_from_pr._cache)
pr7159_filenames = [
'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb',
'OpenFOAM-5.0-20180108-foss-2018a.eb',
'OpenFOAM-5.0-20180108-intel-2018a.eb', 'OpenFOAM-6-foss-2018b.eb',
'OpenFOAM-6-intel-2018a.eb', 'OpenFOAM-v1806-foss-2018b.eb',
'PETSc-3.9.3-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018a.eb',
'SCOTCH-6.0.6-foss-2018b.eb', 'SCOTCH-6.0.6-intel-2018a.eb',
'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb'
]
pr7159_files = gh.fetch_easyconfigs_from_pr(
7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(sorted(pr7159_filenames),
sorted(os.path.basename(f) for f in pr7159_files))
# check that cache has been populated for PR 7159
self.assertEqual(len(gh.fetch_files_from_pr._cache.keys()), 1)
# github_account value is None (results in using default 'easybuilders')
cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix)
self.assertTrue(cache_key in gh.fetch_files_from_pr._cache.keys())
cache_entry = gh.fetch_files_from_pr._cache[cache_key]
self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]),
sorted(pr7159_filenames))
# same query should return result from cache entry
res = gh.fetch_easyconfigs_from_pr(7159,
path=self.test_prefix,
github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(res, pr7159_files)
# inject entry in cache and check result of matching query
pr_id = 12345
tmpdir = os.path.join(self.test_prefix, 'easyblocks-pr-12345')
pr12345_files = [
os.path.join(tmpdir, 'foo.py'),
os.path.join(tmpdir, 'bar.py'),
]
for fp in pr12345_files:
write_file(fp, '')
# github_account value is None (results in using default 'easybuilders')
cache_key = (pr_id, None, 'easybuild-easyblocks', tmpdir)
gh.fetch_files_from_pr.update_cache({cache_key: pr12345_files})
res = gh.fetch_easyblocks_from_pr(12345, tmpdir)
self.assertEqual(sorted(pr12345_files), sorted(res))
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# compose versions older/newer than current version
depr_ver = int(os.environ['EASYBUILD_DEPRECATED'])
older_ver = str(depr_ver - 1)
newer_ver = str(depr_ver + 1)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
self.mock_stderr(True)
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warning("justawarning")
log.deprecated("anotherwarning", newer_ver)
log.deprecated("onemorewarning", '1.0', '2.0')
log.deprecated("lastwarning", '1.0', max_ver='2.0')
log.deprecated("thisisnotprinted", '1.0', max_ver='2.0', silent=True)
log.error("kaput")
log.error("err: %s", 'msg: %s')
stderr = self.get_stderr()
self.mock_stderr(False)
more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information"
expected_stderr = '\n\n'.join([
"\nWARNING: Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info,
"\nWARNING: Deprecated functionality, will no longer work in v2.0: onemorewarning",
"\nWARNING: Deprecated functionality, will no longer work in v2.0: lastwarning",
]) + '\n\n'
self.assertEqual(stderr, expected_stderr)
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"root.test_easybuildlog \[DEBUG\] :: 123 debug",
r"root.test_easybuildlog \[INFO\] :: foobar info",
r"root.test_easybuildlog \[WARNING\] :: justawarning",
r"root.test_easybuildlog \[WARNING\] :: Deprecated functionality.*anotherwarning.*",
r"root.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*",
r"root.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*",
r"root.test_easybuildlog \[WARNING\] :: Deprecated functionality.*thisisnotprinted.*",
r"root.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput",
r"root.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s",
r"root.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops",
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
self.assertErrorRegex(EasyBuildError, r"DEPRECATED \(since .*: kaput", log.deprecated, "kaput", older_ver)
self.assertErrorRegex(EasyBuildError, r"DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0', '1.0')
self.assertErrorRegex(EasyBuildError, r"DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0',
max_ver='1.0')
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warning("%s", "bleh")
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.error("foo %s baz", 'baz')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"root.test_easybuildlog \[WARNING\] :: bleh",
r"root.test_easybuildlog \[INFO\] :: 4\+2 = 42",
r"root.test_easybuildlog \[DEBUG\] :: this is just a test",
r"root.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz",
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
write_file(tmplog, '')
logToFile(tmplog, enable=True)
# also test use of 'more_info' named argument for log.deprecated
self.mock_stderr(True)
log.deprecated("\nthis is just a test\n", newer_ver, more_info="(see URLGOESHERE for more information)")
self.mock_stderr(False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
"[WARNING] :: Deprecated functionality, will no longer work in v10000001: ",
"this is just a test",
"(see URLGOESHERE for more information)",
])
self.assertTrue(logtxt.strip().endswith(expected_logtxt))
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
self.purge_environment()
# check whether configuration via environment variables works as expected
prefix = os.path.join(self.tmpdir, 'testprefix')
buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build',
'path')
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var
options = init_config(args=[])
self.assertEqual(build_path(), buildpath_env_var)
self.assertEqual(install_path(), os.path.join(prefix, 'software'))
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_BUILDPATH']
# check whether configuration via command line arguments works
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--config',
config_file, # force empty oldstyle config file
'--prefix',
prefix,
'--installpath',
install,
'--repositorypath',
repopath,
]
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'software'))
self.assertEqual(install_path(typ='mod'),
os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertEqual(options.config, config_file)
# check mixed command line/env var configuration
prefix = os.path.join(self.tmpdir, 'test3')
install = os.path.join(self.tmpdir, 'test4', 'install')
subdir_software = 'eb-soft'
args = [
'--config',
config_file, # force empty oldstyle config file
'--installpath',
install,
]
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install,
subdir_software))
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def configure_step(self):
"""Configure build by patching UFconfig.mk or SuiteSparse_config.mk."""
if LooseVersion(self.version) < LooseVersion('4.0'):
self.config_name = 'UFconfig'
else:
self.config_name = 'SuiteSparse_config'
cfgvars = {
'CC': os.getenv('MPICC'),
'CFLAGS': os.getenv('CFLAGS'),
'CXX': os.getenv('MPICXX'),
'F77': os.getenv('MPIF77'),
'F77FLAGS': os.getenv('F77FLAGS'),
}
# avoid that (system) Intel compilers are always considered
self.cfg.update('buildopts', 'AUTOCC=no')
# Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt
self.cfg.update('buildopts', 'BLAS="%s"' % os.getenv('LIBBLAS_MT'))
self.cfg.update('buildopts', 'LAPACK="%s"' % os.getenv('LIBLAPACK_MT'))
# Get CUDA and set it up appropriately
cuda = get_software_root('CUDA')
if cuda:
cuda_cc_space_sep = self.cfg.get_cuda_cc_template_value(
'cuda_cc_space_sep').replace('.', '').split()
nvcc_gencode = ' '.join([
'-gencode=arch=compute_' + x + ',code=sm_' + x
for x in cuda_cc_space_sep
])
cfgvars.update({
'NVCCFLAGS':
' '.join(['-Xcompiler', '-fPIC', '-O3', nvcc_gencode]),
})
# Get METIS or ParMETIS settings
metis = get_software_root('METIS')
parmetis = get_software_root('ParMETIS')
if parmetis:
metis_path = parmetis
metis_include = os.path.join(parmetis, 'include')
metis_libs = os.path.join(parmetis,
get_software_libdir('ParMETIS'),
'libmetis.a')
elif metis:
metis_path = metis
metis_include = os.path.join(metis, 'include')
metis_libs = os.path.join(metis, get_software_libdir('METIS'),
'libmetis.a')
else:
raise EasyBuildError("Neither METIS or ParMETIS module loaded.")
if LooseVersion(self.version) >= LooseVersion('4.5.1'):
cfgvars.update({
'MY_METIS_LIB': metis_libs,
'MY_METIS_INC': metis_include,
})
else:
cfgvars.update({
'METIS_PATH': metis_path,
'METIS': metis_libs,
})
# patch file
fp = os.path.join(self.cfg['start_dir'], self.config_name,
'%s.mk' % self.config_name)
try:
for line in fileinput.input(fp, inplace=1, backup='.orig'):
for (var, val) in list(cfgvars.items()):
# Let's overwrite NVCCFLAGS at the end, since the line breaks and the fact that it appears multiple
# times makes it tricky to handle it properly
if var != 'NVCCFLAGS':
orig_line = line
# for variables in cfgvars, substiture lines assignment
# in the file, whatever they are, by assignments to the
# values in cfgvars
line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var,
r"\1 %s # patched by EasyBuild\n" % val,
line)
if line != orig_line:
cfgvars.pop(var)
sys.stdout.write(line)
except IOError as err:
raise EasyBuildError("Failed to patch %s in: %s", fp, err)
# add remaining entries at the end
if cfgvars:
cfgtxt = '# lines below added automatically by EasyBuild\n'
cfgtxt += '\n'.join(
["%s = %s" % (var, val) for (var, val) in cfgvars.items()])
write_file(fp, cfgtxt, append=True)
def mpi_cmd_for(self, cmd, nr_ranks):
"""Construct an MPI command for the given command and number of ranks."""
# parameter values for mpirun command
params = {
'nr_ranks': nr_ranks,
'cmd': cmd,
}
# different known mpirun commands
mpirun_n_cmd = "mpirun -n %(nr_ranks)d %(cmd)s"
mpi_cmds = {
toolchain.OPENMPI: mpirun_n_cmd, # @UndefinedVariable
toolchain.QLOGICMPI: "mpirun -H localhost -np %(nr_ranks)d %(cmd)s", # @UndefinedVariable
toolchain.INTELMPI: "mpirun %(mpdbf)s %(nodesfile)s -np %(nr_ranks)d %(cmd)s", # @UndefinedVariable
toolchain.MVAPICH2: mpirun_n_cmd, # @UndefinedVariable
toolchain.MPICH: mpirun_n_cmd, # @UndefinedVariable
toolchain.MPICH2: mpirun_n_cmd, # @UndefinedVariable
}
mpi_family = self.mpi_family()
# Intel MPI mpirun needs more work
if mpi_family == toolchain.INTELMPI: # @UndefinedVariable
# set temporary dir for mdp
# note: this needs to be kept *short*, to avoid mpirun failing with "socket.error: AF_UNIX path too long"
# exact limit is unknown, but ~20 characters seems to be OK
env.setvar('I_MPI_MPD_TMPDIR', tempfile.gettempdir())
mpd_tmpdir = os.environ['I_MPI_MPD_TMPDIR']
if len(mpd_tmpdir) > 20:
self.log.warning("$I_MPI_MPD_TMPDIR should be (very) short to avoid problems: %s" % mpd_tmpdir)
# temporary location for mpdboot and nodes files
tmpdir = tempfile.mkdtemp(prefix='mpi_cmd_for-')
# set PBS_ENVIRONMENT, so that --file option for mpdboot isn't stripped away
env.setvar('PBS_ENVIRONMENT', "PBS_BATCH_MPI")
# make sure we're always using mpd as process manager
# only required for/picked up by Intel MPI v4.1 or higher, no harm done for others
env.setvar('I_MPI_PROCESS_MANAGER', 'mpd')
# create mpdboot file
fn = os.path.join(tmpdir, 'mpdboot')
try:
if os.path.exists(fn):
os.remove(fn)
write_file(fn, "localhost ifhn=localhost")
except OSError, err:
raise EasyBuildError("Failed to create file %s: %s", fn, err)
params.update({'mpdbf': "--file=%s" % fn})
# create nodes file
fn = os.path.join(tmpdir, 'nodes')
try:
if os.path.exists(fn):
os.remove(fn)
write_file(fn, "localhost\n" * nr_ranks)
except OSError, err:
raise EasyBuildError("Failed to create file %s: %s", fn, err)
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
write_file(oldstyle_config_file, '')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
args = [
'--configfiles',
config_file,
'--debug',
'--buildpath',
testpath1,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [
os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')
]) # default
self.assertEqual(install_path(),
os.path.join(testpath2,
'software')) # via config file
# test with config file passed via environment variable
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath',
testpath2,
]
options = init_config(args=args)
self.assertEqual(install_path(),
os.path.join(os.getenv('HOME'), '.local', 'easybuild',
'software')) # default
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath',
testpath3,
]
options = init_config(args=args)
self.assertEqual(
source_paths(),
[testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(),
os.path.join(testpath3,
'software')) # via command line
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
class EasyConfigTest(TestCase):
"""Baseclass for easyconfig testcases."""
# initialize configuration (required for e.g. default modules_tool setting)
eb_go = eboptions.parse_options()
config.init(eb_go.options, eb_go.get_options_by_section('config'))
build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'force': True,
'optarch': 'test',
'robot_path': get_paths_for("easyconfigs")[0],
'silent': True,
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': config.module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
set_tmpdir()
del eb_go
# put dummy 'craype-test' module in place, which is required for parsing easyconfigs using Cray* toolchains
TMPDIR = tempfile.mkdtemp()
os.environ['MODULEPATH'] = TMPDIR
write_file(os.path.join(TMPDIR, 'craype-test'), '#%Module\n')
log = fancylogger.getLogger("EasyConfigTest", fname=False)
# make sure a logger is present for main
main._log = log
ordered_specs = None
parsed_easyconfigs = []
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not self.parsed_easyconfigs:
for spec in specs:
self.parsed_easyconfigs.extend(process_easyconfig(spec))
# filter out external modules
for ec in self.parsed_easyconfigs:
for dep in ec['dependencies'][:]:
if dep.get('external_module', False):
ec['dependencies'].remove(dep)
self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs,
modules_tool(),
retain_all_deps=True)
def test_dep_graph(self):
"""Unit test that builds a full dependency graph."""
# pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6
if LooseVersion(
sys.version) >= LooseVersion('2.6') and single_tests_ok:
# temporary file for dep graph
(hn, fn) = tempfile.mkstemp(suffix='.dot')
os.close(hn)
if self.ordered_specs is None:
self.process_all_easyconfigs()
dep_graph(fn, self.ordered_specs)
try:
os.remove(fn)
except OSError, err:
log.error("Failed to remove %s: %s" % (fn, err))
else:
def test_legacy_config_file(self):
"""Test finding/using legacy configuration files."""
self.purge_environment()
cfg_fn = self.configure(args=[])
self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py'))
configtxt = """
build_path = '%(buildpath)s'
source_path = '%(sourcepath)s'
install_path = '%(installpath)s'
repository_path = '%(repopath)s'
repository = FileRepository(repository_path)
log_format = ('%(logdir)s', '%(logtmpl)s')
log_dir = '%(tmplogdir)s'
software_install_suffix = '%(softsuffix)s'
modules_install_suffix = '%(modsuffix)s'
"""
buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'test', 'install',
'path')
repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path')
logdir = 'somedir'
logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware'
modsuffix = 'modulesgohere'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create user config file on default location
myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py')
if not os.path.exists(os.path.dirname(myconfigfile)):
os.makedirs(os.path.dirname(myconfigfile))
write_file(myconfigfile, configtxt % configdict)
# redefine home so we can test user config file on default location
home = os.environ.get('HOME', None)
os.environ['HOME'] = self.tmpdir
init_config()
cfg_fn = self.configure(args=[])
if home is not None:
os.environ['HOME'] = home
# check finding and use of config file
self.assertEqual(cfg_fn, myconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'),
os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
# redefine config file entries for proper testing below
buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build',
'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo',
'path')
logdir = 'somedir_custom'
logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'tmplogdir')
softsuffix = 'myfavoritesoftware_custom'
modsuffix = 'modulesgohere_custom'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create custom config file, and point to it
mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py')
if not os.path.exists(os.path.dirname(mycustomconfigfile)):
os.makedirs(os.path.dirname(mycustomconfigfile))
write_file(mycustomconfigfile, configtxt % configdict)
os.environ['EASYBUILDCONFIG'] = mycustomconfigfile
# reconfigure
init_config()
cfg_fn = self.configure(args=[])
# verify configuration
self.assertEqual(cfg_fn, mycustomconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'),
os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
