def test_find_library_path(self):
"""Test find_library_path function (Linux and Darwin only)."""
if get_os_type() == LINUX:
libname = 'libc.so.6'
elif get_os_type() == DARWIN:
libname = 'libSystem.dylib'
else:
libname = None
if libname:
lib_path = find_library_path(libname)
self.assertEqual(os.path.basename(lib_path), libname)
self.assertTrue(os.path.exists(lib_path),
"%s should exist" % libname)
def configure_step(self):
"""Custom configuration procedure for TINKER."""
# make sure FFTW is available
if get_software_root('FFTW') is None:
raise EasyBuildError("FFTW dependency is not available.")
os_dirs = {
LINUX: 'linux',
DARWIN: 'macosx',
}
os_type = get_os_type()
os_dir = os_dirs.get(os_type)
if os_dir is None:
raise EasyBuildError("Failed to determine OS directory for %s (known: %s)", os_type, os_dirs)
comp_dirs = {
toolchain.INTELCOMP: 'intel',
toolchain.GCC: 'gfortran',
}
comp_fam = self.toolchain.comp_family()
comp_dir = comp_dirs.get(comp_fam)
if comp_dir is None:
raise EasyBuildError("Failed to determine compiler directory for %s (known: %s)", comp_fam, comp_dirs)
self.build_subdir = os.path.join(os_dir, comp_dir)
self.log.info("Using build scripts from %s subdirectory" % self.build_subdir)
# patch 'link.make' script to use FFTW provided via EasyBuild
link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir, 'link.make')
for line in fileinput.input(link_make_fp, inplace=1, backup='.orig'):
line = re.sub(r"libfftw3_threads.a libfftw3.a", r"-L$EBROOTFFTW/lib -lfftw3_threads -lfftw3", line)
sys.stdout.write(line)
def configure_step(self):
"""Custom configuration procedure for TINKER."""
# make sure FFTW is available
if get_software_root('FFTW') is None:
raise EasyBuildError("FFTW dependency is not available.")
os_dirs = {
LINUX: 'linux',
DARWIN: 'macosx',
}
os_type = get_os_type()
os_dir = os_dirs.get(os_type)
if os_dir is None:
raise EasyBuildError("Failed to determine OS directory for %s (known: %s)", os_type, os_dirs)
comp_dirs = {
toolchain.INTELCOMP: 'intel',
toolchain.GCC: 'gfortran',
}
comp_fam = self.toolchain.comp_family()
comp_dir = comp_dirs.get(comp_fam)
if comp_dir is None:
raise EasyBuildError("Failed to determine compiler directory for %s (known: %s)", comp_fam, comp_dirs)
self.build_subdir = os.path.join(os_dir, comp_dir)
self.log.info("Using build scripts from %s subdirectory" % self.build_subdir)
# patch 'link.make' script to use FFTW provided via EasyBuild
link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir, 'link.make')
for line in fileinput.input(link_make_fp, inplace=1, backup='.orig'):
line = re.sub(r"libfftw3_threads.a libfftw3.a", r"-L$EBROOTFFTW/lib -lfftw3_threads -lfftw3", line)
sys.stdout.write(line)
def configure_step(self, cmd_prefix=''):
"""
Configure step
Lua does not need a configure step. In this step we just patch the
`luaconf.h` file in the sources to point to the correct Lua Root
"""
luaconf_h = os.path.join(self.start_dir, 'src', 'luaconf.h')
self.log.debug("Patching luaconf.h at %s", luaconf_h)
# note: make sure trailing slash is preserved!
apply_regex_substitutions(luaconf_h,
[(r'/usr/local/', '%s/' % self.installdir)])
os_type = get_os_type()
if os_type == LINUX:
self.cfg.update('buildopts', 'linux')
elif os_type == DARWIN:
self.cfg.update('buildopts', 'macosx')
else:
raise EasyBuildError(
"Don't know which make target to specify for OS type '%s'",
os_type)
# build with compatibility for earlier Lua versions
mycflags = []
if LooseVersion(self.version) > LooseVersion('5.1'):
mycflags.append('-DLUA_COMPAT_5_1')
if LooseVersion(self.version) > LooseVersion('5.2'):
mycflags.append('-DLUA_COMPAT_5_2')
if mycflags:
self.cfg.update('buildopts', 'MYCFLAGS="%s"' % ' '.join(mycflags))
self.cfg['runtest'] = 'test'
self.cfg.update('installopts', 'INSTALL_TOP=%s' % self.installdir)
def test_locate_solib(self):
"""Test locate_solib function (Linux only)."""
if get_os_type() == LINUX:
libname = 'libc.so.6'
libc_obj = None
try:
libc_obj = ctypes.cdll.LoadLibrary(libname)
except OSError:
pass
if libc_obj:
libc_path = locate_solib(libc_obj)
self.assertEqual(os.path.basename(libc_path), libname)
self.assertTrue(os.path.exists(libc_path), "%s should exist" % libname)
def configure_step(self):
"""Custom configuration procedure for TINKER."""
# make sure FFTW is available
if get_software_root('FFTW') is None:
raise EasyBuildError("FFTW dependency is not available.")
os_dirs = {
LINUX: 'linux',
DARWIN: 'macosx',
}
os_type = get_os_type()
os_dir = os_dirs.get(os_type)
if os_dir is None:
raise EasyBuildError(
"Failed to determine OS directory for %s (known: %s)", os_type,
os_dirs)
comp_dirs = {
toolchain.INTELCOMP: 'intel',
toolchain.GCC: 'gfortran',
}
comp_fam = self.toolchain.comp_family()
comp_dir = comp_dirs.get(comp_fam)
if comp_dir is None:
raise EasyBuildError(
"Failed to determine compiler directory for %s (known: %s)",
comp_fam, comp_dirs)
self.build_subdir = os.path.join(os_dir, comp_dir)
self.log.info("Using build scripts from %s subdirectory" %
self.build_subdir)
# patch 'link.make' script to use FFTW provided via EasyBuild
link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir,
'link.make')
regex_subs = [(r"libfftw3_threads.a libfftw3.a",
r"-L$EBROOTFFTW/lib -lfftw3_omp -lfftw3")]
apply_regex_substitutions(link_make_fp, regex_subs)
# patch *.make files to get rid of hardcoded -openmp flag,
# which doesn't work anymore with recent Intel compilers
if comp_fam == toolchain.INTELCOMP:
make_fps = glob.glob(
os.path.join(self.cfg['start_dir'], self.build_subdir,
'*.make'))
regex_subs = [(r'-openmp', r'-fopenmp')]
for make_fp in make_fps:
apply_regex_substitutions(make_fps, regex_subs)
def configure_step(self):
"""Custom configuration procedure for TINKER."""
# make sure FFTW is available
if get_software_root('FFTW') is None:
raise EasyBuildError("FFTW dependency is not available.")
os_dirs = {
LINUX: 'linux',
DARWIN: 'macosx',
}
os_type = get_os_type()
os_dir = os_dirs.get(os_type)
if os_dir is None:
raise EasyBuildError("Failed to determine OS directory for %s (known: %s)", os_type, os_dirs)
comp_dirs = {
toolchain.INTELCOMP: 'intel',
toolchain.GCC: 'gfortran',
}
comp_fam = self.toolchain.comp_family()
comp_dir = comp_dirs.get(comp_fam)
if comp_dir is None:
raise EasyBuildError("Failed to determine compiler directory for %s (known: %s)", comp_fam, comp_dirs)
self.build_subdir = os.path.join(os_dir, comp_dir)
self.log.info("Using build scripts from %s subdirectory" % self.build_subdir)
# patch 'link.make' script to use FFTW provided via EasyBuild
link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir, 'link.make')
regex_subs = [(r"libfftw3_threads.a libfftw3.a", r"-L$EBROOTFFTW/lib -lfftw3_threads -lfftw3")]
apply_regex_substitutions(link_make_fp, regex_subs)
# patch *.make files to get rid of hardcoded -openmp flag,
# which doesn't work anymore with recent Intel compilers
if comp_fam == toolchain.INTELCOMP:
make_fps = glob.glob(os.path.join(self.cfg['start_dir'], self.build_subdir, '*.make'))
regex_subs = [(r'-openmp', r'-fopenmp')]
for make_fp in make_fps:
apply_regex_substitutions(make_fps, regex_subs)
def prepare_rpath_wrappers(self, rpath_filter_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
self.log.experimental(
"Using wrapper scripts for compiler/linker commands that enforce RPATH linking"
)
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError(
"RPATH linking is currently only supported on Linux")
wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# prepend location to wrappers to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug(
"No general RPATH filter specified, falling back to default: %s",
rpath_filter)
rpath_filter = ','.join(rpath_filter +
['%s.*' % d for d in rpath_filter_dirs or []])
self.log.debug("Combined RPATH filter: '%s'" % rpath_filter)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info(
"%s already seems to be an RPATH wrapper script, not wrapping it again!",
orig_cmd)
continue
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(
orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError(
"Refusing the create a fork bomb, which(%s) == %s",
cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(
tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_wrapper_log': rpath_wrapper_log,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd,
which(cmd), rpath_wrapper_log)
else:
self.log.debug(
"Not installing RPATH wrapper for non-existing command '%s'",
cmd)
def test_os_type(self):
"""Test getting OS type."""
os_type = get_os_type()
self.assertTrue(os_type in [DARWIN, LINUX])
#
"""
Easyconfig constants module that provides all constants that can
be used within an Easyconfig file.
:author: Stijn De Weirdt (Ghent University)
:author: Kenneth Hoste (Ghent University)
"""
import os
import platform
from vsc.utils import fancylogger
from easybuild.tools.systemtools import get_shared_lib_ext, get_os_name, get_os_type, get_os_version
_log = fancylogger.getLogger('easyconfig.constants', fname=False)
EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE'
# constants that can be used in easyconfig
EASYCONFIG_CONSTANTS = {
'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"),
'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"),
'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),
'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"),
'OS_VERSION': (get_os_version(), "System version"),
'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"),
}
arch = 'aarch64'
if arch not in KNOWN_ARCH_CONSTANTS:
print_warning("Using unknown value for ARCH constant: %s", arch)
return arch
# constants that can be used in easyconfig
EASYCONFIG_CONSTANTS = {
'ARCH':
(_get_arch_constant(),
"CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"),
'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"),
'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"),
'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),
'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"),
'OS_VERSION': (get_os_version(), "System version"),
'SYS_PYTHON_VERSION':
(platform.python_version(),
"System Python version (platform.python_version())"),
'SYSTEM': ({
'name': 'system',
'version': 'system'
}, "System toolchain"),
'OS_PKG_IBVERBS_DEV':
(('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
"OS packages providing ibverbs/infiniband development support"),
'OS_PKG_OPENSSL_BIN': (('openssl'),
"OS packages providing the openSSL binary"),
'OS_PKG_OPENSSL_LIB': (('libssl', 'libopenssl'),
def sanity_check_step(self):
"""
Custom sanity check for GCC
"""
os_type = get_os_type()
sharedlib_ext = get_shared_lib_ext()
# determine "configuration name" directory, see https://sourceware.org/autobook/autobook/autobook_17.html
# this differs across GCC versions;
# x86_64-unknown-linux-gnu was common for old GCC versions,
# x86_64-pc-linux-gnu is more likely with an updated config.guess script;
# since this is internal to GCC, we don't really care how it is named exactly,
# we only care that it's actually there
# we may get multiple hits (libexec/, lib/), which is fine,
# but we expect the same configuration name subdirectory in each of them
glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc',
'*-linux-gnu', self.version)
matches = glob.glob(glob_pattern)
if matches:
cands = nub(
[os.path.basename(os.path.dirname(x)) for x in matches])
if len(cands) == 1:
config_name_subdir = cands[0]
else:
raise EasyBuildError(
"Found multiple candidates for configuration name: %s",
', '.join(cands))
else:
raise EasyBuildError(
"Failed to determine configuration name: no matches for '%s'",
glob_pattern)
bin_files = ["gcov"]
lib_files = []
if LooseVersion(self.version) >= LooseVersion('4.2'):
# libgomp was added in GCC 4.2.0
["libgomp.%s" % sharedlib_ext, "libgomp.a"]
if os_type == 'Linux':
lib_files.extend(["libgcc_s.%s" % sharedlib_ext])
# libmudflap is replaced by asan (see release notes gcc 4.9.0)
if LooseVersion(self.version) < LooseVersion("4.9.0"):
lib_files.extend(
["libmudflap.%s" % sharedlib_ext, "libmudflap.a"])
else:
lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"])
libexec_files = []
dirs = [os.path.join('lib', 'gcc', config_name_subdir, self.version)]
if not self.cfg['languages']:
# default languages are c, c++, fortran
bin_files = ["c++", "cpp", "g++", "gcc", "gcov", "gfortran"]
lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"])
libexec_files = ['cc1', 'cc1plus', 'collect2', 'f951']
if 'c' in self.cfg['languages']:
bin_files.extend(['cpp', 'gcc'])
if 'c++' in self.cfg['languages']:
bin_files.extend(['c++', 'g++'])
dirs.append('include/c++/%s' % self.version)
lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"])
if 'fortran' in self.cfg['languages']:
bin_files.append('gfortran')
lib_files.extend(
['libgfortran.%s' % sharedlib_ext, 'libgfortran.a'])
if self.cfg['withlto']:
libexec_files.extend(['lto1', 'lto-wrapper'])
if os_type in ['Linux']:
libexec_files.append('liblto_plugin.%s' % sharedlib_ext)
bin_files = ["bin/%s" % x for x in bin_files]
libdirs64 = ['lib64']
libdirs32 = ['lib', 'lib32']
libdirs = libdirs64 + libdirs32
if self.cfg['multilib']:
# with multilib enabled, both lib and lib64 should be there
lib_files64 = [
os.path.join(libdir, x) for libdir in libdirs64
for x in lib_files
]
lib_files32 = [
tuple([os.path.join(libdir, x) for libdir in libdirs32])
for x in lib_files
]
lib_files = lib_files64 + lib_files32
else:
# lib64 on SuSE and Darwin, lib otherwise
lib_files = [
tuple([os.path.join(libdir, x) for libdir in libdirs])
for x in lib_files
]
# lib on SuSE, libexec otherwise
libdirs = ['libexec', 'lib']
common_infix = os.path.join('gcc', config_name_subdir, self.version)
libexec_files = [
tuple([os.path.join(d, common_infix, x) for d in libdirs])
for x in libexec_files
]
old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()]
custom_paths = {
'files': bin_files + lib_files + libexec_files + old_cmds,
'dirs': dirs,
}
super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths)
def test_os_type(self):
"""Test getting OS type."""
os_type = get_os_type()
self.assertTrue(os_type in [DARWIN, LINUX])
def sanity_check_step(self):
"""
Custom sanity check for GCC
"""
os_type = get_os_type()
sharedlib_ext = get_shared_lib_ext()
common_infix = os.path.join('gcc', self.platform_lib, self.version)
bin_files = ["gcov"]
lib_files = []
if LooseVersion(self.version) >= LooseVersion('4.2'):
# libgomp was added in GCC 4.2.0
["libgomp.%s" % sharedlib_ext, "libgomp.a"]
if os_type == 'Linux':
lib_files.extend(["libgcc_s.%s" % sharedlib_ext])
# libmudflap is replaced by asan (see release notes gcc 4.9.0)
if LooseVersion(self.version) < LooseVersion("4.9.0"):
lib_files.extend(
["libmudflap.%s" % sharedlib_ext, "libmudflap.a"])
else:
lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"])
libexec_files = []
dirs = ['lib/%s' % common_infix]
if not self.cfg['languages']:
# default languages are c, c++, fortran
bin_files = ["c++", "cpp", "g++", "gcc", "gcov", "gfortran"]
lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"])
libexec_files = ['cc1', 'cc1plus', 'collect2', 'f951']
if 'c' in self.cfg['languages']:
bin_files.extend(['cpp', 'gcc'])
if 'c++' in self.cfg['languages']:
bin_files.extend(['c++', 'g++'])
dirs.append('include/c++/%s' % self.version)
lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"])
if 'fortran' in self.cfg['languages']:
bin_files.append('gfortran')
lib_files.extend(
['libgfortran.%s' % sharedlib_ext, 'libgfortran.a'])
if self.cfg['withlto']:
libexec_files.extend(['lto1', 'lto-wrapper'])
if os_type in ['Linux']:
libexec_files.append('liblto_plugin.%s' % sharedlib_ext)
bin_files = ["bin/%s" % x for x in bin_files]
libdirs64 = ['lib64']
libdirs32 = ['lib', 'lib32']
libdirs = libdirs64 + libdirs32
if self.cfg['multilib']:
# with multilib enabled, both lib and lib64 should be there
lib_files64 = [
os.path.join(libdir, x) for libdir in libdirs64
for x in lib_files
]
lib_files32 = [
tuple([os.path.join(libdir, x) for libdir in libdirs32])
for x in lib_files
]
lib_files = lib_files64 + lib_files32
else:
# lib64 on SuSE and Darwin, lib otherwise
lib_files = [
tuple([os.path.join(libdir, x) for libdir in libdirs])
for x in lib_files
]
# lib on SuSE, libexec otherwise
libdirs = ['libexec', 'lib']
libexec_files = [
tuple(
[os.path.join(libdir, common_infix, x) for libdir in libdirs])
for x in libexec_files
]
old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()]
custom_paths = {
'files': bin_files + lib_files + libexec_files + old_cmds,
'dirs': dirs,
}
super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths)
def prepare_rpath_wrappers(self, rpath_filter_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
self.log.experimental("Using wrapper scripts for compiler/linker commands that enforce RPATH linking")
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError("RPATH linking is currently only supported on Linux")
wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# prepend location to wrappers to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter)
rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs or []])
self.log.debug("Combined RPATH filter: '%s'" % rpath_filter)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd)
continue
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_wrapper_log': rpath_wrapper_log,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log)
else:
self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd)
def test_check_linked_shared_libs(self):
"""Test for check_linked_shared_libs function."""
txt_path = os.path.join(self.test_prefix, 'test.txt')
write_file(txt_path, "some text")
broken_symlink_path = os.path.join(self.test_prefix, 'broken_symlink')
symlink('/doesnotexist', broken_symlink_path, use_abspath_source=False)
# result is always None for anything other than dynamically linked binaries or shared libraries
self.assertEqual(check_linked_shared_libs(self.test_prefix), None)
self.assertEqual(check_linked_shared_libs(txt_path), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path), None)
bin_ls_path = which('ls')
os_type = get_os_type()
if os_type == LINUX:
out, _ = run_cmd("ldd %s" % bin_ls_path)
elif os_type == DARWIN:
out, _ = run_cmd("otool -L %s" % bin_ls_path)
else:
raise EasyBuildError("Unknown OS type: %s" % os_type)
shlib_ext = get_shared_lib_ext()
lib_path_regex = re.compile(
r'(?P<lib_path>[^\s]*/lib[^ ]+\.%s[^ ]*)' % shlib_ext, re.M)
lib_path = lib_path_regex.search(out).group(1)
test_pattern_named_args = [
# if no patterns are specified, result is always True
{},
{
'required_patterns': ['/lib', shlib_ext]
},
{
'banned_patterns': ['this_pattern_should_not_match']
},
{
'required_patterns': ['/lib', shlib_ext],
'banned_patterns': ['weirdstuff']
},
]
for pattern_named_args in test_pattern_named_args:
# result is always None for anything other than dynamically linked binaries or shared libraries
self.assertEqual(
check_linked_shared_libs(self.test_prefix,
**pattern_named_args), None)
self.assertEqual(
check_linked_shared_libs(txt_path, **pattern_named_args), None)
self.assertEqual(
check_linked_shared_libs(broken_symlink_path,
**pattern_named_args), None)
for path in (bin_ls_path, lib_path):
error_msg = "Check on linked libs should pass for %s with %s" % (
path, pattern_named_args)
self.assertTrue(
check_linked_shared_libs(path, **pattern_named_args),
error_msg)
# also test with input that should result in failing check
test_pattern_named_args = [
{
'required_patterns': ['this_pattern_will_not_match']
},
{
'banned_patterns': ['/lib']
},
{
'required_patterns': ['weirdstuff'],
'banned_patterns': ['/lib', shlib_ext]
},
]
for pattern_named_args in test_pattern_named_args:
# result is always None for anything other than dynamically linked binaries or shared libraries
self.assertEqual(
check_linked_shared_libs(self.test_prefix,
**pattern_named_args), None)
self.assertEqual(
check_linked_shared_libs(txt_path, **pattern_named_args), None)
self.assertEqual(
check_linked_shared_libs(broken_symlink_path,
**pattern_named_args), None)
for path in (bin_ls_path, lib_path):
error_msg = "Check on linked libs should fail for %s with %s" % (
path, pattern_named_args)
self.assertFalse(
check_linked_shared_libs(path, **pattern_named_args),
error_msg)
def __init__(self, *args, **kwargs):
"""Locate the installation files of OpenSSL in the host system"""
super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs)
# Libraries packaged in OpenSSL
openssl_libs = ['libssl', 'libcrypto']
# list of relevant library extensions per system and version of OpenSSL
# the first item should be the extension of an installation from source,
# it will be used in the sanity checks of the component
openssl_libext = {
'1.0': {
LINUX: ('so.1.0.0', 'so.10'),
DARWIN: ('1.0.dylib', ),
},
'1.1': {
LINUX: ('so.1.1', ),
DARWIN: ('1.1.dylib', ),
},
}
os_type = get_os_type()
if self.version in openssl_libext and os_type in openssl_libext[
self.version]:
# generate matrix of versioned .so filenames
system_versioned_libs = [[
'%s.%s' % (lib, ext)
for ext in openssl_libext[self.version][os_type]
] for lib in openssl_libs]
self.log.info("Matrix of version library names: %s",
system_versioned_libs)
else:
raise EasyBuildError(
"Don't know name of OpenSSL system library for version %s and OS type %s",
self.version, os_type)
# by default target the first option of each OpenSSL library,
# which corresponds to installation from source
self.target_ssl_libs = [
lib_name[0] for lib_name in system_versioned_libs
]
self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs)
# folders containing engines libraries
openssl_engines = {
'1.0': 'engines',
'1.1': 'engines-1.1',
}
self.target_ssl_engine = openssl_engines[self.version]
# Paths to system libraries and headers of OpenSSL
self.system_ssl = {
'bin': None,
'engines': None,
'include': None,
'lib': None,
}
# early return when we're not wrapping the system OpenSSL installation
if not self.cfg.get('wrap_system_openssl'):
self.log.info("Not wrapping system OpenSSL installation!")
return
# Check the system libraries of OpenSSL
# (only needs first library in openssl_libs to find path to libraries)
for idx, libssl in enumerate(system_versioned_libs[0]):
self.system_ssl['lib'] = find_library_path(libssl)
if self.system_ssl['lib']:
# change target libraries to the ones found in the system
self.target_ssl_libs = [
lib_name[idx] for lib_name in system_versioned_libs
]
self.log.info("Target system OpenSSL libraries: %s",
self.target_ssl_libs)
break
if self.system_ssl['lib']:
self.log.info("Found library '%s' in: %s", openssl_libs[0],
self.system_ssl['lib'])
else:
self.log.info(
"OpenSSL library '%s' not found, falling back to OpenSSL in EasyBuild",
openssl_libs[0])
# Directory with engine libraries
if self.system_ssl['lib']:
lib_dir = os.path.dirname(self.system_ssl['lib'])
lib_engines_dir = [
os.path.join(lib_dir, 'openssl', self.target_ssl_engine),
os.path.join(lib_dir, self.target_ssl_engine),
]
for engines_path in lib_engines_dir:
if os.path.isdir(engines_path):
self.system_ssl['engines'] = engines_path
self.log.debug("Found OpenSSL engines in: %s",
self.system_ssl['engines'])
break
if not self.system_ssl['engines']:
self.system_ssl['lib'] = None
self.log.info(
"OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild"
)
# Check system include paths for OpenSSL headers
cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null"
(out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False)
sys_include_dirs = []
for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE):
sys_include_dirs.extend(match.groups())
self.log.debug(
"Found the following include directories in host system: %s",
', '.join(sys_include_dirs))
# headers are located in 'include/openssl' by default
ssl_include_subdirs = [self.name.lower()]
if self.version == '1.1':
# but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7
# prefer 'include/openssl' as long as the version of headers matches
ssl_include_subdirs.append(
os.path.join('openssl11', self.name.lower()))
ssl_include_dirs = [
os.path.join(incd, subd) for incd in sys_include_dirs
for subd in ssl_include_subdirs
]
ssl_include_dirs = [
include for include in ssl_include_dirs if os.path.isdir(include)
]
# find location of header files, verify that the headers match our OpenSSL version
openssl_version_regex = re.compile(
r"SHLIB_VERSION_NUMBER\s\"([0-9]+\.[0-9]+)", re.M)
for include_dir in ssl_include_dirs:
opensslv_path = os.path.join(include_dir, 'opensslv.h')
self.log.debug("Checking OpenSSL version in %s...", opensslv_path)
if os.path.exists(opensslv_path):
opensslv = read_file(opensslv_path)
header_majmin_version = openssl_version_regex.search(opensslv)
if header_majmin_version:
header_majmin_version = header_majmin_version.group(1)
if re.match('^' + header_majmin_version, self.version):
self.system_ssl['include'] = include_dir
self.log.info(
"Found OpenSSL headers in host system: %s",
self.system_ssl['include'])
break
else:
self.log.debug(
"Header major/minor version '%s' doesn't match with %s",
header_majmin_version, self.version)
else:
self.log.debug("Pattern '%s' not found in %s",
openssl_version_regex.pattern,
opensslv_path)
else:
self.log.info("OpenSSL header file %s not found",
opensslv_path)
if not self.system_ssl['include']:
self.log.info(
"OpenSSL headers not found in host system, falling back to OpenSSL in EasyBuild"
)
# Check system OpenSSL binary
if self.version == '1.1':
# prefer 'openssl11' over 'openssl' with v1.1
self.system_ssl['bin'] = which('openssl11')
if not self.system_ssl['bin']:
self.system_ssl['bin'] = which(self.name.lower())
if self.system_ssl['bin']:
self.log.info("System OpenSSL binary found: %s",
self.system_ssl['bin'])
else:
self.log.info("System OpenSSL binary not found!")
def __init__(self, *args, **kwargs):
"""Locate the installation files of OpenSSL in the host system"""
super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs)
# Wrapper should have at least a major minor version numbers
try:
subversions = self.version.split('.')
self.majmin_version = '%s.%s' % (subversions[0], subversions[1])
except (AttributeError, IndexError):
err_msg = "Wrapper OpenSSL version does not have any subversion: %s"
raise EasyBuildError(err_msg, self.version)
# Set minimum OpenSSL version
min_openssl_version = self.cfg.get('minimum_openssl_version')
if not min_openssl_version:
min_openssl_version = self.version
elif not isinstance(min_openssl_version, string_type):
min_openssl_version = str(min_openssl_version)
# Minimum OpenSSL version can only increase depth of wrapper version
if min_openssl_version.startswith(self.version):
self.log.debug("Requiring minimum OpenSSL version: %s",
min_openssl_version)
else:
err_msg = "Requested minimum OpenSSL version '%s' does not fit in wrapper easyconfig version '%s'"
raise EasyBuildError(err_msg, min_openssl_version, self.version)
# Regex pattern to find version strings in OpenSSL libraries and headers
full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?')
openssl_version_regex = re.compile(
r'OpenSSL\s+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M)
# Libraries packaged in OpenSSL
openssl_libs = ['libssl', 'libcrypto']
# list of relevant library extensions per system and version of OpenSSL
# the first item should be the extension of an installation from source,
# it will be used in the sanity checks of the component
openssl_libext = {
'1.0': {
LINUX: ('so.1.0.0', 'so.10'),
DARWIN: ('1.0.dylib', ),
},
'1.1': {
LINUX: ('so.1.1', ),
DARWIN: ('1.1.dylib', ),
},
'3.0': {
LINUX: ('so.3', ),
DARWIN: ('3.dylib', ),
},
}
os_type = get_os_type()
if self.majmin_version in openssl_libext and os_type in openssl_libext[
self.majmin_version]:
# generate matrix of versioned .so filenames
system_versioned_libs = [[
'%s.%s' % (lib, ext) for lib in openssl_libs
] for ext in openssl_libext[self.majmin_version][os_type]]
self.log.info("Matrix of version library names: %s",
system_versioned_libs)
else:
err_msg = "Don't know name of OpenSSL system library for version %s and OS type %s"
raise EasyBuildError(err_msg, self.majmin_version, os_type)
# by default target the first option of each OpenSSL library,
# which corresponds to installation from source
self.target_ssl_libs = system_versioned_libs[0]
self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs)
# folders containing engines libraries
openssl_engines = {
'1.0': 'engines',
'1.1': 'engines-1.1',
'3.0': 'engines-3',
}
self.target_ssl_engine = openssl_engines[self.majmin_version]
# Paths to system libraries and headers of OpenSSL
self.system_ssl = {
'bin': None,
'engines': None,
'include': None,
'libs': [],
}
# early return when we're not wrapping the system OpenSSL installation
if not self.cfg.get('wrap_system_openssl'):
self.log.info(
"Not wrapping system OpenSSL installation by user request")
return
# Check the system libraries of OpenSSL
# Find library file and compare its version string
for idx, solibs in enumerate(system_versioned_libs):
for solib in solibs:
system_solib = find_library_path(solib)
if system_solib:
openssl_version = 0
# get version of system library filename
try:
openssl_version = full_version_regex.search(
os.path.realpath(system_solib)).group(0)
except AttributeError:
# filename lacks the full version, fallback to version strings within the library
solib_strings = read_file(system_solib,
mode="rb").decode(
'utf-8', 'replace')
try:
openssl_version = openssl_version_regex.search(
solib_strings).group(1)
except AttributeError:
dbg_msg = "Could not detect the full version of system OpenSSL library: %s"
self.log.debug(dbg_msg, system_solib)
# check that system version fulfills requirements
if LooseVersion(openssl_version) >= LooseVersion(
min_openssl_version):
dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s"
self.log.debug(dbg_msg, system_solib, openssl_version,
min_openssl_version)
self.system_ssl['libs'].append(system_solib)
else:
dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s"
self.log.debug(dbg_msg, system_solib, openssl_version,
min_openssl_version)
else:
# one of the OpenSSL libraries is missing, switch to next group of versioned libs
self.system_ssl['libs'] = []
break
if len(self.system_ssl['libs']) == len(openssl_libs):
# keep these libraries as possible targets for this installation
target_system_ssl_libs = system_versioned_libs[idx]
break
if len(self.system_ssl['libs']) == len(openssl_libs):
self.system_ssl['version'] = openssl_version
info_msg = "Found OpenSSL library version %s in host system: %s"
self.log.info(info_msg, self.system_ssl['version'],
os.path.dirname(self.system_ssl['libs'][0]))
else:
self.log.info(
"OpenSSL library not found in host system, falling back to OpenSSL in EasyBuild"
)
return
# Directory with engine libraries
lib_dir = os.path.dirname(self.system_ssl['libs'][0])
lib_engines_dir = [
os.path.join(lib_dir, 'openssl', self.target_ssl_engine),
os.path.join(lib_dir, self.target_ssl_engine),
]
for engines_path in lib_engines_dir:
if os.path.isdir(engines_path):
self.system_ssl['engines'] = engines_path
self.log.debug("Found OpenSSL engines in: %s",
self.system_ssl['engines'])
break
if not self.system_ssl['engines']:
self.log.info(
"OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild"
)
return
# Check system include paths for OpenSSL headers
cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null"
(out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False)
sys_include_dirs = []
for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE):
sys_include_dirs.extend(match.groups())
self.log.debug(
"Found the following include directories in host system: %s",
', '.join(sys_include_dirs))
# headers are located in 'include/openssl' by default
ssl_include_subdirs = ['openssl']
if self.majmin_version == '1.1':
# but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7
# prefer 'include/openssl' as long as the version of headers matches
ssl_include_subdirs.append(
os.path.join('openssl11', self.name.lower()))
ssl_include_dirs = [
os.path.join(incd, subd) for incd in sys_include_dirs
for subd in ssl_include_subdirs
]
ssl_include_dirs = [
include for include in ssl_include_dirs if os.path.isdir(include)
]
# find location of header files for this version of the OpenSSL libraries
for include_dir in ssl_include_dirs:
opensslv_path = os.path.join(include_dir, 'opensslv.h')
self.log.debug("Checking OpenSSL version in %s...", opensslv_path)
if os.path.exists(opensslv_path):
# check version reported by opensslv.h
opensslv = read_file(opensslv_path)
try:
header_version = openssl_version_regex.search(
opensslv).group(1)
except AttributeError:
err_msg = "System OpenSSL header '%s' does not contain any recognizable version string"
raise EasyBuildError(err_msg, opensslv_path)
if header_version == self.system_ssl['version']:
self.system_ssl['include'] = include_dir
info_msg = "Found OpenSSL headers v%s in host system: %s"
self.log.info(info_msg, header_version,
self.system_ssl['include'])
break
else:
dbg_msg = "System OpenSSL header version '%s' doesn not match library version '%s'"
self.log.debug(dbg_msg, header_version,
self.system_ssl['version'])
else:
self.log.info("System OpenSSL header file %s not found",
opensslv_path)
if not self.system_ssl['include']:
err_msg = (
"OpenSSL v%s headers not found in host system, but libraries for v%s are present. "
"Install the development package of OpenSSL for your system or force building OpenSSL from "
"source in EasyBuild by setting 'wrap_system_openssl = False' in the OpenSSL easyconfig."
)
raise EasyBuildError(err_msg, self.version,
self.system_ssl['version'])
# Check system OpenSSL binary
if self.majmin_version == '1.1':
# prefer 'openssl11' over 'openssl' with v1.1
self.system_ssl['bin'] = which('openssl11')
if not self.system_ssl['bin']:
self.system_ssl['bin'] = which('openssl')
if self.system_ssl['bin']:
self.log.info("System OpenSSL binary found: %s",
self.system_ssl['bin'])
else:
self.log.info("System OpenSSL binary not found!")
return
# system OpenSSL is fine, change target libraries to the ones found in it
self.target_ssl_libs = target_system_ssl_libs
self.log.info("Target system OpenSSL libraries: %s",
self.target_ssl_libs)
def test_check_linked_shared_libs(self):
"""Test for check_linked_shared_libs function."""
txt_path = os.path.join(self.test_prefix, 'test.txt')
write_file(txt_path, "some text")
broken_symlink_path = os.path.join(self.test_prefix, 'broken_symlink')
symlink('/doesnotexist', broken_symlink_path, use_abspath_source=False)
# result is always None for anything other than dynamically linked binaries or shared libraries
self.assertEqual(check_linked_shared_libs(self.test_prefix), None)
self.assertEqual(check_linked_shared_libs(txt_path), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path), None)
bin_ls_path = which('ls')
os_type = get_os_type()
if os_type == LINUX:
out, _ = run_cmd("ldd %s" % bin_ls_path)
elif os_type == DARWIN:
out, _ = run_cmd("otool -L %s" % bin_ls_path)
else:
raise EasyBuildError("Unknown OS type: %s" % os_type)
shlib_ext = get_shared_lib_ext()
lib_path_regex = re.compile(r'(?P<lib_path>[^\s]*/lib[^ ]+\.%s[^ ]*)' % shlib_ext, re.M)
lib_path = lib_path_regex.search(out).group(1)
test_pattern_named_args = [
# if no patterns are specified, result is always True
{},
{'required_patterns': ['/lib', shlib_ext]},
{'banned_patterns': ['this_pattern_should_not_match']},
{'required_patterns': ['/lib', shlib_ext], 'banned_patterns': ['weirdstuff']},
]
for pattern_named_args in test_pattern_named_args:
# result is always None for anything other than dynamically linked binaries or shared libraries
self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None)
for path in (bin_ls_path, lib_path):
# path may not exist, especially for library paths obtained via 'otool -L' on macOS
if os.path.exists(path):
error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args)
self.assertTrue(check_linked_shared_libs(path, **pattern_named_args), error_msg)
# also test with input that should result in failing check
test_pattern_named_args = [
{'required_patterns': ['this_pattern_will_not_match']},
{'banned_patterns': ['/lib']},
{'required_patterns': ['weirdstuff'], 'banned_patterns': ['/lib', shlib_ext]},
]
for pattern_named_args in test_pattern_named_args:
# result is always None for anything other than dynamically linked binaries or shared libraries
self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None)
for path in (bin_ls_path, lib_path):
error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args)
self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg)
if get_os_type() == LINUX:
# inject fake 'file' command which always reports that the file is "dynamically linked"
file_cmd = os.path.join(self.test_prefix, 'bin', 'file')
write_file(file_cmd, "echo '(dynamically linked)'")
adjust_permissions(file_cmd, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.join(self.test_prefix, 'bin') + ':' + os.getenv('PATH')
test_file = os.path.join(self.test_prefix, 'test.txt')
write_file(test_file, 'test')
warning_regex = re.compile(r"WARNING: Determining linked libraries.* via 'ldd .*/test.txt' failed!", re.M)
self.mock_stderr(True)
self.mock_stdout(True)
res = check_linked_shared_libs(test_file, banned_patterns=['/lib'])
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)
fail_msg = "Pattern '%s' should be found in: %s" % (warning_regex.pattern, stderr)
self.assertTrue(warning_regex.search(stderr), fail_msg)
self.assertFalse(stdout)
self.assertEqual(res, None)
def sanity_check_step(self):
"""
Custom sanity check for GCC
"""
os_type = get_os_type()
sharedlib_ext = get_shared_lib_ext()
# determine "configuration name" directory, see https://sourceware.org/autobook/autobook/autobook_17.html
# this differs across GCC versions;
# x86_64-unknown-linux-gnu was common for old GCC versions,
# x86_64-pc-linux-gnu is more likely with an updated config.guess script;
# since this is internal to GCC, we don't really care how it is named exactly,
# we only care that it's actually there
# we may get multiple hits (libexec/, lib/), which is fine,
# but we expect the same configuration name subdirectory in each of them
glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc',
'*-linux-gnu', self.version)
matches = glob.glob(glob_pattern)
if matches:
cands = nub(
[os.path.basename(os.path.dirname(x)) for x in matches])
if len(cands) == 1:
config_name_subdir = cands[0]
else:
raise EasyBuildError(
"Found multiple candidates for configuration name: %s",
', '.join(cands))
else:
raise EasyBuildError(
"Failed to determine configuration name: no matches for '%s'",
glob_pattern)
bin_files = ["gcov"]
lib_files = []
if LooseVersion(self.version) >= LooseVersion('4.2'):
# libgomp was added in GCC 4.2.0
["libgomp.%s" % sharedlib_ext, "libgomp.a"]
if os_type == 'Linux':
lib_files.extend(["libgcc_s.%s" % sharedlib_ext])
# libmudflap is replaced by asan (see release notes gcc 4.9.0)
if LooseVersion(self.version) < LooseVersion("4.9.0"):
lib_files.extend(
["libmudflap.%s" % sharedlib_ext, "libmudflap.a"])
else:
lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"])
libexec_files = []
dirs = [os.path.join('lib', 'gcc', config_name_subdir, self.version)]
languages = self.cfg['languages'] or ['c', 'c++', 'fortran'
] # default languages
if 'c' in languages:
bin_files.extend(['cpp', 'gcc'])
libexec_files.extend(['cc1', 'collect2'])
if 'c++' in languages:
bin_files.extend(['c++', 'g++'])
dirs.append('include/c++/%s' % self.version)
lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"])
libexec_files.append(
'cc1plus') # c++ requires c, so collect2 not mentioned again
if 'fortran' in languages:
bin_files.append('gfortran')
lib_files.extend(
['libgfortran.%s' % sharedlib_ext, 'libgfortran.a'])
libexec_files.append('f951')
if self.cfg['withlto']:
libexec_files.extend(['lto1', 'lto-wrapper'])
if os_type in ['Linux']:
libexec_files.append('liblto_plugin.%s' % sharedlib_ext)
if self.cfg['withnvptx']:
bin_files.extend(['nvptx-none-as', 'nvptx-none-ld'])
lib_files.append('libgomp-plugin-nvptx.%s' % sharedlib_ext)
bin_files = ["bin/%s" % x for x in bin_files]
libdirs64 = ['lib64']
libdirs32 = ['lib', 'lib32']
libdirs = libdirs64 + libdirs32
if self.cfg['multilib']:
# with multilib enabled, both lib and lib64 should be there
lib_files64 = [
os.path.join(libdir, x) for libdir in libdirs64
for x in lib_files
]
lib_files32 = [
tuple([os.path.join(libdir, x) for libdir in libdirs32])
for x in lib_files
]
lib_files = lib_files64 + lib_files32
else:
# lib64 on SuSE and Darwin, lib otherwise
lib_files = [
tuple([os.path.join(libdir, x) for libdir in libdirs])
for x in lib_files
]
# lib on SuSE, libexec otherwise
libdirs = ['libexec', 'lib']
common_infix = os.path.join('gcc', config_name_subdir, self.version)
libexec_files = [
tuple([os.path.join(d, common_infix, x) for d in libdirs])
for x in libexec_files
]
old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()]
custom_paths = {
'files': bin_files + lib_files + libexec_files + old_cmds,
'dirs': dirs,
}
custom_commands = []
for lang, compiler in (('c', 'gcc'), ('c++', 'g++')):
if lang in languages:
# Simple test compile
cmd = 'echo "int main(){} " | %s -x %s -o/dev/null -'
compiler_path = os.path.join(self.installdir, 'bin', compiler)
custom_commands.append(cmd % (compiler_path, lang))
if self.cfg['withlto']:
custom_commands.append(
cmd %
(compiler_path, lang + ' -flto -fuse-linker-plugin'))
if custom_commands:
# Load binutils to do the compile tests
extra_modules = [
d['short_mod_name'] for d in self.cfg.dependencies()
if d['name'] == 'binutils'
]
else:
extra_modules = None
super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths,
custom_commands=custom_commands,
extra_modules=extra_modules)
def prepare_rpath_wrappers(self,
rpath_filter_dirs=None,
rpath_include_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError(
"RPATH linking is currently only supported on Linux")
if rpath_filter_dirs is None:
rpath_filter_dirs = []
# always include filter for 'stubs' library directory,
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
rpath_filter_dirs.append('.*/lib(64)?/stubs/?')
# directory where all wrappers will be placed
wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug(
"No general RPATH filter specified, falling back to default: %s",
rpath_filter)
rpath_filter = ','.join(rpath_filter +
['%s.*' % d for d in rpath_filter_dirs])
self.log.debug("Combined RPATH filter: '%s'", rpath_filter)
rpath_include = ','.join(rpath_include_dirs or [])
self.log.debug("Combined RPATH include paths: '%s'", rpath_include)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info(
"%s already seems to be an RPATH wrapper script, not wrapping it again!",
orig_cmd)
continue
# determine location for this wrapper
# each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately
# avoid '+' character in directory name (for example with 'g++' command), which can cause trouble
# (see https://github.com/easybuilders/easybuild-easyconfigs/issues/7339)
wrapper_dir_name = '%s_wrapper' % cmd.replace('+', 'x')
wrapper_dir = os.path.join(wrappers_dir, wrapper_dir_name)
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(
orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError(
"Refusing the create a fork bomb, which(%s) == %s",
cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(
tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_include': rpath_include,
'rpath_wrapper_log': rpath_wrapper_log,
'wrapper_dir': wrapper_dir,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd,
which(cmd), rpath_wrapper_log)
# prepend location to this wrapper to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
else:
self.log.debug(
"Not installing RPATH wrapper for non-existing command '%s'",
cmd)
def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError("RPATH linking is currently only supported on Linux")
if rpath_filter_dirs is None:
rpath_filter_dirs = []
# always include filter for 'stubs' library directory,
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
rpath_filter_dirs.append('.*/lib(64)?/stubs/?')
# directory where all wrappers will be placed
wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter)
rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs])
self.log.debug("Combined RPATH filter: '%s'", rpath_filter)
rpath_include = ','.join(rpath_include_dirs or [])
self.log.debug("Combined RPATH include paths: '%s'", rpath_include)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd)
continue
# determine location for this wrapper
# each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately
# avoid '+' character in directory name (for example with 'g++' command), which can cause trouble
# (see https://github.com/easybuilders/easybuild-easyconfigs/issues/7339)
wrapper_dir_name = '%s_wrapper' % cmd.replace('+', 'x')
wrapper_dir = os.path.join(wrappers_dir, wrapper_dir_name)
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_include': rpath_include,
'rpath_wrapper_log': rpath_wrapper_log,
'wrapper_dir': wrapper_dir,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log)
# prepend location to this wrapper to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
else:
self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd)
def sanity_check_step(self):
"""
Custom sanity check for GCC
"""
os_type = get_os_type()
sharedlib_ext = get_shared_lib_ext()
common_infix = os.path.join('gcc', self.platform_lib, self.version)
bin_files = ["gcov"]
lib_files = []
if LooseVersion(self.version) >= LooseVersion('4.2'):
# libgomp was added in GCC 4.2.0
["libgomp.%s" % sharedlib_ext, "libgomp.a"]
if os_type == 'Linux':
lib_files.extend(["libgcc_s.%s" % sharedlib_ext])
# libmudflap is replaced by asan (see release notes gcc 4.9.0)
if LooseVersion(self.version) < LooseVersion("4.9.0"):
lib_files.extend(["libmudflap.%s" % sharedlib_ext, "libmudflap.a"])
else:
lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"])
libexec_files = []
dirs = ['lib/%s' % common_infix]
if not self.cfg['languages']:
# default languages are c, c++, fortran
bin_files = ["c++", "cpp", "g++", "gcc", "gcov", "gfortran"]
lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"])
libexec_files = ['cc1', 'cc1plus', 'collect2', 'f951']
if 'c' in self.cfg['languages']:
bin_files.extend(['cpp', 'gcc'])
if 'c++' in self.cfg['languages']:
bin_files.extend(['c++', 'g++'])
dirs.append('include/c++/%s' % self.version)
lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"])
if 'fortran' in self.cfg['languages']:
bin_files.append('gfortran')
lib_files.extend(['libgfortran.%s' % sharedlib_ext, 'libgfortran.a'])
if self.cfg['withlto']:
libexec_files.extend(['lto1', 'lto-wrapper'])
if os_type in ['Linux']:
libexec_files.append('liblto_plugin.%s' % sharedlib_ext)
bin_files = ["bin/%s" % x for x in bin_files]
libdirs64 = ['lib64']
libdirs32 = ['lib', 'lib32']
libdirs = libdirs64 + libdirs32
if self.cfg['multilib']:
# with multilib enabled, both lib and lib64 should be there
lib_files64 = [os.path.join(libdir, x) for libdir in libdirs64 for x in lib_files]
lib_files32 = [tuple([os.path.join(libdir, x) for libdir in libdirs32]) for x in lib_files]
lib_files = lib_files64 + lib_files32
else:
# lib64 on SuSE and Darwin, lib otherwise
lib_files = [tuple([os.path.join(libdir, x) for libdir in libdirs]) for x in lib_files]
# lib on SuSE, libexec otherwise
libdirs = ['libexec', 'lib']
libexec_files = [tuple([os.path.join(libdir, common_infix, x) for libdir in libdirs]) for x in libexec_files]
old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()]
custom_paths = {
'files': bin_files + lib_files + libexec_files + old_cmds,
'dirs': dirs,
}
super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths)
