def test_find_library_path(self): """Test find_library_path function (Linux and Darwin only).""" if get_os_type() == LINUX: libname = 'libc.so.6' elif get_os_type() == DARWIN: libname = 'libSystem.dylib' else: libname = None if libname: lib_path = find_library_path(libname) self.assertEqual(os.path.basename(lib_path), libname) self.assertTrue(os.path.exists(lib_path), "%s should exist" % libname)
def configure_step(self): """Custom configuration procedure for TINKER.""" # make sure FFTW is available if get_software_root('FFTW') is None: raise EasyBuildError("FFTW dependency is not available.") os_dirs = { LINUX: 'linux', DARWIN: 'macosx', } os_type = get_os_type() os_dir = os_dirs.get(os_type) if os_dir is None: raise EasyBuildError("Failed to determine OS directory for %s (known: %s)", os_type, os_dirs) comp_dirs = { toolchain.INTELCOMP: 'intel', toolchain.GCC: 'gfortran', } comp_fam = self.toolchain.comp_family() comp_dir = comp_dirs.get(comp_fam) if comp_dir is None: raise EasyBuildError("Failed to determine compiler directory for %s (known: %s)", comp_fam, comp_dirs) self.build_subdir = os.path.join(os_dir, comp_dir) self.log.info("Using build scripts from %s subdirectory" % self.build_subdir) # patch 'link.make' script to use FFTW provided via EasyBuild link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir, 'link.make') for line in fileinput.input(link_make_fp, inplace=1, backup='.orig'): line = re.sub(r"libfftw3_threads.a libfftw3.a", r"-L$EBROOTFFTW/lib -lfftw3_threads -lfftw3", line) sys.stdout.write(line)
def configure_step(self): """Custom configuration procedure for TINKER.""" # make sure FFTW is available if get_software_root('FFTW') is None: raise EasyBuildError("FFTW dependency is not available.") os_dirs = { LINUX: 'linux', DARWIN: 'macosx', } os_type = get_os_type() os_dir = os_dirs.get(os_type) if os_dir is None: raise EasyBuildError("Failed to determine OS directory for %s (known: %s)", os_type, os_dirs) comp_dirs = { toolchain.INTELCOMP: 'intel', toolchain.GCC: 'gfortran', } comp_fam = self.toolchain.comp_family() comp_dir = comp_dirs.get(comp_fam) if comp_dir is None: raise EasyBuildError("Failed to determine compiler directory for %s (known: %s)", comp_fam, comp_dirs) self.build_subdir = os.path.join(os_dir, comp_dir) self.log.info("Using build scripts from %s subdirectory" % self.build_subdir) # patch 'link.make' script to use FFTW provided via EasyBuild link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir, 'link.make') for line in fileinput.input(link_make_fp, inplace=1, backup='.orig'): line = re.sub(r"libfftw3_threads.a libfftw3.a", r"-L$EBROOTFFTW/lib -lfftw3_threads -lfftw3", line) sys.stdout.write(line)
def configure_step(self, cmd_prefix=''): """ Configure step Lua does not need a configure step. In this step we just patch the `luaconf.h` file in the sources to point to the correct Lua Root """ luaconf_h = os.path.join(self.start_dir, 'src', 'luaconf.h') self.log.debug("Patching luaconf.h at %s", luaconf_h) # note: make sure trailing slash is preserved! apply_regex_substitutions(luaconf_h, [(r'/usr/local/', '%s/' % self.installdir)]) os_type = get_os_type() if os_type == LINUX: self.cfg.update('buildopts', 'linux') elif os_type == DARWIN: self.cfg.update('buildopts', 'macosx') else: raise EasyBuildError( "Don't know which make target to specify for OS type '%s'", os_type) # build with compatibility for earlier Lua versions mycflags = [] if LooseVersion(self.version) > LooseVersion('5.1'): mycflags.append('-DLUA_COMPAT_5_1') if LooseVersion(self.version) > LooseVersion('5.2'): mycflags.append('-DLUA_COMPAT_5_2') if mycflags: self.cfg.update('buildopts', 'MYCFLAGS="%s"' % ' '.join(mycflags)) self.cfg['runtest'] = 'test' self.cfg.update('installopts', 'INSTALL_TOP=%s' % self.installdir)
def test_locate_solib(self): """Test locate_solib function (Linux only).""" if get_os_type() == LINUX: libname = 'libc.so.6' libc_obj = None try: libc_obj = ctypes.cdll.LoadLibrary(libname) except OSError: pass if libc_obj: libc_path = locate_solib(libc_obj) self.assertEqual(os.path.basename(libc_path), libname) self.assertTrue(os.path.exists(libc_path), "%s should exist" % libname)
def configure_step(self): """Custom configuration procedure for TINKER.""" # make sure FFTW is available if get_software_root('FFTW') is None: raise EasyBuildError("FFTW dependency is not available.") os_dirs = { LINUX: 'linux', DARWIN: 'macosx', } os_type = get_os_type() os_dir = os_dirs.get(os_type) if os_dir is None: raise EasyBuildError( "Failed to determine OS directory for %s (known: %s)", os_type, os_dirs) comp_dirs = { toolchain.INTELCOMP: 'intel', toolchain.GCC: 'gfortran', } comp_fam = self.toolchain.comp_family() comp_dir = comp_dirs.get(comp_fam) if comp_dir is None: raise EasyBuildError( "Failed to determine compiler directory for %s (known: %s)", comp_fam, comp_dirs) self.build_subdir = os.path.join(os_dir, comp_dir) self.log.info("Using build scripts from %s subdirectory" % self.build_subdir) # patch 'link.make' script to use FFTW provided via EasyBuild link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir, 'link.make') regex_subs = [(r"libfftw3_threads.a libfftw3.a", r"-L$EBROOTFFTW/lib -lfftw3_omp -lfftw3")] apply_regex_substitutions(link_make_fp, regex_subs) # patch *.make files to get rid of hardcoded -openmp flag, # which doesn't work anymore with recent Intel compilers if comp_fam == toolchain.INTELCOMP: make_fps = glob.glob( os.path.join(self.cfg['start_dir'], self.build_subdir, '*.make')) regex_subs = [(r'-openmp', r'-fopenmp')] for make_fp in make_fps: apply_regex_substitutions(make_fps, regex_subs)
def configure_step(self): """Custom configuration procedure for TINKER.""" # make sure FFTW is available if get_software_root('FFTW') is None: raise EasyBuildError("FFTW dependency is not available.") os_dirs = { LINUX: 'linux', DARWIN: 'macosx', } os_type = get_os_type() os_dir = os_dirs.get(os_type) if os_dir is None: raise EasyBuildError("Failed to determine OS directory for %s (known: %s)", os_type, os_dirs) comp_dirs = { toolchain.INTELCOMP: 'intel', toolchain.GCC: 'gfortran', } comp_fam = self.toolchain.comp_family() comp_dir = comp_dirs.get(comp_fam) if comp_dir is None: raise EasyBuildError("Failed to determine compiler directory for %s (known: %s)", comp_fam, comp_dirs) self.build_subdir = os.path.join(os_dir, comp_dir) self.log.info("Using build scripts from %s subdirectory" % self.build_subdir) # patch 'link.make' script to use FFTW provided via EasyBuild link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir, 'link.make') regex_subs = [(r"libfftw3_threads.a libfftw3.a", r"-L$EBROOTFFTW/lib -lfftw3_threads -lfftw3")] apply_regex_substitutions(link_make_fp, regex_subs) # patch *.make files to get rid of hardcoded -openmp flag, # which doesn't work anymore with recent Intel compilers if comp_fam == toolchain.INTELCOMP: make_fps = glob.glob(os.path.join(self.cfg['start_dir'], self.build_subdir, '*.make')) regex_subs = [(r'-openmp', r'-fopenmp')] for make_fp in make_fps: apply_regex_substitutions(make_fps, regex_subs)
def prepare_rpath_wrappers(self, rpath_filter_dirs=None): """ Put RPATH wrapper script in place for compiler and linker commands :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) """ self.log.experimental( "Using wrapper scripts for compiler/linker commands that enforce RPATH linking" ) if get_os_type() == LINUX: self.log.info("Putting RPATH wrappers in place...") else: raise EasyBuildError( "RPATH linking is currently only supported on Linux") wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)? c_comps, fortran_comps = self.compilers() rpath_args_py = find_eb_script('rpath_args.py') rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in') # prepend location to wrappers to $PATH setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH'))) # figure out list of patterns to use in rpath filter rpath_filter = build_option('rpath_filter') if rpath_filter is None: rpath_filter = ['/lib.*', '/usr.*'] self.log.debug( "No general RPATH filter specified, falling back to default: %s", rpath_filter) rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs or []]) self.log.debug("Combined RPATH filter: '%s'" % rpath_filter) # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold']): orig_cmd = which(cmd) if orig_cmd: # bail out early if command already is a wrapped; # this may occur when building extensions if self.is_rpath_wrapper(orig_cmd): self.log.info( "%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd) continue cmd_wrapper = os.path.join(wrapper_dir, cmd) # make *very* sure we don't wrap around ourselves and create a fork bomb... if os.path.exists(cmd_wrapper) and os.path.exists( orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper): raise EasyBuildError( "Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd) # enable debug mode in wrapper script by specifying location for log file if build_option('debug'): rpath_wrapper_log = os.path.join( tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd) else: rpath_wrapper_log = '/dev/null' # complete template script and put it in place cmd_wrapper_txt = read_file(rpath_wrapper_template) % { 'orig_cmd': orig_cmd, 'python': sys.executable, 'rpath_args_py': rpath_args_py, 'rpath_filter': rpath_filter, 'rpath_wrapper_log': rpath_wrapper_log, } write_file(cmd_wrapper, cmd_wrapper_txt) adjust_permissions(cmd_wrapper, stat.S_IXUSR) self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log) else: self.log.debug( "Not installing RPATH wrapper for non-existing command '%s'", cmd)
def test_os_type(self): """Test getting OS type.""" os_type = get_os_type() self.assertTrue(os_type in [DARWIN, LINUX])
# """ Easyconfig constants module that provides all constants that can be used within an Easyconfig file. :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ import os import platform from vsc.utils import fancylogger from easybuild.tools.systemtools import get_shared_lib_ext, get_os_name, get_os_type, get_os_version _log = fancylogger.getLogger('easyconfig.constants', fname=False) EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE' # constants that can be used in easyconfig EASYCONFIG_CONSTANTS = { 'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"), 'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"), 'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"), 'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"), 'OS_VERSION': (get_os_version(), "System version"), 'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"), }
arch = 'aarch64' if arch not in KNOWN_ARCH_CONSTANTS: print_warning("Using unknown value for ARCH constant: %s", arch) return arch # constants that can be used in easyconfig EASYCONFIG_CONSTANTS = { 'ARCH': (_get_arch_constant(), "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"), 'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"), 'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"), 'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"), 'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"), 'OS_VERSION': (get_os_version(), "System version"), 'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"), 'SYSTEM': ({ 'name': 'system', 'version': 'system' }, "System toolchain"), 'OS_PKG_IBVERBS_DEV': (('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'), "OS packages providing ibverbs/infiniband development support"), 'OS_PKG_OPENSSL_BIN': (('openssl'), "OS packages providing the openSSL binary"), 'OS_PKG_OPENSSL_LIB': (('libssl', 'libopenssl'),
def sanity_check_step(self): """ Custom sanity check for GCC """ os_type = get_os_type() sharedlib_ext = get_shared_lib_ext() # determine "configuration name" directory, see https://sourceware.org/autobook/autobook/autobook_17.html # this differs across GCC versions; # x86_64-unknown-linux-gnu was common for old GCC versions, # x86_64-pc-linux-gnu is more likely with an updated config.guess script; # since this is internal to GCC, we don't really care how it is named exactly, # we only care that it's actually there # we may get multiple hits (libexec/, lib/), which is fine, # but we expect the same configuration name subdirectory in each of them glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc', '*-linux-gnu', self.version) matches = glob.glob(glob_pattern) if matches: cands = nub( [os.path.basename(os.path.dirname(x)) for x in matches]) if len(cands) == 1: config_name_subdir = cands[0] else: raise EasyBuildError( "Found multiple candidates for configuration name: %s", ', '.join(cands)) else: raise EasyBuildError( "Failed to determine configuration name: no matches for '%s'", glob_pattern) bin_files = ["gcov"] lib_files = [] if LooseVersion(self.version) >= LooseVersion('4.2'): # libgomp was added in GCC 4.2.0 ["libgomp.%s" % sharedlib_ext, "libgomp.a"] if os_type == 'Linux': lib_files.extend(["libgcc_s.%s" % sharedlib_ext]) # libmudflap is replaced by asan (see release notes gcc 4.9.0) if LooseVersion(self.version) < LooseVersion("4.9.0"): lib_files.extend( ["libmudflap.%s" % sharedlib_ext, "libmudflap.a"]) else: lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"]) libexec_files = [] dirs = [os.path.join('lib', 'gcc', config_name_subdir, self.version)] if not self.cfg['languages']: # default languages are c, c++, fortran bin_files = ["c++", "cpp", "g++", "gcc", "gcov", "gfortran"] lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) libexec_files = ['cc1', 'cc1plus', 'collect2', 'f951'] if 'c' in self.cfg['languages']: bin_files.extend(['cpp', 'gcc']) if 'c++' in self.cfg['languages']: bin_files.extend(['c++', 'g++']) dirs.append('include/c++/%s' % self.version) lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) if 'fortran' in self.cfg['languages']: bin_files.append('gfortran') lib_files.extend( ['libgfortran.%s' % sharedlib_ext, 'libgfortran.a']) if self.cfg['withlto']: libexec_files.extend(['lto1', 'lto-wrapper']) if os_type in ['Linux']: libexec_files.append('liblto_plugin.%s' % sharedlib_ext) bin_files = ["bin/%s" % x for x in bin_files] libdirs64 = ['lib64'] libdirs32 = ['lib', 'lib32'] libdirs = libdirs64 + libdirs32 if self.cfg['multilib']: # with multilib enabled, both lib and lib64 should be there lib_files64 = [ os.path.join(libdir, x) for libdir in libdirs64 for x in lib_files ] lib_files32 = [ tuple([os.path.join(libdir, x) for libdir in libdirs32]) for x in lib_files ] lib_files = lib_files64 + lib_files32 else: # lib64 on SuSE and Darwin, lib otherwise lib_files = [ tuple([os.path.join(libdir, x) for libdir in libdirs]) for x in lib_files ] # lib on SuSE, libexec otherwise libdirs = ['libexec', 'lib'] common_infix = os.path.join('gcc', config_name_subdir, self.version) libexec_files = [ tuple([os.path.join(d, common_infix, x) for d in libdirs]) for x in libexec_files ] old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()] custom_paths = { 'files': bin_files + lib_files + libexec_files + old_cmds, 'dirs': dirs, } super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths)
def test_os_type(self): """Test getting OS type.""" os_type = get_os_type() self.assertTrue(os_type in [DARWIN, LINUX])
def sanity_check_step(self): """ Custom sanity check for GCC """ os_type = get_os_type() sharedlib_ext = get_shared_lib_ext() common_infix = os.path.join('gcc', self.platform_lib, self.version) bin_files = ["gcov"] lib_files = [] if LooseVersion(self.version) >= LooseVersion('4.2'): # libgomp was added in GCC 4.2.0 ["libgomp.%s" % sharedlib_ext, "libgomp.a"] if os_type == 'Linux': lib_files.extend(["libgcc_s.%s" % sharedlib_ext]) # libmudflap is replaced by asan (see release notes gcc 4.9.0) if LooseVersion(self.version) < LooseVersion("4.9.0"): lib_files.extend( ["libmudflap.%s" % sharedlib_ext, "libmudflap.a"]) else: lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"]) libexec_files = [] dirs = ['lib/%s' % common_infix] if not self.cfg['languages']: # default languages are c, c++, fortran bin_files = ["c++", "cpp", "g++", "gcc", "gcov", "gfortran"] lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) libexec_files = ['cc1', 'cc1plus', 'collect2', 'f951'] if 'c' in self.cfg['languages']: bin_files.extend(['cpp', 'gcc']) if 'c++' in self.cfg['languages']: bin_files.extend(['c++', 'g++']) dirs.append('include/c++/%s' % self.version) lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) if 'fortran' in self.cfg['languages']: bin_files.append('gfortran') lib_files.extend( ['libgfortran.%s' % sharedlib_ext, 'libgfortran.a']) if self.cfg['withlto']: libexec_files.extend(['lto1', 'lto-wrapper']) if os_type in ['Linux']: libexec_files.append('liblto_plugin.%s' % sharedlib_ext) bin_files = ["bin/%s" % x for x in bin_files] libdirs64 = ['lib64'] libdirs32 = ['lib', 'lib32'] libdirs = libdirs64 + libdirs32 if self.cfg['multilib']: # with multilib enabled, both lib and lib64 should be there lib_files64 = [ os.path.join(libdir, x) for libdir in libdirs64 for x in lib_files ] lib_files32 = [ tuple([os.path.join(libdir, x) for libdir in libdirs32]) for x in lib_files ] lib_files = lib_files64 + lib_files32 else: # lib64 on SuSE and Darwin, lib otherwise lib_files = [ tuple([os.path.join(libdir, x) for libdir in libdirs]) for x in lib_files ] # lib on SuSE, libexec otherwise libdirs = ['libexec', 'lib'] libexec_files = [ tuple( [os.path.join(libdir, common_infix, x) for libdir in libdirs]) for x in libexec_files ] old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()] custom_paths = { 'files': bin_files + lib_files + libexec_files + old_cmds, 'dirs': dirs, } super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths)
def prepare_rpath_wrappers(self, rpath_filter_dirs=None): """ Put RPATH wrapper script in place for compiler and linker commands :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) """ self.log.experimental("Using wrapper scripts for compiler/linker commands that enforce RPATH linking") if get_os_type() == LINUX: self.log.info("Putting RPATH wrappers in place...") else: raise EasyBuildError("RPATH linking is currently only supported on Linux") wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)? c_comps, fortran_comps = self.compilers() rpath_args_py = find_eb_script('rpath_args.py') rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in') # prepend location to wrappers to $PATH setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH'))) # figure out list of patterns to use in rpath filter rpath_filter = build_option('rpath_filter') if rpath_filter is None: rpath_filter = ['/lib.*', '/usr.*'] self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter) rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs or []]) self.log.debug("Combined RPATH filter: '%s'" % rpath_filter) # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold']): orig_cmd = which(cmd) if orig_cmd: # bail out early if command already is a wrapped; # this may occur when building extensions if self.is_rpath_wrapper(orig_cmd): self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd) continue cmd_wrapper = os.path.join(wrapper_dir, cmd) # make *very* sure we don't wrap around ourselves and create a fork bomb... if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper): raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd) # enable debug mode in wrapper script by specifying location for log file if build_option('debug'): rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd) else: rpath_wrapper_log = '/dev/null' # complete template script and put it in place cmd_wrapper_txt = read_file(rpath_wrapper_template) % { 'orig_cmd': orig_cmd, 'python': sys.executable, 'rpath_args_py': rpath_args_py, 'rpath_filter': rpath_filter, 'rpath_wrapper_log': rpath_wrapper_log, } write_file(cmd_wrapper, cmd_wrapper_txt) adjust_permissions(cmd_wrapper, stat.S_IXUSR) self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log) else: self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd)
def test_check_linked_shared_libs(self): """Test for check_linked_shared_libs function.""" txt_path = os.path.join(self.test_prefix, 'test.txt') write_file(txt_path, "some text") broken_symlink_path = os.path.join(self.test_prefix, 'broken_symlink') symlink('/doesnotexist', broken_symlink_path, use_abspath_source=False) # result is always None for anything other than dynamically linked binaries or shared libraries self.assertEqual(check_linked_shared_libs(self.test_prefix), None) self.assertEqual(check_linked_shared_libs(txt_path), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path), None) bin_ls_path = which('ls') os_type = get_os_type() if os_type == LINUX: out, _ = run_cmd("ldd %s" % bin_ls_path) elif os_type == DARWIN: out, _ = run_cmd("otool -L %s" % bin_ls_path) else: raise EasyBuildError("Unknown OS type: %s" % os_type) shlib_ext = get_shared_lib_ext() lib_path_regex = re.compile( r'(?P<lib_path>[^\s]*/lib[^ ]+\.%s[^ ]*)' % shlib_ext, re.M) lib_path = lib_path_regex.search(out).group(1) test_pattern_named_args = [ # if no patterns are specified, result is always True {}, { 'required_patterns': ['/lib', shlib_ext] }, { 'banned_patterns': ['this_pattern_should_not_match'] }, { 'required_patterns': ['/lib', shlib_ext], 'banned_patterns': ['weirdstuff'] }, ] for pattern_named_args in test_pattern_named_args: # result is always None for anything other than dynamically linked binaries or shared libraries self.assertEqual( check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) self.assertEqual( check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual( check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) for path in (bin_ls_path, lib_path): error_msg = "Check on linked libs should pass for %s with %s" % ( path, pattern_named_args) self.assertTrue( check_linked_shared_libs(path, **pattern_named_args), error_msg) # also test with input that should result in failing check test_pattern_named_args = [ { 'required_patterns': ['this_pattern_will_not_match'] }, { 'banned_patterns': ['/lib'] }, { 'required_patterns': ['weirdstuff'], 'banned_patterns': ['/lib', shlib_ext] }, ] for pattern_named_args in test_pattern_named_args: # result is always None for anything other than dynamically linked binaries or shared libraries self.assertEqual( check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) self.assertEqual( check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual( check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) for path in (bin_ls_path, lib_path): error_msg = "Check on linked libs should fail for %s with %s" % ( path, pattern_named_args) self.assertFalse( check_linked_shared_libs(path, **pattern_named_args), error_msg)
def __init__(self, *args, **kwargs): """Locate the installation files of OpenSSL in the host system""" super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs) # Libraries packaged in OpenSSL openssl_libs = ['libssl', 'libcrypto'] # list of relevant library extensions per system and version of OpenSSL # the first item should be the extension of an installation from source, # it will be used in the sanity checks of the component openssl_libext = { '1.0': { LINUX: ('so.1.0.0', 'so.10'), DARWIN: ('1.0.dylib', ), }, '1.1': { LINUX: ('so.1.1', ), DARWIN: ('1.1.dylib', ), }, } os_type = get_os_type() if self.version in openssl_libext and os_type in openssl_libext[ self.version]: # generate matrix of versioned .so filenames system_versioned_libs = [[ '%s.%s' % (lib, ext) for ext in openssl_libext[self.version][os_type] ] for lib in openssl_libs] self.log.info("Matrix of version library names: %s", system_versioned_libs) else: raise EasyBuildError( "Don't know name of OpenSSL system library for version %s and OS type %s", self.version, os_type) # by default target the first option of each OpenSSL library, # which corresponds to installation from source self.target_ssl_libs = [ lib_name[0] for lib_name in system_versioned_libs ] self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs) # folders containing engines libraries openssl_engines = { '1.0': 'engines', '1.1': 'engines-1.1', } self.target_ssl_engine = openssl_engines[self.version] # Paths to system libraries and headers of OpenSSL self.system_ssl = { 'bin': None, 'engines': None, 'include': None, 'lib': None, } # early return when we're not wrapping the system OpenSSL installation if not self.cfg.get('wrap_system_openssl'): self.log.info("Not wrapping system OpenSSL installation!") return # Check the system libraries of OpenSSL # (only needs first library in openssl_libs to find path to libraries) for idx, libssl in enumerate(system_versioned_libs[0]): self.system_ssl['lib'] = find_library_path(libssl) if self.system_ssl['lib']: # change target libraries to the ones found in the system self.target_ssl_libs = [ lib_name[idx] for lib_name in system_versioned_libs ] self.log.info("Target system OpenSSL libraries: %s", self.target_ssl_libs) break if self.system_ssl['lib']: self.log.info("Found library '%s' in: %s", openssl_libs[0], self.system_ssl['lib']) else: self.log.info( "OpenSSL library '%s' not found, falling back to OpenSSL in EasyBuild", openssl_libs[0]) # Directory with engine libraries if self.system_ssl['lib']: lib_dir = os.path.dirname(self.system_ssl['lib']) lib_engines_dir = [ os.path.join(lib_dir, 'openssl', self.target_ssl_engine), os.path.join(lib_dir, self.target_ssl_engine), ] for engines_path in lib_engines_dir: if os.path.isdir(engines_path): self.system_ssl['engines'] = engines_path self.log.debug("Found OpenSSL engines in: %s", self.system_ssl['engines']) break if not self.system_ssl['engines']: self.system_ssl['lib'] = None self.log.info( "OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild" ) # Check system include paths for OpenSSL headers cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null" (out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False) sys_include_dirs = [] for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE): sys_include_dirs.extend(match.groups()) self.log.debug( "Found the following include directories in host system: %s", ', '.join(sys_include_dirs)) # headers are located in 'include/openssl' by default ssl_include_subdirs = [self.name.lower()] if self.version == '1.1': # but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7 # prefer 'include/openssl' as long as the version of headers matches ssl_include_subdirs.append( os.path.join('openssl11', self.name.lower())) ssl_include_dirs = [ os.path.join(incd, subd) for incd in sys_include_dirs for subd in ssl_include_subdirs ] ssl_include_dirs = [ include for include in ssl_include_dirs if os.path.isdir(include) ] # find location of header files, verify that the headers match our OpenSSL version openssl_version_regex = re.compile( r"SHLIB_VERSION_NUMBER\s\"([0-9]+\.[0-9]+)", re.M) for include_dir in ssl_include_dirs: opensslv_path = os.path.join(include_dir, 'opensslv.h') self.log.debug("Checking OpenSSL version in %s...", opensslv_path) if os.path.exists(opensslv_path): opensslv = read_file(opensslv_path) header_majmin_version = openssl_version_regex.search(opensslv) if header_majmin_version: header_majmin_version = header_majmin_version.group(1) if re.match('^' + header_majmin_version, self.version): self.system_ssl['include'] = include_dir self.log.info( "Found OpenSSL headers in host system: %s", self.system_ssl['include']) break else: self.log.debug( "Header major/minor version '%s' doesn't match with %s", header_majmin_version, self.version) else: self.log.debug("Pattern '%s' not found in %s", openssl_version_regex.pattern, opensslv_path) else: self.log.info("OpenSSL header file %s not found", opensslv_path) if not self.system_ssl['include']: self.log.info( "OpenSSL headers not found in host system, falling back to OpenSSL in EasyBuild" ) # Check system OpenSSL binary if self.version == '1.1': # prefer 'openssl11' over 'openssl' with v1.1 self.system_ssl['bin'] = which('openssl11') if not self.system_ssl['bin']: self.system_ssl['bin'] = which(self.name.lower()) if self.system_ssl['bin']: self.log.info("System OpenSSL binary found: %s", self.system_ssl['bin']) else: self.log.info("System OpenSSL binary not found!")
def __init__(self, *args, **kwargs): """Locate the installation files of OpenSSL in the host system""" super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs) # Wrapper should have at least a major minor version numbers try: subversions = self.version.split('.') self.majmin_version = '%s.%s' % (subversions[0], subversions[1]) except (AttributeError, IndexError): err_msg = "Wrapper OpenSSL version does not have any subversion: %s" raise EasyBuildError(err_msg, self.version) # Set minimum OpenSSL version min_openssl_version = self.cfg.get('minimum_openssl_version') if not min_openssl_version: min_openssl_version = self.version elif not isinstance(min_openssl_version, string_type): min_openssl_version = str(min_openssl_version) # Minimum OpenSSL version can only increase depth of wrapper version if min_openssl_version.startswith(self.version): self.log.debug("Requiring minimum OpenSSL version: %s", min_openssl_version) else: err_msg = "Requested minimum OpenSSL version '%s' does not fit in wrapper easyconfig version '%s'" raise EasyBuildError(err_msg, min_openssl_version, self.version) # Regex pattern to find version strings in OpenSSL libraries and headers full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?') openssl_version_regex = re.compile( r'OpenSSL\s+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M) # Libraries packaged in OpenSSL openssl_libs = ['libssl', 'libcrypto'] # list of relevant library extensions per system and version of OpenSSL # the first item should be the extension of an installation from source, # it will be used in the sanity checks of the component openssl_libext = { '1.0': { LINUX: ('so.1.0.0', 'so.10'), DARWIN: ('1.0.dylib', ), }, '1.1': { LINUX: ('so.1.1', ), DARWIN: ('1.1.dylib', ), }, '3.0': { LINUX: ('so.3', ), DARWIN: ('3.dylib', ), }, } os_type = get_os_type() if self.majmin_version in openssl_libext and os_type in openssl_libext[ self.majmin_version]: # generate matrix of versioned .so filenames system_versioned_libs = [[ '%s.%s' % (lib, ext) for lib in openssl_libs ] for ext in openssl_libext[self.majmin_version][os_type]] self.log.info("Matrix of version library names: %s", system_versioned_libs) else: err_msg = "Don't know name of OpenSSL system library for version %s and OS type %s" raise EasyBuildError(err_msg, self.majmin_version, os_type) # by default target the first option of each OpenSSL library, # which corresponds to installation from source self.target_ssl_libs = system_versioned_libs[0] self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs) # folders containing engines libraries openssl_engines = { '1.0': 'engines', '1.1': 'engines-1.1', '3.0': 'engines-3', } self.target_ssl_engine = openssl_engines[self.majmin_version] # Paths to system libraries and headers of OpenSSL self.system_ssl = { 'bin': None, 'engines': None, 'include': None, 'libs': [], } # early return when we're not wrapping the system OpenSSL installation if not self.cfg.get('wrap_system_openssl'): self.log.info( "Not wrapping system OpenSSL installation by user request") return # Check the system libraries of OpenSSL # Find library file and compare its version string for idx, solibs in enumerate(system_versioned_libs): for solib in solibs: system_solib = find_library_path(solib) if system_solib: openssl_version = 0 # get version of system library filename try: openssl_version = full_version_regex.search( os.path.realpath(system_solib)).group(0) except AttributeError: # filename lacks the full version, fallback to version strings within the library solib_strings = read_file(system_solib, mode="rb").decode( 'utf-8', 'replace') try: openssl_version = openssl_version_regex.search( solib_strings).group(1) except AttributeError: dbg_msg = "Could not detect the full version of system OpenSSL library: %s" self.log.debug(dbg_msg, system_solib) # check that system version fulfills requirements if LooseVersion(openssl_version) >= LooseVersion( min_openssl_version): dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) self.system_ssl['libs'].append(system_solib) else: dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) else: # one of the OpenSSL libraries is missing, switch to next group of versioned libs self.system_ssl['libs'] = [] break if len(self.system_ssl['libs']) == len(openssl_libs): # keep these libraries as possible targets for this installation target_system_ssl_libs = system_versioned_libs[idx] break if len(self.system_ssl['libs']) == len(openssl_libs): self.system_ssl['version'] = openssl_version info_msg = "Found OpenSSL library version %s in host system: %s" self.log.info(info_msg, self.system_ssl['version'], os.path.dirname(self.system_ssl['libs'][0])) else: self.log.info( "OpenSSL library not found in host system, falling back to OpenSSL in EasyBuild" ) return # Directory with engine libraries lib_dir = os.path.dirname(self.system_ssl['libs'][0]) lib_engines_dir = [ os.path.join(lib_dir, 'openssl', self.target_ssl_engine), os.path.join(lib_dir, self.target_ssl_engine), ] for engines_path in lib_engines_dir: if os.path.isdir(engines_path): self.system_ssl['engines'] = engines_path self.log.debug("Found OpenSSL engines in: %s", self.system_ssl['engines']) break if not self.system_ssl['engines']: self.log.info( "OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild" ) return # Check system include paths for OpenSSL headers cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null" (out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False) sys_include_dirs = [] for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE): sys_include_dirs.extend(match.groups()) self.log.debug( "Found the following include directories in host system: %s", ', '.join(sys_include_dirs)) # headers are located in 'include/openssl' by default ssl_include_subdirs = ['openssl'] if self.majmin_version == '1.1': # but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7 # prefer 'include/openssl' as long as the version of headers matches ssl_include_subdirs.append( os.path.join('openssl11', self.name.lower())) ssl_include_dirs = [ os.path.join(incd, subd) for incd in sys_include_dirs for subd in ssl_include_subdirs ] ssl_include_dirs = [ include for include in ssl_include_dirs if os.path.isdir(include) ] # find location of header files for this version of the OpenSSL libraries for include_dir in ssl_include_dirs: opensslv_path = os.path.join(include_dir, 'opensslv.h') self.log.debug("Checking OpenSSL version in %s...", opensslv_path) if os.path.exists(opensslv_path): # check version reported by opensslv.h opensslv = read_file(opensslv_path) try: header_version = openssl_version_regex.search( opensslv).group(1) except AttributeError: err_msg = "System OpenSSL header '%s' does not contain any recognizable version string" raise EasyBuildError(err_msg, opensslv_path) if header_version == self.system_ssl['version']: self.system_ssl['include'] = include_dir info_msg = "Found OpenSSL headers v%s in host system: %s" self.log.info(info_msg, header_version, self.system_ssl['include']) break else: dbg_msg = "System OpenSSL header version '%s' doesn not match library version '%s'" self.log.debug(dbg_msg, header_version, self.system_ssl['version']) else: self.log.info("System OpenSSL header file %s not found", opensslv_path) if not self.system_ssl['include']: err_msg = ( "OpenSSL v%s headers not found in host system, but libraries for v%s are present. " "Install the development package of OpenSSL for your system or force building OpenSSL from " "source in EasyBuild by setting 'wrap_system_openssl = False' in the OpenSSL easyconfig." ) raise EasyBuildError(err_msg, self.version, self.system_ssl['version']) # Check system OpenSSL binary if self.majmin_version == '1.1': # prefer 'openssl11' over 'openssl' with v1.1 self.system_ssl['bin'] = which('openssl11') if not self.system_ssl['bin']: self.system_ssl['bin'] = which('openssl') if self.system_ssl['bin']: self.log.info("System OpenSSL binary found: %s", self.system_ssl['bin']) else: self.log.info("System OpenSSL binary not found!") return # system OpenSSL is fine, change target libraries to the ones found in it self.target_ssl_libs = target_system_ssl_libs self.log.info("Target system OpenSSL libraries: %s", self.target_ssl_libs)
def test_check_linked_shared_libs(self): """Test for check_linked_shared_libs function.""" txt_path = os.path.join(self.test_prefix, 'test.txt') write_file(txt_path, "some text") broken_symlink_path = os.path.join(self.test_prefix, 'broken_symlink') symlink('/doesnotexist', broken_symlink_path, use_abspath_source=False) # result is always None for anything other than dynamically linked binaries or shared libraries self.assertEqual(check_linked_shared_libs(self.test_prefix), None) self.assertEqual(check_linked_shared_libs(txt_path), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path), None) bin_ls_path = which('ls') os_type = get_os_type() if os_type == LINUX: out, _ = run_cmd("ldd %s" % bin_ls_path) elif os_type == DARWIN: out, _ = run_cmd("otool -L %s" % bin_ls_path) else: raise EasyBuildError("Unknown OS type: %s" % os_type) shlib_ext = get_shared_lib_ext() lib_path_regex = re.compile(r'(?P<lib_path>[^\s]*/lib[^ ]+\.%s[^ ]*)' % shlib_ext, re.M) lib_path = lib_path_regex.search(out).group(1) test_pattern_named_args = [ # if no patterns are specified, result is always True {}, {'required_patterns': ['/lib', shlib_ext]}, {'banned_patterns': ['this_pattern_should_not_match']}, {'required_patterns': ['/lib', shlib_ext], 'banned_patterns': ['weirdstuff']}, ] for pattern_named_args in test_pattern_named_args: # result is always None for anything other than dynamically linked binaries or shared libraries self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) for path in (bin_ls_path, lib_path): # path may not exist, especially for library paths obtained via 'otool -L' on macOS if os.path.exists(path): error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args) self.assertTrue(check_linked_shared_libs(path, **pattern_named_args), error_msg) # also test with input that should result in failing check test_pattern_named_args = [ {'required_patterns': ['this_pattern_will_not_match']}, {'banned_patterns': ['/lib']}, {'required_patterns': ['weirdstuff'], 'banned_patterns': ['/lib', shlib_ext]}, ] for pattern_named_args in test_pattern_named_args: # result is always None for anything other than dynamically linked binaries or shared libraries self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) for path in (bin_ls_path, lib_path): error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args) self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg) if get_os_type() == LINUX: # inject fake 'file' command which always reports that the file is "dynamically linked" file_cmd = os.path.join(self.test_prefix, 'bin', 'file') write_file(file_cmd, "echo '(dynamically linked)'") adjust_permissions(file_cmd, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.join(self.test_prefix, 'bin') + ':' + os.getenv('PATH') test_file = os.path.join(self.test_prefix, 'test.txt') write_file(test_file, 'test') warning_regex = re.compile(r"WARNING: Determining linked libraries.* via 'ldd .*/test.txt' failed!", re.M) self.mock_stderr(True) self.mock_stdout(True) res = check_linked_shared_libs(test_file, banned_patterns=['/lib']) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) fail_msg = "Pattern '%s' should be found in: %s" % (warning_regex.pattern, stderr) self.assertTrue(warning_regex.search(stderr), fail_msg) self.assertFalse(stdout) self.assertEqual(res, None)
def sanity_check_step(self): """ Custom sanity check for GCC """ os_type = get_os_type() sharedlib_ext = get_shared_lib_ext() # determine "configuration name" directory, see https://sourceware.org/autobook/autobook/autobook_17.html # this differs across GCC versions; # x86_64-unknown-linux-gnu was common for old GCC versions, # x86_64-pc-linux-gnu is more likely with an updated config.guess script; # since this is internal to GCC, we don't really care how it is named exactly, # we only care that it's actually there # we may get multiple hits (libexec/, lib/), which is fine, # but we expect the same configuration name subdirectory in each of them glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc', '*-linux-gnu', self.version) matches = glob.glob(glob_pattern) if matches: cands = nub( [os.path.basename(os.path.dirname(x)) for x in matches]) if len(cands) == 1: config_name_subdir = cands[0] else: raise EasyBuildError( "Found multiple candidates for configuration name: %s", ', '.join(cands)) else: raise EasyBuildError( "Failed to determine configuration name: no matches for '%s'", glob_pattern) bin_files = ["gcov"] lib_files = [] if LooseVersion(self.version) >= LooseVersion('4.2'): # libgomp was added in GCC 4.2.0 ["libgomp.%s" % sharedlib_ext, "libgomp.a"] if os_type == 'Linux': lib_files.extend(["libgcc_s.%s" % sharedlib_ext]) # libmudflap is replaced by asan (see release notes gcc 4.9.0) if LooseVersion(self.version) < LooseVersion("4.9.0"): lib_files.extend( ["libmudflap.%s" % sharedlib_ext, "libmudflap.a"]) else: lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"]) libexec_files = [] dirs = [os.path.join('lib', 'gcc', config_name_subdir, self.version)] languages = self.cfg['languages'] or ['c', 'c++', 'fortran' ] # default languages if 'c' in languages: bin_files.extend(['cpp', 'gcc']) libexec_files.extend(['cc1', 'collect2']) if 'c++' in languages: bin_files.extend(['c++', 'g++']) dirs.append('include/c++/%s' % self.version) lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) libexec_files.append( 'cc1plus') # c++ requires c, so collect2 not mentioned again if 'fortran' in languages: bin_files.append('gfortran') lib_files.extend( ['libgfortran.%s' % sharedlib_ext, 'libgfortran.a']) libexec_files.append('f951') if self.cfg['withlto']: libexec_files.extend(['lto1', 'lto-wrapper']) if os_type in ['Linux']: libexec_files.append('liblto_plugin.%s' % sharedlib_ext) if self.cfg['withnvptx']: bin_files.extend(['nvptx-none-as', 'nvptx-none-ld']) lib_files.append('libgomp-plugin-nvptx.%s' % sharedlib_ext) bin_files = ["bin/%s" % x for x in bin_files] libdirs64 = ['lib64'] libdirs32 = ['lib', 'lib32'] libdirs = libdirs64 + libdirs32 if self.cfg['multilib']: # with multilib enabled, both lib and lib64 should be there lib_files64 = [ os.path.join(libdir, x) for libdir in libdirs64 for x in lib_files ] lib_files32 = [ tuple([os.path.join(libdir, x) for libdir in libdirs32]) for x in lib_files ] lib_files = lib_files64 + lib_files32 else: # lib64 on SuSE and Darwin, lib otherwise lib_files = [ tuple([os.path.join(libdir, x) for libdir in libdirs]) for x in lib_files ] # lib on SuSE, libexec otherwise libdirs = ['libexec', 'lib'] common_infix = os.path.join('gcc', config_name_subdir, self.version) libexec_files = [ tuple([os.path.join(d, common_infix, x) for d in libdirs]) for x in libexec_files ] old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()] custom_paths = { 'files': bin_files + lib_files + libexec_files + old_cmds, 'dirs': dirs, } custom_commands = [] for lang, compiler in (('c', 'gcc'), ('c++', 'g++')): if lang in languages: # Simple test compile cmd = 'echo "int main(){} " | %s -x %s -o/dev/null -' compiler_path = os.path.join(self.installdir, 'bin', compiler) custom_commands.append(cmd % (compiler_path, lang)) if self.cfg['withlto']: custom_commands.append( cmd % (compiler_path, lang + ' -flto -fuse-linker-plugin')) if custom_commands: # Load binutils to do the compile tests extra_modules = [ d['short_mod_name'] for d in self.cfg.dependencies() if d['name'] == 'binutils' ] else: extra_modules = None super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands, extra_modules=extra_modules)
def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None): """ Put RPATH wrapper script in place for compiler and linker commands :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) """ if get_os_type() == LINUX: self.log.info("Putting RPATH wrappers in place...") else: raise EasyBuildError( "RPATH linking is currently only supported on Linux") if rpath_filter_dirs is None: rpath_filter_dirs = [] # always include filter for 'stubs' library directory, # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683 rpath_filter_dirs.append('.*/lib(64)?/stubs/?') # directory where all wrappers will be placed wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)? c_comps, fortran_comps = self.compilers() rpath_args_py = find_eb_script('rpath_args.py') rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in') # figure out list of patterns to use in rpath filter rpath_filter = build_option('rpath_filter') if rpath_filter is None: rpath_filter = ['/lib.*', '/usr.*'] self.log.debug( "No general RPATH filter specified, falling back to default: %s", rpath_filter) rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs]) self.log.debug("Combined RPATH filter: '%s'", rpath_filter) rpath_include = ','.join(rpath_include_dirs or []) self.log.debug("Combined RPATH include paths: '%s'", rpath_include) # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']): orig_cmd = which(cmd) if orig_cmd: # bail out early if command already is a wrapped; # this may occur when building extensions if self.is_rpath_wrapper(orig_cmd): self.log.info( "%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd) continue # determine location for this wrapper # each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately # avoid '+' character in directory name (for example with 'g++' command), which can cause trouble # (see https://github.com/easybuilders/easybuild-easyconfigs/issues/7339) wrapper_dir_name = '%s_wrapper' % cmd.replace('+', 'x') wrapper_dir = os.path.join(wrappers_dir, wrapper_dir_name) cmd_wrapper = os.path.join(wrapper_dir, cmd) # make *very* sure we don't wrap around ourselves and create a fork bomb... if os.path.exists(cmd_wrapper) and os.path.exists( orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper): raise EasyBuildError( "Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd) # enable debug mode in wrapper script by specifying location for log file if build_option('debug'): rpath_wrapper_log = os.path.join( tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd) else: rpath_wrapper_log = '/dev/null' # complete template script and put it in place cmd_wrapper_txt = read_file(rpath_wrapper_template) % { 'orig_cmd': orig_cmd, 'python': sys.executable, 'rpath_args_py': rpath_args_py, 'rpath_filter': rpath_filter, 'rpath_include': rpath_include, 'rpath_wrapper_log': rpath_wrapper_log, 'wrapper_dir': wrapper_dir, } write_file(cmd_wrapper, cmd_wrapper_txt) adjust_permissions(cmd_wrapper, stat.S_IXUSR) self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log) # prepend location to this wrapper to $PATH setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH'))) else: self.log.debug( "Not installing RPATH wrapper for non-existing command '%s'", cmd)
def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None): """ Put RPATH wrapper script in place for compiler and linker commands :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) """ if get_os_type() == LINUX: self.log.info("Putting RPATH wrappers in place...") else: raise EasyBuildError("RPATH linking is currently only supported on Linux") if rpath_filter_dirs is None: rpath_filter_dirs = [] # always include filter for 'stubs' library directory, # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683 rpath_filter_dirs.append('.*/lib(64)?/stubs/?') # directory where all wrappers will be placed wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)? c_comps, fortran_comps = self.compilers() rpath_args_py = find_eb_script('rpath_args.py') rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in') # figure out list of patterns to use in rpath filter rpath_filter = build_option('rpath_filter') if rpath_filter is None: rpath_filter = ['/lib.*', '/usr.*'] self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter) rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs]) self.log.debug("Combined RPATH filter: '%s'", rpath_filter) rpath_include = ','.join(rpath_include_dirs or []) self.log.debug("Combined RPATH include paths: '%s'", rpath_include) # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']): orig_cmd = which(cmd) if orig_cmd: # bail out early if command already is a wrapped; # this may occur when building extensions if self.is_rpath_wrapper(orig_cmd): self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd) continue # determine location for this wrapper # each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately # avoid '+' character in directory name (for example with 'g++' command), which can cause trouble # (see https://github.com/easybuilders/easybuild-easyconfigs/issues/7339) wrapper_dir_name = '%s_wrapper' % cmd.replace('+', 'x') wrapper_dir = os.path.join(wrappers_dir, wrapper_dir_name) cmd_wrapper = os.path.join(wrapper_dir, cmd) # make *very* sure we don't wrap around ourselves and create a fork bomb... if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper): raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd) # enable debug mode in wrapper script by specifying location for log file if build_option('debug'): rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd) else: rpath_wrapper_log = '/dev/null' # complete template script and put it in place cmd_wrapper_txt = read_file(rpath_wrapper_template) % { 'orig_cmd': orig_cmd, 'python': sys.executable, 'rpath_args_py': rpath_args_py, 'rpath_filter': rpath_filter, 'rpath_include': rpath_include, 'rpath_wrapper_log': rpath_wrapper_log, 'wrapper_dir': wrapper_dir, } write_file(cmd_wrapper, cmd_wrapper_txt) adjust_permissions(cmd_wrapper, stat.S_IXUSR) self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log) # prepend location to this wrapper to $PATH setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH'))) else: self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd)
def sanity_check_step(self): """ Custom sanity check for GCC """ os_type = get_os_type() sharedlib_ext = get_shared_lib_ext() common_infix = os.path.join('gcc', self.platform_lib, self.version) bin_files = ["gcov"] lib_files = [] if LooseVersion(self.version) >= LooseVersion('4.2'): # libgomp was added in GCC 4.2.0 ["libgomp.%s" % sharedlib_ext, "libgomp.a"] if os_type == 'Linux': lib_files.extend(["libgcc_s.%s" % sharedlib_ext]) # libmudflap is replaced by asan (see release notes gcc 4.9.0) if LooseVersion(self.version) < LooseVersion("4.9.0"): lib_files.extend(["libmudflap.%s" % sharedlib_ext, "libmudflap.a"]) else: lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"]) libexec_files = [] dirs = ['lib/%s' % common_infix] if not self.cfg['languages']: # default languages are c, c++, fortran bin_files = ["c++", "cpp", "g++", "gcc", "gcov", "gfortran"] lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) libexec_files = ['cc1', 'cc1plus', 'collect2', 'f951'] if 'c' in self.cfg['languages']: bin_files.extend(['cpp', 'gcc']) if 'c++' in self.cfg['languages']: bin_files.extend(['c++', 'g++']) dirs.append('include/c++/%s' % self.version) lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) if 'fortran' in self.cfg['languages']: bin_files.append('gfortran') lib_files.extend(['libgfortran.%s' % sharedlib_ext, 'libgfortran.a']) if self.cfg['withlto']: libexec_files.extend(['lto1', 'lto-wrapper']) if os_type in ['Linux']: libexec_files.append('liblto_plugin.%s' % sharedlib_ext) bin_files = ["bin/%s" % x for x in bin_files] libdirs64 = ['lib64'] libdirs32 = ['lib', 'lib32'] libdirs = libdirs64 + libdirs32 if self.cfg['multilib']: # with multilib enabled, both lib and lib64 should be there lib_files64 = [os.path.join(libdir, x) for libdir in libdirs64 for x in lib_files] lib_files32 = [tuple([os.path.join(libdir, x) for libdir in libdirs32]) for x in lib_files] lib_files = lib_files64 + lib_files32 else: # lib64 on SuSE and Darwin, lib otherwise lib_files = [tuple([os.path.join(libdir, x) for libdir in libdirs]) for x in lib_files] # lib on SuSE, libexec otherwise libdirs = ['libexec', 'lib'] libexec_files = [tuple([os.path.join(libdir, common_infix, x) for libdir in libdirs]) for x in libexec_files] old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()] custom_paths = { 'files': bin_files + lib_files + libexec_files + old_cmds, 'dirs': dirs, } super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths)