def test_find_library_path(self):
"""Test find_library_path function (Linux and Darwin only)."""
if get_os_type() == LINUX:
libname = 'libc.so.6'
elif get_os_type() == DARWIN:
libname = 'libSystem.dylib'
else:
libname = None
if libname:
lib_path = find_library_path(libname)
self.assertEqual(os.path.basename(lib_path), libname)
self.assertTrue(os.path.exists(lib_path),
"%s should exist" % libname)
def __init__(self, *args, **kwargs):
"""Locate the installation files of OpenSSL in the host system"""
super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs)
# Wrapper should have at least a major minor version numbers
try:
subversions = self.version.split('.')
self.majmin_version = '%s.%s' % (subversions[0], subversions[1])
except (AttributeError, IndexError):
err_msg = "Wrapper OpenSSL version does not have any subversion: %s"
raise EasyBuildError(err_msg, self.version)
# Set minimum OpenSSL version
min_openssl_version = self.cfg.get('minimum_openssl_version')
if not min_openssl_version:
min_openssl_version = self.version
elif not isinstance(min_openssl_version, string_type):
min_openssl_version = str(min_openssl_version)
# Minimum OpenSSL version can only increase depth of wrapper version
if min_openssl_version.startswith(self.version):
self.log.debug("Requiring minimum OpenSSL version: %s",
min_openssl_version)
else:
err_msg = "Requested minimum OpenSSL version '%s' does not fit in wrapper easyconfig version '%s'"
raise EasyBuildError(err_msg, min_openssl_version, self.version)
# Regex pattern to find version strings in OpenSSL libraries and headers
full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?')
openssl_version_regex = re.compile(
r'OpenSSL\s+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M)
# Libraries packaged in OpenSSL
openssl_libs = ['libssl', 'libcrypto']
# list of relevant library extensions per system and version of OpenSSL
# the first item should be the extension of an installation from source,
# it will be used in the sanity checks of the component
openssl_libext = {
'1.0': {
LINUX: ('so.1.0.0', 'so.10'),
DARWIN: ('1.0.dylib', ),
},
'1.1': {
LINUX: ('so.1.1', ),
DARWIN: ('1.1.dylib', ),
},
'3.0': {
LINUX: ('so.3', ),
DARWIN: ('3.dylib', ),
},
}
os_type = get_os_type()
if self.majmin_version in openssl_libext and os_type in openssl_libext[
self.majmin_version]:
# generate matrix of versioned .so filenames
system_versioned_libs = [[
'%s.%s' % (lib, ext) for lib in openssl_libs
] for ext in openssl_libext[self.majmin_version][os_type]]
self.log.info("Matrix of version library names: %s",
system_versioned_libs)
else:
err_msg = "Don't know name of OpenSSL system library for version %s and OS type %s"
raise EasyBuildError(err_msg, self.majmin_version, os_type)
# by default target the first option of each OpenSSL library,
# which corresponds to installation from source
self.target_ssl_libs = system_versioned_libs[0]
self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs)
# folders containing engines libraries
openssl_engines = {
'1.0': 'engines',
'1.1': 'engines-1.1',
'3.0': 'engines-3',
}
self.target_ssl_engine = openssl_engines[self.majmin_version]
# Paths to system libraries and headers of OpenSSL
self.system_ssl = {
'bin': None,
'engines': None,
'include': None,
'libs': [],
}
# early return when we're not wrapping the system OpenSSL installation
if not self.cfg.get('wrap_system_openssl'):
self.log.info(
"Not wrapping system OpenSSL installation by user request")
return
# Check the system libraries of OpenSSL
# Find library file and compare its version string
for idx, solibs in enumerate(system_versioned_libs):
for solib in solibs:
system_solib = find_library_path(solib)
if system_solib:
openssl_version = 0
# get version of system library filename
try:
openssl_version = full_version_regex.search(
os.path.realpath(system_solib)).group(0)
except AttributeError:
# filename lacks the full version, fallback to version strings within the library
solib_strings = read_file(system_solib,
mode="rb").decode(
'utf-8', 'replace')
try:
openssl_version = openssl_version_regex.search(
solib_strings).group(1)
except AttributeError:
dbg_msg = "Could not detect the full version of system OpenSSL library: %s"
self.log.debug(dbg_msg, system_solib)
# check that system version fulfills requirements
if LooseVersion(openssl_version) >= LooseVersion(
min_openssl_version):
dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s"
self.log.debug(dbg_msg, system_solib, openssl_version,
min_openssl_version)
self.system_ssl['libs'].append(system_solib)
else:
dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s"
self.log.debug(dbg_msg, system_solib, openssl_version,
min_openssl_version)
else:
# one of the OpenSSL libraries is missing, switch to next group of versioned libs
self.system_ssl['libs'] = []
break
if len(self.system_ssl['libs']) == len(openssl_libs):
# keep these libraries as possible targets for this installation
target_system_ssl_libs = system_versioned_libs[idx]
break
if len(self.system_ssl['libs']) == len(openssl_libs):
self.system_ssl['version'] = openssl_version
info_msg = "Found OpenSSL library version %s in host system: %s"
self.log.info(info_msg, self.system_ssl['version'],
os.path.dirname(self.system_ssl['libs'][0]))
else:
self.log.info(
"OpenSSL library not found in host system, falling back to OpenSSL in EasyBuild"
)
return
# Directory with engine libraries
lib_dir = os.path.dirname(self.system_ssl['libs'][0])
lib_engines_dir = [
os.path.join(lib_dir, 'openssl', self.target_ssl_engine),
os.path.join(lib_dir, self.target_ssl_engine),
]
for engines_path in lib_engines_dir:
if os.path.isdir(engines_path):
self.system_ssl['engines'] = engines_path
self.log.debug("Found OpenSSL engines in: %s",
self.system_ssl['engines'])
break
if not self.system_ssl['engines']:
self.log.info(
"OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild"
)
return
# Check system include paths for OpenSSL headers
cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null"
(out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False)
sys_include_dirs = []
for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE):
sys_include_dirs.extend(match.groups())
self.log.debug(
"Found the following include directories in host system: %s",
', '.join(sys_include_dirs))
# headers are located in 'include/openssl' by default
ssl_include_subdirs = ['openssl']
if self.majmin_version == '1.1':
# but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7
# prefer 'include/openssl' as long as the version of headers matches
ssl_include_subdirs.append(
os.path.join('openssl11', self.name.lower()))
ssl_include_dirs = [
os.path.join(incd, subd) for incd in sys_include_dirs
for subd in ssl_include_subdirs
]
ssl_include_dirs = [
include for include in ssl_include_dirs if os.path.isdir(include)
]
# find location of header files for this version of the OpenSSL libraries
for include_dir in ssl_include_dirs:
opensslv_path = os.path.join(include_dir, 'opensslv.h')
self.log.debug("Checking OpenSSL version in %s...", opensslv_path)
if os.path.exists(opensslv_path):
# check version reported by opensslv.h
opensslv = read_file(opensslv_path)
try:
header_version = openssl_version_regex.search(
opensslv).group(1)
except AttributeError:
err_msg = "System OpenSSL header '%s' does not contain any recognizable version string"
raise EasyBuildError(err_msg, opensslv_path)
if header_version == self.system_ssl['version']:
self.system_ssl['include'] = include_dir
info_msg = "Found OpenSSL headers v%s in host system: %s"
self.log.info(info_msg, header_version,
self.system_ssl['include'])
break
else:
dbg_msg = "System OpenSSL header version '%s' doesn not match library version '%s'"
self.log.debug(dbg_msg, header_version,
self.system_ssl['version'])
else:
self.log.info("System OpenSSL header file %s not found",
opensslv_path)
if not self.system_ssl['include']:
err_msg = (
"OpenSSL v%s headers not found in host system, but libraries for v%s are present. "
"Install the development package of OpenSSL for your system or force building OpenSSL from "
"source in EasyBuild by setting 'wrap_system_openssl = False' in the OpenSSL easyconfig."
)
raise EasyBuildError(err_msg, self.version,
self.system_ssl['version'])
# Check system OpenSSL binary
if self.majmin_version == '1.1':
# prefer 'openssl11' over 'openssl' with v1.1
self.system_ssl['bin'] = which('openssl11')
if not self.system_ssl['bin']:
self.system_ssl['bin'] = which('openssl')
if self.system_ssl['bin']:
self.log.info("System OpenSSL binary found: %s",
self.system_ssl['bin'])
else:
self.log.info("System OpenSSL binary not found!")
return
# system OpenSSL is fine, change target libraries to the ones found in it
self.target_ssl_libs = target_system_ssl_libs
self.log.info("Target system OpenSSL libraries: %s",
self.target_ssl_libs)
def __init__(self, *args, **kwargs):
"""Locate the installation files of OpenSSL in the host system"""
super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs)
# Libraries packaged in OpenSSL
openssl_libs = ['libssl', 'libcrypto']
# list of relevant library extensions per system and version of OpenSSL
# the first item should be the extension of an installation from source,
# it will be used in the sanity checks of the component
openssl_libext = {
'1.0': {
LINUX: ('so.1.0.0', 'so.10'),
DARWIN: ('1.0.dylib', ),
},
'1.1': {
LINUX: ('so.1.1', ),
DARWIN: ('1.1.dylib', ),
},
}
os_type = get_os_type()
if self.version in openssl_libext and os_type in openssl_libext[
self.version]:
# generate matrix of versioned .so filenames
system_versioned_libs = [[
'%s.%s' % (lib, ext)
for ext in openssl_libext[self.version][os_type]
] for lib in openssl_libs]
self.log.info("Matrix of version library names: %s",
system_versioned_libs)
else:
raise EasyBuildError(
"Don't know name of OpenSSL system library for version %s and OS type %s",
self.version, os_type)
# by default target the first option of each OpenSSL library,
# which corresponds to installation from source
self.target_ssl_libs = [
lib_name[0] for lib_name in system_versioned_libs
]
self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs)
# folders containing engines libraries
openssl_engines = {
'1.0': 'engines',
'1.1': 'engines-1.1',
}
self.target_ssl_engine = openssl_engines[self.version]
# Paths to system libraries and headers of OpenSSL
self.system_ssl = {
'bin': None,
'engines': None,
'include': None,
'lib': None,
}
# early return when we're not wrapping the system OpenSSL installation
if not self.cfg.get('wrap_system_openssl'):
self.log.info("Not wrapping system OpenSSL installation!")
return
# Check the system libraries of OpenSSL
# (only needs first library in openssl_libs to find path to libraries)
for idx, libssl in enumerate(system_versioned_libs[0]):
self.system_ssl['lib'] = find_library_path(libssl)
if self.system_ssl['lib']:
# change target libraries to the ones found in the system
self.target_ssl_libs = [
lib_name[idx] for lib_name in system_versioned_libs
]
self.log.info("Target system OpenSSL libraries: %s",
self.target_ssl_libs)
break
if self.system_ssl['lib']:
self.log.info("Found library '%s' in: %s", openssl_libs[0],
self.system_ssl['lib'])
else:
self.log.info(
"OpenSSL library '%s' not found, falling back to OpenSSL in EasyBuild",
openssl_libs[0])
# Directory with engine libraries
if self.system_ssl['lib']:
lib_dir = os.path.dirname(self.system_ssl['lib'])
lib_engines_dir = [
os.path.join(lib_dir, 'openssl', self.target_ssl_engine),
os.path.join(lib_dir, self.target_ssl_engine),
]
for engines_path in lib_engines_dir:
if os.path.isdir(engines_path):
self.system_ssl['engines'] = engines_path
self.log.debug("Found OpenSSL engines in: %s",
self.system_ssl['engines'])
break
if not self.system_ssl['engines']:
self.system_ssl['lib'] = None
self.log.info(
"OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild"
)
# Check system include paths for OpenSSL headers
cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null"
(out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False)
sys_include_dirs = []
for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE):
sys_include_dirs.extend(match.groups())
self.log.debug(
"Found the following include directories in host system: %s",
', '.join(sys_include_dirs))
# headers are located in 'include/openssl' by default
ssl_include_subdirs = [self.name.lower()]
if self.version == '1.1':
# but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7
# prefer 'include/openssl' as long as the version of headers matches
ssl_include_subdirs.append(
os.path.join('openssl11', self.name.lower()))
ssl_include_dirs = [
os.path.join(incd, subd) for incd in sys_include_dirs
for subd in ssl_include_subdirs
]
ssl_include_dirs = [
include for include in ssl_include_dirs if os.path.isdir(include)
]
# find location of header files, verify that the headers match our OpenSSL version
openssl_version_regex = re.compile(
r"SHLIB_VERSION_NUMBER\s\"([0-9]+\.[0-9]+)", re.M)
for include_dir in ssl_include_dirs:
opensslv_path = os.path.join(include_dir, 'opensslv.h')
self.log.debug("Checking OpenSSL version in %s...", opensslv_path)
if os.path.exists(opensslv_path):
opensslv = read_file(opensslv_path)
header_majmin_version = openssl_version_regex.search(opensslv)
if header_majmin_version:
header_majmin_version = header_majmin_version.group(1)
if re.match('^' + header_majmin_version, self.version):
self.system_ssl['include'] = include_dir
self.log.info(
"Found OpenSSL headers in host system: %s",
self.system_ssl['include'])
break
else:
self.log.debug(
"Header major/minor version '%s' doesn't match with %s",
header_majmin_version, self.version)
else:
self.log.debug("Pattern '%s' not found in %s",
openssl_version_regex.pattern,
opensslv_path)
else:
self.log.info("OpenSSL header file %s not found",
opensslv_path)
if not self.system_ssl['include']:
self.log.info(
"OpenSSL headers not found in host system, falling back to OpenSSL in EasyBuild"
)
# Check system OpenSSL binary
if self.version == '1.1':
# prefer 'openssl11' over 'openssl' with v1.1
self.system_ssl['bin'] = which('openssl11')
if not self.system_ssl['bin']:
self.system_ssl['bin'] = which(self.name.lower())
if self.system_ssl['bin']:
self.log.info("System OpenSSL binary found: %s",
self.system_ssl['bin'])
else:
self.log.info("System OpenSSL binary not found!")