def get_lib_tester() -> LibTestBase:
if is_mac():
return LibTestMac()
if is_linux():
return LibTestLinux()
fatal(f"Unsupported platform: {platform.system()}")
def set_compiler(self, compiler_type: str) -> None:
if is_mac():
if compiler_type != 'clang':
raise ValueError(
"Cannot set compiler type to %s on macOS, only clang is supported"
% compiler_type)
self.compiler_type = 'clang'
else:
self.compiler_type = compiler_type
self.find_compiler_by_type(compiler_type)
c_compiler = self.get_c_compiler()
cxx_compiler = self.get_cxx_compiler()
if self.use_compiler_wrapper:
os.environ['YB_THIRDPARTY_REAL_C_COMPILER'] = c_compiler
os.environ['YB_THIRDPARTY_REAL_CXX_COMPILER'] = cxx_compiler
os.environ[
'YB_THIRDPARTY_USE_CCACHE'] = '1' if self.use_ccache else '0'
python_scripts_dir = os.path.join(YB_THIRDPARTY_DIR, 'python',
'yugabyte_db_thirdparty')
os.environ['CC'] = os.path.join(python_scripts_dir,
'compiler_wrapper_cc.py')
os.environ['CXX'] = os.path.join(python_scripts_dir,
'compiler_wrapper_cxx.py')
else:
os.environ['CC'] = c_compiler
os.environ['CXX'] = cxx_compiler
self._identify_compiler_version()
log(f"C compiler: {self.cc_identification}")
log(f"C++ compiler: {self.cxx_identification}")
def run(self) -> None:
self.compiler_choice.set_compiler(
'clang' if self.compiler_choice.use_only_clang() else 'gcc')
if self.args.clean or self.args.clean_downloads:
self.fs_layout.clean(self.selected_dependencies, self.args.clean_downloads)
self.prepare_out_dirs()
os.environ['PATH'] = ':'.join([
os.path.join(self.fs_layout.tp_installed_common_dir, 'bin'),
os.path.join(self.fs_layout.tp_installed_llvm7_common_dir, 'bin'),
os.environ['PATH']
])
self.build_one_build_type(BUILD_TYPE_COMMON)
build_types = []
if is_linux():
build_types.append(BUILD_TYPE_UNINSTRUMENTED)
if self.compiler_choice.use_only_gcc():
if is_linux() and not self.compiler_choice.using_linuxbrew():
# Starting to support ASAN for GCC compilers
# (not for the current GCC 5.5 build on Linuxbrew, though).
build_types.append(BUILD_TYPE_ASAN)
else:
if self.compiler_choice.using_linuxbrew() or is_mac():
build_types.append(BUILD_TYPE_CLANG_UNINSTRUMENTED)
if is_linux() and not self.args.skip_sanitizers:
build_types.append(BUILD_TYPE_ASAN)
build_types.append(BUILD_TYPE_TSAN)
log(f"Full list of build types: {build_types}")
for build_type in build_types:
self.build_one_build_type(build_type)
def init_compiler_independent_flags(self, dep: Dependency) -> None:
"""
Initialize compiler and linker flags for a particular build type. We try to limit this
function to flags that will work for most compilers we are using, which include various
versions of GCC and Clang.
"""
self.preprocessor_flags = []
self.ld_flags = []
self.executable_only_ld_flags = []
self.compiler_flags = []
self.c_flags = []
self.cxx_flags = []
self.libs = []
self.add_linuxbrew_flags()
for include_dir_component in set([BUILD_TYPE_COMMON, self.build_type]):
self.add_include_path(
os.path.join(self.fs_layout.tp_installed_dir,
include_dir_component, 'include'))
self.add_lib_dir_and_rpath(
os.path.join(self.fs_layout.tp_installed_dir,
include_dir_component, 'lib'))
self.compiler_flags += self.preprocessor_flags
# -fPIC is there to always generate position-independent code, even for static libraries.
self.compiler_flags += [
'-fno-omit-frame-pointer', '-fPIC', '-O2', '-Wall'
]
if is_linux():
# On Linux, ensure we set a long enough rpath so we can change it later with chrpath or
# a similar tool.
self.add_rpath(PLACEHOLDER_RPATH)
self.dylib_suffix = "so"
elif is_mac():
self.dylib_suffix = "dylib"
# YugaByte builds with C++11, which on OS X requires using libc++ as the standard
# library implementation. Some of the dependencies do not compile against libc++ by
# default, so we specify it explicitly.
self.cxx_flags.append("-stdlib=libc++")
self.ld_flags += ["-lc++", "-lc++abi"]
# Build for macOS Mojave or later. See https://bit.ly/37myHbk
self.compiler_flags.append("-mmacosx-version-min=10.14")
self.ld_flags.append("-Wl,-headerpad_max_install_names")
else:
fatal("Unsupported platform: {}".format(platform.system()))
# The C++ standard must match CMAKE_CXX_STANDARD in the top-level CMakeLists.txt file in
# the YugabyteDB source tree.
self.cxx_flags.append('-std=c++14')
self.cxx_flags.append('-frtti')
if self.build_type == BUILD_TYPE_ASAN:
self.compiler_flags += ASAN_FLAGS
if self.build_type == BUILD_TYPE_TSAN:
self.compiler_flags += TSAN_FLAGS
def init_flags(self, dep: Dependency) -> None:
"""
Initializes compiler and linker flags. No flag customizations should be transferred from one
dependency to another.
"""
self.init_compiler_independent_flags(dep)
if not is_mac() and self.compiler_choice.building_with_clang(self.build_type):
# Special setup for Clang on Linux.
if self.compiler_choice.single_compiler_type == 'clang':
# We are assuming that --single-compiler-type will only be used for Clang 10 and
# newer.
self.init_linux_clang1x_flags(dep)
else:
self.init_linux_clang7_flags(dep)
def fix_shared_library_references(
install_prefix: str,
lib_name_prefix: str) -> None:
if not is_mac():
return
lib_dir = os.path.realpath(os.path.join(install_prefix, "lib"))
lib_paths = glob.glob(os.path.join(lib_dir, lib_name_prefix + "*.dylib"))
bin_dir = os.path.realpath(os.path.join(install_prefix, "sbin"))
bin_paths = glob.glob(os.path.join(bin_dir, "*"))
for lib in lib_paths + bin_paths:
if os.path.islink(lib):
continue
lib_basename = os.path.basename(lib)
otool_output = subprocess.check_output(['otool', '-L', lib]).decode('utf-8')
for line in otool_output.split('\n'):
if line.startswith('\t' + lib_name_prefix):
dependency_name = line.strip().split()[0]
dependency_real_name = os.path.relpath(
os.path.realpath(os.path.join(lib_dir, dependency_name)),
lib_dir)
if lib_basename in [dependency_name, dependency_real_name]:
log("Making %s refer to itself using @rpath", lib)
subprocess.check_call([
'install_name_tool',
'-id',
'@rpath/' + dependency_name,
lib
])
else:
log("Making %s refer to %s using @loader_path",
lib, dependency_name)
subprocess.check_call([
'install_name_tool',
'-change',
dependency_name,
'@loader_path/' + dependency_name,
lib
])
def parse_cmd_line_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument(
'--build-type',
default=None,
type=str,
choices=BUILD_TYPES,
help='Build only specific part of thirdparty dependencies.')
parser.add_argument(
'--skip-sanitizers',
action='store_true',
help='Do not build ASAN and TSAN instrumented dependencies.')
parser.add_argument('--clean',
action='store_true',
default=False,
help='Clean, but keep downloads.')
parser.add_argument('--clean-downloads',
action='store_true',
default=False,
help='Clean, including downloads.')
parser.add_argument('--add_checksum',
help='Compute and add unknown checksums to %s' %
CHECKSUM_FILE_NAME,
action='store_true')
parser.add_argument('--skip', help='Dependencies to skip')
parser.add_argument(
'--single-compiler-type',
type=str,
choices=['gcc', 'clang'],
default=None,
help=
'Produce a third-party dependencies build using only a single compiler. '
'This also implies that we are not using Linuxbrew.')
parser.add_argument(
'--compiler-prefix',
type=str,
help=
'The prefix directory for looking for compiler executables. We will look for '
'compiler executable in the bin subdirectory of this directory.')
parser.add_argument(
'--compiler-suffix',
type=str,
default='',
help=
'Suffix to append to compiler executables, such as the version number, '
'potentially prefixed with a dash, to obtain names such as gcc-8, g++-8, '
'clang-10, or clang++-10.')
parser.add_argument('--devtoolset',
type=int,
help='Specifies a CentOS devtoolset')
parser.add_argument(
'-j',
'--make-parallelism',
help='How many cores should the build use. This is passed to '
'Make/Ninja child processes. This can also be specified using the '
'YB_MAKE_PARALLELISM environment variable.',
type=int)
parser.add_argument('--use-ccache',
action='store_true',
help='Use ccache to speed up compilation')
parser.add_argument(
'--use-compiler-wrapper',
action='store_true',
help='Use a compiler wrapper script. Allows additional validation but '
'makes the build slower.')
parser.add_argument(
'--llvm-version',
default=None,
help='Version (tag) to use for dependencies based on LLVM codebase')
parser.add_argument(
'--remote-build-server',
help=
'Build third-party dependencies remotely on this server. The default value is '
'determined by YB_THIRDPARTY_REMOTE_BUILD_SERVER environment variable.',
default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_SERVER'))
parser.add_argument(
'--remote-build-dir',
help=
'The directory on the remote server to build third-party dependencies in. The '
'value is determined by the YB_THIRDPARTY_REMOTE_BUILD_DIR environment variable.',
default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_DIR'))
parser.add_argument(
'--local',
help=
'Forces the local build even if --remote-... options are specified or the '
'corresponding environment variables are set.',
action='store_true')
parser.add_argument(
'--download-extract-only',
help=
'Only download and extract archives. Do not build any dependencies.',
action='store_true')
parser.add_argument(
'--multi-build',
action='store_true',
help=
'Build multiple configurations in parallel. Most other arguments will get ignored '
'if this is specified. Configurations to build are taken from .circleci/config.yml. '
'The set of configurations to build can be customized using the '
'--multi-build-conf-name-pattern flag.')
parser.add_argument(
'--multi-build-conf-name-pattern',
help=
'Only build configurations matching this glob-style pattern, anchored on both ends. '
'This implies --multi-build.')
parser.add_argument('--license-report',
action='store_true',
help='Generate a license report.')
parser.add_argument(
'--toolchain',
help='Automatically download, install and use the given toolchain',
choices=TOOLCHAIN_TYPES)
parser.add_argument('dependencies',
nargs=argparse.REMAINDER,
help='Dependencies to build.')
args = parser.parse_args()
# ---------------------------------------------------------------------------------------------
# Validating arguments
# ---------------------------------------------------------------------------------------------
if args.dependencies and args.skip:
raise ValueError(
"--skip is not compatible with specifying a list of dependencies to build"
)
if is_mac():
if args.single_compiler_type not in [None, 'clang']:
raise ValueError(
"--single-compiler-type=%s is not allowed on macOS" %
args.single_compiler_type)
args.single_compiler_type = 'clang'
if args.local and (args.remote_build_server is not None
or args.remote_build_dir is not None):
log("Forcing a local build")
args.remote_build_server = None
args.remote_build_dir = None
if (args.remote_build_server is None) != (args.remote_build_dir is None):
raise ValueError(
'--remote-build-server and --remote-build-dir have to be specified or unspecified '
'at the same time. Note that their default values are provided by corresponding '
'environment variables, YB_THIRDPARTY_REMOTE_BUILD_SERVER and '
'YB_THIRDPARTY_REMOTE_BUILD_DIR.')
if args.remote_build_dir is not None:
assert os.path.isabs(args.remote_build_dir), (
'Remote build directory path must be an absolute path: %s' %
args.remote_build_dir)
is_remote_build = args.remote_build_server is not None
if args.devtoolset is not None and not is_remote_build:
if not is_centos():
raise ValueError("--devtoolset can only be used on CentOS Linux")
if args.single_compiler_type not in [None, 'gcc']:
raise ValueError(
"--devtoolset is not compatible with compiler type: %s" %
args.single_compiler_type)
args.single_compiler_type = 'gcc'
if args.toolchain:
if args.devtoolset:
raise ValueError("--devtoolset and --toolchain are incompatible")
if args.compiler_prefix:
raise ValueError(
"--compiler-prefix and --toolchain are incompatible")
if args.compiler_suffix:
raise ValueError(
"--compiler-suffix and --toolchain are incompatible")
if args.multi_build_conf_name_pattern:
args.multi_build = True
return args
def is_linux_clang1x(self) -> bool:
# TODO: actually check compiler version.
return (not is_mac() and self.single_compiler_type == 'clang'
and not self.using_linuxbrew())
def use_only_clang(self) -> bool:
return is_mac() or self.single_compiler_type == 'clang'
