def get_lib_tester() -> LibTestBase: if is_mac(): return LibTestMac() if is_linux(): return LibTestLinux() fatal(f"Unsupported platform: {platform.system()}")
def set_compiler(self, compiler_type: str) -> None: if is_mac(): if compiler_type != 'clang': raise ValueError( "Cannot set compiler type to %s on macOS, only clang is supported" % compiler_type) self.compiler_type = 'clang' else: self.compiler_type = compiler_type self.find_compiler_by_type(compiler_type) c_compiler = self.get_c_compiler() cxx_compiler = self.get_cxx_compiler() if self.use_compiler_wrapper: os.environ['YB_THIRDPARTY_REAL_C_COMPILER'] = c_compiler os.environ['YB_THIRDPARTY_REAL_CXX_COMPILER'] = cxx_compiler os.environ[ 'YB_THIRDPARTY_USE_CCACHE'] = '1' if self.use_ccache else '0' python_scripts_dir = os.path.join(YB_THIRDPARTY_DIR, 'python', 'yugabyte_db_thirdparty') os.environ['CC'] = os.path.join(python_scripts_dir, 'compiler_wrapper_cc.py') os.environ['CXX'] = os.path.join(python_scripts_dir, 'compiler_wrapper_cxx.py') else: os.environ['CC'] = c_compiler os.environ['CXX'] = cxx_compiler self._identify_compiler_version() log(f"C compiler: {self.cc_identification}") log(f"C++ compiler: {self.cxx_identification}")
def run(self) -> None: self.compiler_choice.set_compiler( 'clang' if self.compiler_choice.use_only_clang() else 'gcc') if self.args.clean or self.args.clean_downloads: self.fs_layout.clean(self.selected_dependencies, self.args.clean_downloads) self.prepare_out_dirs() os.environ['PATH'] = ':'.join([ os.path.join(self.fs_layout.tp_installed_common_dir, 'bin'), os.path.join(self.fs_layout.tp_installed_llvm7_common_dir, 'bin'), os.environ['PATH'] ]) self.build_one_build_type(BUILD_TYPE_COMMON) build_types = [] if is_linux(): build_types.append(BUILD_TYPE_UNINSTRUMENTED) if self.compiler_choice.use_only_gcc(): if is_linux() and not self.compiler_choice.using_linuxbrew(): # Starting to support ASAN for GCC compilers # (not for the current GCC 5.5 build on Linuxbrew, though). build_types.append(BUILD_TYPE_ASAN) else: if self.compiler_choice.using_linuxbrew() or is_mac(): build_types.append(BUILD_TYPE_CLANG_UNINSTRUMENTED) if is_linux() and not self.args.skip_sanitizers: build_types.append(BUILD_TYPE_ASAN) build_types.append(BUILD_TYPE_TSAN) log(f"Full list of build types: {build_types}") for build_type in build_types: self.build_one_build_type(build_type)
def init_compiler_independent_flags(self, dep: Dependency) -> None: """ Initialize compiler and linker flags for a particular build type. We try to limit this function to flags that will work for most compilers we are using, which include various versions of GCC and Clang. """ self.preprocessor_flags = [] self.ld_flags = [] self.executable_only_ld_flags = [] self.compiler_flags = [] self.c_flags = [] self.cxx_flags = [] self.libs = [] self.add_linuxbrew_flags() for include_dir_component in set([BUILD_TYPE_COMMON, self.build_type]): self.add_include_path( os.path.join(self.fs_layout.tp_installed_dir, include_dir_component, 'include')) self.add_lib_dir_and_rpath( os.path.join(self.fs_layout.tp_installed_dir, include_dir_component, 'lib')) self.compiler_flags += self.preprocessor_flags # -fPIC is there to always generate position-independent code, even for static libraries. self.compiler_flags += [ '-fno-omit-frame-pointer', '-fPIC', '-O2', '-Wall' ] if is_linux(): # On Linux, ensure we set a long enough rpath so we can change it later with chrpath or # a similar tool. self.add_rpath(PLACEHOLDER_RPATH) self.dylib_suffix = "so" elif is_mac(): self.dylib_suffix = "dylib" # YugaByte builds with C++11, which on OS X requires using libc++ as the standard # library implementation. Some of the dependencies do not compile against libc++ by # default, so we specify it explicitly. self.cxx_flags.append("-stdlib=libc++") self.ld_flags += ["-lc++", "-lc++abi"] # Build for macOS Mojave or later. See https://bit.ly/37myHbk self.compiler_flags.append("-mmacosx-version-min=10.14") self.ld_flags.append("-Wl,-headerpad_max_install_names") else: fatal("Unsupported platform: {}".format(platform.system())) # The C++ standard must match CMAKE_CXX_STANDARD in the top-level CMakeLists.txt file in # the YugabyteDB source tree. self.cxx_flags.append('-std=c++14') self.cxx_flags.append('-frtti') if self.build_type == BUILD_TYPE_ASAN: self.compiler_flags += ASAN_FLAGS if self.build_type == BUILD_TYPE_TSAN: self.compiler_flags += TSAN_FLAGS
def init_flags(self, dep: Dependency) -> None: """ Initializes compiler and linker flags. No flag customizations should be transferred from one dependency to another. """ self.init_compiler_independent_flags(dep) if not is_mac() and self.compiler_choice.building_with_clang(self.build_type): # Special setup for Clang on Linux. if self.compiler_choice.single_compiler_type == 'clang': # We are assuming that --single-compiler-type will only be used for Clang 10 and # newer. self.init_linux_clang1x_flags(dep) else: self.init_linux_clang7_flags(dep)
def fix_shared_library_references( install_prefix: str, lib_name_prefix: str) -> None: if not is_mac(): return lib_dir = os.path.realpath(os.path.join(install_prefix, "lib")) lib_paths = glob.glob(os.path.join(lib_dir, lib_name_prefix + "*.dylib")) bin_dir = os.path.realpath(os.path.join(install_prefix, "sbin")) bin_paths = glob.glob(os.path.join(bin_dir, "*")) for lib in lib_paths + bin_paths: if os.path.islink(lib): continue lib_basename = os.path.basename(lib) otool_output = subprocess.check_output(['otool', '-L', lib]).decode('utf-8') for line in otool_output.split('\n'): if line.startswith('\t' + lib_name_prefix): dependency_name = line.strip().split()[0] dependency_real_name = os.path.relpath( os.path.realpath(os.path.join(lib_dir, dependency_name)), lib_dir) if lib_basename in [dependency_name, dependency_real_name]: log("Making %s refer to itself using @rpath", lib) subprocess.check_call([ 'install_name_tool', '-id', '@rpath/' + dependency_name, lib ]) else: log("Making %s refer to %s using @loader_path", lib, dependency_name) subprocess.check_call([ 'install_name_tool', '-change', dependency_name, '@loader_path/' + dependency_name, lib ])
def parse_cmd_line_args() -> argparse.Namespace: parser = argparse.ArgumentParser(prog=sys.argv[0]) parser.add_argument( '--build-type', default=None, type=str, choices=BUILD_TYPES, help='Build only specific part of thirdparty dependencies.') parser.add_argument( '--skip-sanitizers', action='store_true', help='Do not build ASAN and TSAN instrumented dependencies.') parser.add_argument('--clean', action='store_true', default=False, help='Clean, but keep downloads.') parser.add_argument('--clean-downloads', action='store_true', default=False, help='Clean, including downloads.') parser.add_argument('--add_checksum', help='Compute and add unknown checksums to %s' % CHECKSUM_FILE_NAME, action='store_true') parser.add_argument('--skip', help='Dependencies to skip') parser.add_argument( '--single-compiler-type', type=str, choices=['gcc', 'clang'], default=None, help= 'Produce a third-party dependencies build using only a single compiler. ' 'This also implies that we are not using Linuxbrew.') parser.add_argument( '--compiler-prefix', type=str, help= 'The prefix directory for looking for compiler executables. We will look for ' 'compiler executable in the bin subdirectory of this directory.') parser.add_argument( '--compiler-suffix', type=str, default='', help= 'Suffix to append to compiler executables, such as the version number, ' 'potentially prefixed with a dash, to obtain names such as gcc-8, g++-8, ' 'clang-10, or clang++-10.') parser.add_argument('--devtoolset', type=int, help='Specifies a CentOS devtoolset') parser.add_argument( '-j', '--make-parallelism', help='How many cores should the build use. This is passed to ' 'Make/Ninja child processes. This can also be specified using the ' 'YB_MAKE_PARALLELISM environment variable.', type=int) parser.add_argument('--use-ccache', action='store_true', help='Use ccache to speed up compilation') parser.add_argument( '--use-compiler-wrapper', action='store_true', help='Use a compiler wrapper script. Allows additional validation but ' 'makes the build slower.') parser.add_argument( '--llvm-version', default=None, help='Version (tag) to use for dependencies based on LLVM codebase') parser.add_argument( '--remote-build-server', help= 'Build third-party dependencies remotely on this server. The default value is ' 'determined by YB_THIRDPARTY_REMOTE_BUILD_SERVER environment variable.', default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_SERVER')) parser.add_argument( '--remote-build-dir', help= 'The directory on the remote server to build third-party dependencies in. The ' 'value is determined by the YB_THIRDPARTY_REMOTE_BUILD_DIR environment variable.', default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_DIR')) parser.add_argument( '--local', help= 'Forces the local build even if --remote-... options are specified or the ' 'corresponding environment variables are set.', action='store_true') parser.add_argument( '--download-extract-only', help= 'Only download and extract archives. Do not build any dependencies.', action='store_true') parser.add_argument( '--multi-build', action='store_true', help= 'Build multiple configurations in parallel. Most other arguments will get ignored ' 'if this is specified. Configurations to build are taken from .circleci/config.yml. ' 'The set of configurations to build can be customized using the ' '--multi-build-conf-name-pattern flag.') parser.add_argument( '--multi-build-conf-name-pattern', help= 'Only build configurations matching this glob-style pattern, anchored on both ends. ' 'This implies --multi-build.') parser.add_argument('--license-report', action='store_true', help='Generate a license report.') parser.add_argument( '--toolchain', help='Automatically download, install and use the given toolchain', choices=TOOLCHAIN_TYPES) parser.add_argument('dependencies', nargs=argparse.REMAINDER, help='Dependencies to build.') args = parser.parse_args() # --------------------------------------------------------------------------------------------- # Validating arguments # --------------------------------------------------------------------------------------------- if args.dependencies and args.skip: raise ValueError( "--skip is not compatible with specifying a list of dependencies to build" ) if is_mac(): if args.single_compiler_type not in [None, 'clang']: raise ValueError( "--single-compiler-type=%s is not allowed on macOS" % args.single_compiler_type) args.single_compiler_type = 'clang' if args.local and (args.remote_build_server is not None or args.remote_build_dir is not None): log("Forcing a local build") args.remote_build_server = None args.remote_build_dir = None if (args.remote_build_server is None) != (args.remote_build_dir is None): raise ValueError( '--remote-build-server and --remote-build-dir have to be specified or unspecified ' 'at the same time. Note that their default values are provided by corresponding ' 'environment variables, YB_THIRDPARTY_REMOTE_BUILD_SERVER and ' 'YB_THIRDPARTY_REMOTE_BUILD_DIR.') if args.remote_build_dir is not None: assert os.path.isabs(args.remote_build_dir), ( 'Remote build directory path must be an absolute path: %s' % args.remote_build_dir) is_remote_build = args.remote_build_server is not None if args.devtoolset is not None and not is_remote_build: if not is_centos(): raise ValueError("--devtoolset can only be used on CentOS Linux") if args.single_compiler_type not in [None, 'gcc']: raise ValueError( "--devtoolset is not compatible with compiler type: %s" % args.single_compiler_type) args.single_compiler_type = 'gcc' if args.toolchain: if args.devtoolset: raise ValueError("--devtoolset and --toolchain are incompatible") if args.compiler_prefix: raise ValueError( "--compiler-prefix and --toolchain are incompatible") if args.compiler_suffix: raise ValueError( "--compiler-suffix and --toolchain are incompatible") if args.multi_build_conf_name_pattern: args.multi_build = True return args
def is_linux_clang1x(self) -> bool: # TODO: actually check compiler version. return (not is_mac() and self.single_compiler_type == 'clang' and not self.using_linuxbrew())
def use_only_clang(self) -> bool: return is_mac() or self.single_compiler_type == 'clang'