def get_lib_tester() -> LibTestBase: if is_macos(): return LibTestMac() if is_linux(): return LibTestLinux() fatal(f"Unsupported platform: {platform.system()}")
def is_linux_clang1x(self) -> bool: llvm_major_version: Optional[int] = self.get_llvm_major_version() return ( not is_macos() and self.single_compiler_type == 'clang' and llvm_major_version is not None and llvm_major_version >= 10 )
def get_min_supported_macos_version() -> str: assert is_macos() target_arch = get_target_arch() if target_arch == 'x86_64': return MIN_SUPPORTED_MACOS_VERSION_X86_64 if target_arch == 'arm64': return MIN_SUPPORTED_MACOS_VERSION_ARM64 raise ValueError( "Could not determine minimum supported macOS version for target " "architecture %s" % target_arch)
def get_arch_switch_cmd_prefix() -> List[str]: """ Returns a command line prefix that will switch to the target architecture. """ if not is_macos(): return [] actual_arch = platform.machine() target_arch = get_target_arch() if actual_arch == target_arch: return [] return ['arch', '-%s' % target_arch]
def get_target_arch() -> str: global g_target_arch if g_target_arch is not None: return g_target_arch if not is_macos(): g_target_arch = platform.machine() return g_target_arch g_target_arch = os.getenv('YB_TARGET_ARCH') if g_target_arch not in MACOS_CPU_ARCHITECTURES: raise ValueError("Unsupported value of YB_TARGET_ARCH on maOS: %s" % g_target_arch) return g_target_arch
def fix_shared_library_references( install_prefix: str, lib_name_prefix: str) -> None: if not is_macos(): return lib_dir = os.path.realpath(os.path.join(install_prefix, "lib")) lib_paths = glob.glob(os.path.join(lib_dir, lib_name_prefix + "*.dylib")) bin_dir = os.path.realpath(os.path.join(install_prefix, "sbin")) bin_paths = glob.glob(os.path.join(bin_dir, "*")) for lib in lib_paths + bin_paths: if os.path.islink(lib): continue lib_basename = os.path.basename(lib) otool_output = subprocess.check_output(['otool', '-L', lib]).decode('utf-8') for line in otool_output.split('\n'): if line.startswith('\t' + lib_name_prefix): dependency_name = line.strip().split()[0] dependency_real_name = os.path.relpath( os.path.realpath(os.path.join(lib_dir, dependency_name)), lib_dir) if lib_basename in [dependency_name, dependency_real_name]: log("Making %s refer to itself using @rpath", lib) subprocess.check_call([ 'install_name_tool', '-id', '@rpath/' + dependency_name, lib ]) else: log("Making %s refer to %s using @loader_path", lib, dependency_name) subprocess.check_call([ 'install_name_tool', '-change', dependency_name, '@loader_path/' + dependency_name, lib ])
def set_compiler(self, compiler_type: str) -> None: if is_macos(): if compiler_type != 'clang': raise ValueError( "Cannot set compiler type to %s on macOS, only clang is supported" % compiler_type) self.compiler_type = 'clang' else: self.compiler_type = compiler_type self.find_compiler_by_type(compiler_type) c_compiler = self.get_c_compiler() cxx_compiler = self.get_cxx_compiler() if self.use_compiler_wrapper: os.environ['YB_THIRDPARTY_REAL_C_COMPILER'] = c_compiler os.environ['YB_THIRDPARTY_REAL_CXX_COMPILER'] = cxx_compiler os.environ[ 'YB_THIRDPARTY_USE_CCACHE'] = '1' if self.use_ccache else '0' python_scripts_dir = os.path.join(YB_THIRDPARTY_DIR, 'python', 'yugabyte_db_thirdparty') self.c_compiler_or_wrapper = os.path.join( python_scripts_dir, 'compiler_wrapper_cc.py') self.cxx_compiler_or_wrapper = os.path.join( python_scripts_dir, 'compiler_wrapper_cxx.py') else: c_compiler_or_wrapper = c_compiler cxx_compiler_or_wrapper = cxx_compiler os.environ['CC'] = c_compiler_or_wrapper os.environ['CXX'] = cxx_compiler_or_wrapper self._identify_compiler_version() log(f"C compiler: {self.cc_identification}") log(f"C++ compiler: {self.cxx_identification}") if self.expected_major_compiler_version: self.check_compiler_major_version()
def is_linux_clang1x(self) -> bool: # TODO: actually check compiler version. return (not is_macos() and self.single_compiler_type == 'clang' and not self.using_linuxbrew())
def use_only_clang(self) -> bool: return is_macos() or self.single_compiler_type == 'clang'
def parse_cmd_line_args() -> argparse.Namespace: parser = argparse.ArgumentParser(prog=sys.argv[0]) parser.add_argument( '--build-type', default=None, type=str, choices=BUILD_TYPES, help='Build only specific part of thirdparty dependencies.') parser.add_argument( '--skip-sanitizers', action='store_true', help='Do not build ASAN and TSAN instrumented dependencies.') parser.add_argument('--clean', action='store_true', default=False, help='Clean, but keep downloads.') parser.add_argument('--clean-downloads', action='store_true', default=False, help='Clean, including downloads.') parser.add_argument('--add-checksum', help='Compute and add unknown checksums to %s' % CHECKSUM_FILE_NAME, action='store_true') parser.add_argument('--skip', help='Dependencies to skip') parser.add_argument( '--single-compiler-type', type=str, choices=['gcc', 'clang'], default=None, help= 'Produce a third-party dependencies build using only a single compiler. ' 'This also implies that we are not using Linuxbrew.') parser.add_argument( '--compiler-prefix', type=str, help= 'The prefix directory for looking for compiler executables. We will look for ' 'compiler executable in the bin subdirectory of this directory.') parser.add_argument( '--compiler-suffix', type=str, default='', help= 'Suffix to append to compiler executables, such as the version number, ' 'potentially prefixed with a dash, to obtain names such as gcc-8, g++-8, ' 'clang-10, or clang++-10.') parser.add_argument('--devtoolset', type=int, help='Specifies a CentOS devtoolset') parser.add_argument( '-j', '--make-parallelism', help='How many cores should the build use. This is passed to ' 'Make/Ninja child processes. This can also be specified using the ' 'YB_MAKE_PARALLELISM environment variable.', type=int) parser.add_argument('--use-ccache', action='store_true', help='Use ccache to speed up compilation') parser.add_argument( '--use-compiler-wrapper', action='store_true', help='Use a compiler wrapper script. Allows additional validation but ' 'makes the build slower.') parser.add_argument( '--llvm-version', default=None, help='Version (tag) to use for dependencies based on LLVM codebase') parser.add_argument( '--remote-build-server', help= 'Build third-party dependencies remotely on this server. The default value is ' 'determined by YB_THIRDPARTY_REMOTE_BUILD_SERVER environment variable.', default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_SERVER')) parser.add_argument( '--remote-build-dir', help= 'The directory on the remote server to build third-party dependencies in. The ' 'value is determined by the YB_THIRDPARTY_REMOTE_BUILD_DIR environment variable.', default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_DIR')) parser.add_argument( '--local', help= 'Forces the local build even if --remote-... options are specified or the ' 'corresponding environment variables are set.', action='store_true') parser.add_argument( '--download-extract-only', help= 'Only download and extract archives. Do not build any dependencies.', action='store_true') parser.add_argument('--license-report', action='store_true', help='Generate a license report.') parser.add_argument( '--toolchain', help='Automatically download, install and use the given toolchain', choices=TOOLCHAIN_TYPES) parser.add_argument('--create-package', help='Create the package tarball', action='store_true') parser.add_argument( '--upload-as-tag', help='Upload the package tarball as a GitHub release under this tag. ' 'Implies --create-package. Requires GITHUB_TOKEN to be set. If GITHUB_TOKEN is not ' 'set, this is a no-op (with success exit code).') parser.add_argument( '--expected-major-compiler-version', type=int, help='Expect the major version of the compiler to be as specified') parser.add_argument('--verbose', help='Show verbose output', action='store_true') parser.add_argument('dependencies', nargs=argparse.REMAINDER, help='Dependencies to build.') parser.add_argument( '--enforce_arch', help= 'Ensure that we use the given architecture, such as arm64. Useful for macOS systems ' 'with Apple Silicon CPUs and Rosetta 2 installed that can switch between ' 'architectures.') parser.add_argument( '--force', help= 'Build dependencies even though the system does not detect any changes compared ' 'to an earlier completed build.', action='store_true') parser.add_argument( '--delete-build-dir', help= "Delete each dependency's build directory to start each build from scratch. " "Note that this does not affect the corresponding source directory.", action='store_true') args = parser.parse_args() # --------------------------------------------------------------------------------------------- # Validating arguments # --------------------------------------------------------------------------------------------- if args.dependencies and args.skip: raise ValueError( "--skip is not compatible with specifying a list of dependencies to build" ) if is_macos(): if args.single_compiler_type not in [None, 'clang']: raise ValueError( "--single-compiler-type=%s is not allowed on macOS" % args.single_compiler_type) args.single_compiler_type = 'clang' if args.local and (args.remote_build_server is not None or args.remote_build_dir is not None): log("Forcing a local build") args.remote_build_server = None args.remote_build_dir = None if (args.remote_build_server is None) != (args.remote_build_dir is None): raise ValueError( '--remote-build-server and --remote-build-dir have to be specified or unspecified ' 'at the same time. Note that their default values are provided by corresponding ' 'environment variables, YB_THIRDPARTY_REMOTE_BUILD_SERVER and ' 'YB_THIRDPARTY_REMOTE_BUILD_DIR.') if args.remote_build_dir is not None: assert os.path.isabs(args.remote_build_dir), ( 'Remote build directory path must be an absolute path: %s' % args.remote_build_dir) is_remote_build = args.remote_build_server is not None if args.devtoolset is not None and not is_remote_build: if not local_sys_conf().is_redhat_family(): raise ValueError( "--devtoolset can only be used on Red Hat Enterprise Linux OS family" ) if args.single_compiler_type not in [None, 'gcc']: raise ValueError( "--devtoolset is not compatible with compiler type: %s" % args.single_compiler_type) args.single_compiler_type = 'gcc' if args.toolchain: if args.devtoolset: raise ValueError("--devtoolset and --toolchain are incompatible") if args.compiler_prefix: raise ValueError( "--compiler-prefix and --toolchain are incompatible") if args.compiler_suffix: raise ValueError( "--compiler-suffix and --toolchain are incompatible") if args.enforce_arch and platform.machine() != args.enforce_arch: raise ValueError("Machine architecture is %s but we expect %s" % (platform.machine(), args.enforce_arch)) return args
def add_homebrew_to_path() -> None: """ On macOS, adds the Homebrew bin directory for the correct target architecture to PATH. """ if is_macos(): add_path_entry(HOMEBREW_BIN_DIR_BY_ARCH[get_target_arch()])
def is_macos_arm64_build() -> bool: return is_macos() and get_target_arch() == 'arm64'