def get_lib_tester() -> LibTestBase:
if is_macos():
return LibTestMac()
if is_linux():
return LibTestLinux()
fatal(f"Unsupported platform: {platform.system()}")
def is_linux_clang1x(self) -> bool:
llvm_major_version: Optional[int] = self.get_llvm_major_version()
return (
not is_macos() and
self.single_compiler_type == 'clang' and
llvm_major_version is not None and
llvm_major_version >= 10
)
def get_min_supported_macos_version() -> str:
assert is_macos()
target_arch = get_target_arch()
if target_arch == 'x86_64':
return MIN_SUPPORTED_MACOS_VERSION_X86_64
if target_arch == 'arm64':
return MIN_SUPPORTED_MACOS_VERSION_ARM64
raise ValueError(
"Could not determine minimum supported macOS version for target "
"architecture %s" % target_arch)
def get_arch_switch_cmd_prefix() -> List[str]:
"""
Returns a command line prefix that will switch to the target architecture.
"""
if not is_macos():
return []
actual_arch = platform.machine()
target_arch = get_target_arch()
if actual_arch == target_arch:
return []
return ['arch', '-%s' % target_arch]
def get_target_arch() -> str:
global g_target_arch
if g_target_arch is not None:
return g_target_arch
if not is_macos():
g_target_arch = platform.machine()
return g_target_arch
g_target_arch = os.getenv('YB_TARGET_ARCH')
if g_target_arch not in MACOS_CPU_ARCHITECTURES:
raise ValueError("Unsupported value of YB_TARGET_ARCH on maOS: %s" % g_target_arch)
return g_target_arch
def fix_shared_library_references(
install_prefix: str,
lib_name_prefix: str) -> None:
if not is_macos():
return
lib_dir = os.path.realpath(os.path.join(install_prefix, "lib"))
lib_paths = glob.glob(os.path.join(lib_dir, lib_name_prefix + "*.dylib"))
bin_dir = os.path.realpath(os.path.join(install_prefix, "sbin"))
bin_paths = glob.glob(os.path.join(bin_dir, "*"))
for lib in lib_paths + bin_paths:
if os.path.islink(lib):
continue
lib_basename = os.path.basename(lib)
otool_output = subprocess.check_output(['otool', '-L', lib]).decode('utf-8')
for line in otool_output.split('\n'):
if line.startswith('\t' + lib_name_prefix):
dependency_name = line.strip().split()[0]
dependency_real_name = os.path.relpath(
os.path.realpath(os.path.join(lib_dir, dependency_name)),
lib_dir)
if lib_basename in [dependency_name, dependency_real_name]:
log("Making %s refer to itself using @rpath", lib)
subprocess.check_call([
'install_name_tool',
'-id',
'@rpath/' + dependency_name,
lib
])
else:
log("Making %s refer to %s using @loader_path",
lib, dependency_name)
subprocess.check_call([
'install_name_tool',
'-change',
dependency_name,
'@loader_path/' + dependency_name,
lib
])
def set_compiler(self, compiler_type: str) -> None:
if is_macos():
if compiler_type != 'clang':
raise ValueError(
"Cannot set compiler type to %s on macOS, only clang is supported"
% compiler_type)
self.compiler_type = 'clang'
else:
self.compiler_type = compiler_type
self.find_compiler_by_type(compiler_type)
c_compiler = self.get_c_compiler()
cxx_compiler = self.get_cxx_compiler()
if self.use_compiler_wrapper:
os.environ['YB_THIRDPARTY_REAL_C_COMPILER'] = c_compiler
os.environ['YB_THIRDPARTY_REAL_CXX_COMPILER'] = cxx_compiler
os.environ[
'YB_THIRDPARTY_USE_CCACHE'] = '1' if self.use_ccache else '0'
python_scripts_dir = os.path.join(YB_THIRDPARTY_DIR, 'python',
'yugabyte_db_thirdparty')
self.c_compiler_or_wrapper = os.path.join(
python_scripts_dir, 'compiler_wrapper_cc.py')
self.cxx_compiler_or_wrapper = os.path.join(
python_scripts_dir, 'compiler_wrapper_cxx.py')
else:
c_compiler_or_wrapper = c_compiler
cxx_compiler_or_wrapper = cxx_compiler
os.environ['CC'] = c_compiler_or_wrapper
os.environ['CXX'] = cxx_compiler_or_wrapper
self._identify_compiler_version()
log(f"C compiler: {self.cc_identification}")
log(f"C++ compiler: {self.cxx_identification}")
if self.expected_major_compiler_version:
self.check_compiler_major_version()
def is_linux_clang1x(self) -> bool:
# TODO: actually check compiler version.
return (not is_macos() and self.single_compiler_type == 'clang'
and not self.using_linuxbrew())
def use_only_clang(self) -> bool:
return is_macos() or self.single_compiler_type == 'clang'
def parse_cmd_line_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument(
'--build-type',
default=None,
type=str,
choices=BUILD_TYPES,
help='Build only specific part of thirdparty dependencies.')
parser.add_argument(
'--skip-sanitizers',
action='store_true',
help='Do not build ASAN and TSAN instrumented dependencies.')
parser.add_argument('--clean',
action='store_true',
default=False,
help='Clean, but keep downloads.')
parser.add_argument('--clean-downloads',
action='store_true',
default=False,
help='Clean, including downloads.')
parser.add_argument('--add-checksum',
help='Compute and add unknown checksums to %s' %
CHECKSUM_FILE_NAME,
action='store_true')
parser.add_argument('--skip', help='Dependencies to skip')
parser.add_argument(
'--single-compiler-type',
type=str,
choices=['gcc', 'clang'],
default=None,
help=
'Produce a third-party dependencies build using only a single compiler. '
'This also implies that we are not using Linuxbrew.')
parser.add_argument(
'--compiler-prefix',
type=str,
help=
'The prefix directory for looking for compiler executables. We will look for '
'compiler executable in the bin subdirectory of this directory.')
parser.add_argument(
'--compiler-suffix',
type=str,
default='',
help=
'Suffix to append to compiler executables, such as the version number, '
'potentially prefixed with a dash, to obtain names such as gcc-8, g++-8, '
'clang-10, or clang++-10.')
parser.add_argument('--devtoolset',
type=int,
help='Specifies a CentOS devtoolset')
parser.add_argument(
'-j',
'--make-parallelism',
help='How many cores should the build use. This is passed to '
'Make/Ninja child processes. This can also be specified using the '
'YB_MAKE_PARALLELISM environment variable.',
type=int)
parser.add_argument('--use-ccache',
action='store_true',
help='Use ccache to speed up compilation')
parser.add_argument(
'--use-compiler-wrapper',
action='store_true',
help='Use a compiler wrapper script. Allows additional validation but '
'makes the build slower.')
parser.add_argument(
'--llvm-version',
default=None,
help='Version (tag) to use for dependencies based on LLVM codebase')
parser.add_argument(
'--remote-build-server',
help=
'Build third-party dependencies remotely on this server. The default value is '
'determined by YB_THIRDPARTY_REMOTE_BUILD_SERVER environment variable.',
default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_SERVER'))
parser.add_argument(
'--remote-build-dir',
help=
'The directory on the remote server to build third-party dependencies in. The '
'value is determined by the YB_THIRDPARTY_REMOTE_BUILD_DIR environment variable.',
default=os.getenv('YB_THIRDPARTY_REMOTE_BUILD_DIR'))
parser.add_argument(
'--local',
help=
'Forces the local build even if --remote-... options are specified or the '
'corresponding environment variables are set.',
action='store_true')
parser.add_argument(
'--download-extract-only',
help=
'Only download and extract archives. Do not build any dependencies.',
action='store_true')
parser.add_argument('--license-report',
action='store_true',
help='Generate a license report.')
parser.add_argument(
'--toolchain',
help='Automatically download, install and use the given toolchain',
choices=TOOLCHAIN_TYPES)
parser.add_argument('--create-package',
help='Create the package tarball',
action='store_true')
parser.add_argument(
'--upload-as-tag',
help='Upload the package tarball as a GitHub release under this tag. '
'Implies --create-package. Requires GITHUB_TOKEN to be set. If GITHUB_TOKEN is not '
'set, this is a no-op (with success exit code).')
parser.add_argument(
'--expected-major-compiler-version',
type=int,
help='Expect the major version of the compiler to be as specified')
parser.add_argument('--verbose',
help='Show verbose output',
action='store_true')
parser.add_argument('dependencies',
nargs=argparse.REMAINDER,
help='Dependencies to build.')
parser.add_argument(
'--enforce_arch',
help=
'Ensure that we use the given architecture, such as arm64. Useful for macOS systems '
'with Apple Silicon CPUs and Rosetta 2 installed that can switch between '
'architectures.')
parser.add_argument(
'--force',
help=
'Build dependencies even though the system does not detect any changes compared '
'to an earlier completed build.',
action='store_true')
parser.add_argument(
'--delete-build-dir',
help=
"Delete each dependency's build directory to start each build from scratch. "
"Note that this does not affect the corresponding source directory.",
action='store_true')
args = parser.parse_args()
# ---------------------------------------------------------------------------------------------
# Validating arguments
# ---------------------------------------------------------------------------------------------
if args.dependencies and args.skip:
raise ValueError(
"--skip is not compatible with specifying a list of dependencies to build"
)
if is_macos():
if args.single_compiler_type not in [None, 'clang']:
raise ValueError(
"--single-compiler-type=%s is not allowed on macOS" %
args.single_compiler_type)
args.single_compiler_type = 'clang'
if args.local and (args.remote_build_server is not None
or args.remote_build_dir is not None):
log("Forcing a local build")
args.remote_build_server = None
args.remote_build_dir = None
if (args.remote_build_server is None) != (args.remote_build_dir is None):
raise ValueError(
'--remote-build-server and --remote-build-dir have to be specified or unspecified '
'at the same time. Note that their default values are provided by corresponding '
'environment variables, YB_THIRDPARTY_REMOTE_BUILD_SERVER and '
'YB_THIRDPARTY_REMOTE_BUILD_DIR.')
if args.remote_build_dir is not None:
assert os.path.isabs(args.remote_build_dir), (
'Remote build directory path must be an absolute path: %s' %
args.remote_build_dir)
is_remote_build = args.remote_build_server is not None
if args.devtoolset is not None and not is_remote_build:
if not local_sys_conf().is_redhat_family():
raise ValueError(
"--devtoolset can only be used on Red Hat Enterprise Linux OS family"
)
if args.single_compiler_type not in [None, 'gcc']:
raise ValueError(
"--devtoolset is not compatible with compiler type: %s" %
args.single_compiler_type)
args.single_compiler_type = 'gcc'
if args.toolchain:
if args.devtoolset:
raise ValueError("--devtoolset and --toolchain are incompatible")
if args.compiler_prefix:
raise ValueError(
"--compiler-prefix and --toolchain are incompatible")
if args.compiler_suffix:
raise ValueError(
"--compiler-suffix and --toolchain are incompatible")
if args.enforce_arch and platform.machine() != args.enforce_arch:
raise ValueError("Machine architecture is %s but we expect %s" %
(platform.machine(), args.enforce_arch))
return args
def add_homebrew_to_path() -> None:
"""
On macOS, adds the Homebrew bin directory for the correct target architecture to PATH.
"""
if is_macos():
add_path_entry(HOMEBREW_BIN_DIR_BY_ARCH[get_target_arch()])
def is_macos_arm64_build() -> bool:
return is_macos() and get_target_arch() == 'arm64'