def get_lib_tester() -> LibTestBase:
if is_mac():
return LibTestMac()
if is_linux():
return LibTestLinux()
fatal(f"Unsupported platform: {platform.system()}")
def run(self) -> None:
self.init_regex()
heading("Scanning installed executables and libraries...")
test_pass = True
# files to examine are much reduced if we look only at bin and lib directories
dir_pattern = re.compile('^(lib|libcxx|[s]bin)$')
dirs = [os.path.join(self.tp_installed_dir, type) for type in BUILD_TYPES]
for installed_dir in dirs:
if not os.path.isdir(installed_dir):
logging.info("Directory %s does not exist, skipping", installed_dir)
continue
with os.scandir(installed_dir) as candidate_dirs:
for candidate in candidate_dirs:
if dir_pattern.match(candidate.name):
examine_path = os.path.join(installed_dir, candidate.name)
for dirpath, dirnames, files in os.walk(examine_path):
for file_name in files:
full_path = os.path.join(dirpath, file_name)
if os.path.islink(full_path):
continue
if not self.good_libs(full_path):
test_pass = False
if not test_pass:
fatal(f"Found problematic library dependencies, using tool: {self.tool}")
else:
log("No problems found with library dependencies.")
def select_dependencies_to_build(self) -> None:
self.selected_dependencies = []
if self.args.dependencies:
names = set([dep.name for dep in self.dependencies])
for dep in self.args.dependencies:
if dep not in names:
fatal(
"Unknown dependency name: %s. Valid dependency names:\n%s",
dep, (" " * 4 + ("\n" + " " * 4).join(sorted(names))))
for dep in self.dependencies:
if dep.name in self.args.dependencies:
self.selected_dependencies.append(dep)
elif self.args.skip:
skipped = set(self.args.skip.split(','))
log("Skipping dependencies: %s", sorted(skipped))
self.selected_dependencies = []
for dependency in self.dependencies:
if dependency.name in skipped:
skipped.remove(dependency.name)
else:
self.selected_dependencies.append(dependency)
if skipped:
raise ValueError("Unknown dependencies, cannot skip: %s" %
sorted(skipped))
else:
self.selected_dependencies = self.dependencies
def get_build_stamp_for_dependency(self, dep: Dependency) -> str:
module_name = dep.__class__.__module__
assert isinstance(module_name, str), "Dependency's module is not a string: %s" % module_name
assert module_name.startswith('build_definitions.'), "Invalid module name: %s" % module_name
module_name_components = module_name.split('.')
assert len(module_name_components) == 2, (
"Expected two components: %s" % module_name_components)
module_name_final = module_name_components[-1]
input_files_for_stamp = [
'python/yugabyte_db_thirdparty/yb_build_thirdparty_main.py',
'build_thirdparty.sh',
os.path.join('python', 'build_definitions', '%s.py' % module_name_final)
]
for path in input_files_for_stamp:
abs_path = os.path.join(YB_THIRDPARTY_DIR, path)
if not os.path.exists(abs_path):
fatal("File '%s' does not exist -- expecting it to exist when creating a 'stamp' "
"for the build configuration of '%s'.", abs_path, dep.name)
with PushDir(YB_THIRDPARTY_DIR):
git_commit_sha1 = subprocess.check_output(
['git', 'log', '--pretty=%H', '-n', '1'] + input_files_for_stamp
).strip().decode('utf-8')
build_stamp = 'git_commit_sha1={}\n'.format(git_commit_sha1)
for git_extra_arg in (None, '--cached'):
git_extra_args = [git_extra_arg] if git_extra_arg else []
git_diff = subprocess.check_output(
['git', 'diff'] + git_extra_args + input_files_for_stamp)
git_diff_sha256 = hashlib.sha256(git_diff).hexdigest()
build_stamp += 'git_diff_sha256{}={}\n'.format(
'_'.join(git_extra_args).replace('--', '_'),
git_diff_sha256)
return build_stamp
def mkdir_if_missing(path: str) -> None:
if os.path.exists(path):
if not os.path.isdir(path):
fatal("Trying to create dir {}, but file with the same path already exists"
.format(path))
return
os.makedirs(path)
def init_compiler_independent_flags(self, dep: Dependency) -> None:
"""
Initialize compiler and linker flags for a particular build type. We try to limit this
function to flags that will work for most compilers we are using, which include various
versions of GCC and Clang.
"""
self.preprocessor_flags = []
self.ld_flags = []
self.executable_only_ld_flags = []
self.compiler_flags = []
self.c_flags = []
self.cxx_flags = []
self.libs = []
self.add_linuxbrew_flags()
for include_dir_component in set([BUILD_TYPE_COMMON, self.build_type]):
self.add_include_path(
os.path.join(self.fs_layout.tp_installed_dir,
include_dir_component, 'include'))
self.add_lib_dir_and_rpath(
os.path.join(self.fs_layout.tp_installed_dir,
include_dir_component, 'lib'))
self.compiler_flags += self.preprocessor_flags
# -fPIC is there to always generate position-independent code, even for static libraries.
self.compiler_flags += [
'-fno-omit-frame-pointer', '-fPIC', '-O2', '-Wall'
]
if is_linux():
# On Linux, ensure we set a long enough rpath so we can change it later with chrpath or
# a similar tool.
self.add_rpath(PLACEHOLDER_RPATH)
self.dylib_suffix = "so"
elif is_mac():
self.dylib_suffix = "dylib"
# YugaByte builds with C++11, which on OS X requires using libc++ as the standard
# library implementation. Some of the dependencies do not compile against libc++ by
# default, so we specify it explicitly.
self.cxx_flags.append("-stdlib=libc++")
self.ld_flags += ["-lc++", "-lc++abi"]
# Build for macOS Mojave or later. See https://bit.ly/37myHbk
self.compiler_flags.append("-mmacosx-version-min=10.14")
self.ld_flags.append("-Wl,-headerpad_max_install_names")
else:
fatal("Unsupported platform: {}".format(platform.system()))
# The C++ standard must match CMAKE_CXX_STANDARD in the top-level CMakeLists.txt file in
# the YugabyteDB source tree.
self.cxx_flags.append('-std=c++14')
self.cxx_flags.append('-frtti')
if self.build_type == BUILD_TYPE_ASAN:
self.compiler_flags += ASAN_FLAGS
if self.build_type == BUILD_TYPE_TSAN:
self.compiler_flags += TSAN_FLAGS
def verify_checksum(self, file_name: str, expected_checksum: Optional[str]) -> bool:
real_checksum = compute_file_sha256(file_name)
file_basename = os.path.basename(file_name)
if expected_checksum is None:
fatal(
f"No expected checksum provided for file '{file_basename}'. Consider adding the "
f"following line to thirdparty_src_checksums.txt (or re-run with --add-checksum):\n"
f"{real_checksum} {file_basename}\n"
)
return real_checksum == expected_checksum
def create_build_dir_and_prepare(self, dep: Dependency) -> str:
src_dir = self.fs_layout.get_source_path(dep)
if not os.path.isdir(src_dir):
fatal("Directory '{}' does not exist".format(src_dir))
build_dir = self.fs_layout.get_build_dir_for_dependency(dep, self.build_type)
mkdir_if_missing(build_dir)
if dep.copy_sources:
log("Bootstrapping %s from %s", build_dir, src_dir)
subprocess.check_call(['rsync', '-a', src_dir + '/', build_dir])
return build_dir
def load_expected_checksums(self) -> None:
if not os.path.exists(self.checksum_file_path):
fatal("Expected checksum file not found at %s", self.checksum_file_path)
self.filename2checksum = {}
with open(self.checksum_file_path, 'rt') as inp:
for line in inp:
line = line.strip()
if not line or line.startswith('#'):
continue
sum, fname = line.split(None, 1)
if not re.match('^[0-9a-f]{64}$', sum):
fatal("Invalid checksum: '%s' for archive name: '%s' in %s. Expected to be a "
"SHA-256 sum (64 hex characters).", sum, fname, self.checksum_file_path)
self.filename2checksum[fname] = sum
def _do_find_gcc(self, c_compiler: str,
cxx_compiler: str) -> Tuple[str, str]:
if self.using_linuxbrew():
gcc_dir = self.get_linuxbrew_dir()
elif self.compiler_prefix:
gcc_dir = self.compiler_prefix
else:
c_compiler_path = which_must_exist(c_compiler)
cxx_compiler_path = which_must_exist(cxx_compiler)
return c_compiler_path, cxx_compiler_path
gcc_bin_dir = os.path.join(gcc_dir, 'bin')
if not os.path.isdir(gcc_bin_dir):
fatal("Directory {} does not exist".format(gcc_bin_dir))
return (os.path.join(gcc_bin_dir, 'gcc') + self.compiler_suffix,
os.path.join(gcc_bin_dir, 'g++') + self.compiler_suffix)
def get_expected_checksum_and_maybe_add_to_file(
self, filename: str, downloaded_path: str) -> str:
if filename not in self.filename2checksum:
if self.should_add_checksum:
with open(self.checksum_file_path, 'rt') as inp:
lines = inp.readlines()
lines = [line.rstrip() for line in lines]
checksum = compute_file_sha256(downloaded_path)
lines.append("%s %s" % (checksum, filename))
with open(self.checksum_file_path, 'wt') as out:
for line in lines:
out.write(line + "\n")
self.filename2checksum[filename] = checksum
log("Added checksum for %s to %s: %s", filename, self.checksum_file_path, checksum)
return checksum
fatal("No expected checksum provided for {}".format(filename))
return self.filename2checksum[filename]
def find_compiler_by_type(self, compiler_type: str) -> None:
compilers: Tuple[str, str]
if compiler_type == 'gcc':
if self.use_only_clang():
raise ValueError('Not allowed to use GCC')
compilers = self.find_gcc()
elif compiler_type == 'clang':
if self.use_only_gcc():
raise ValueError('Not allowed to use Clang')
compilers = self.find_clang()
else:
fatal("Unknown compiler type {}".format(compiler_type))
assert len(compilers) == 2
for compiler in compilers:
if compiler is None or not os.path.exists(compiler):
fatal("Compiler executable does not exist: {}".format(compiler))
self.cc = compilers[0]
self.validate_compiler_path(self.cc)
self.cxx = compilers[1]
self.validate_compiler_path(self.cxx)
def find_clang(self) -> Tuple[str, str]:
clang_prefix: Optional[str] = None
if self.compiler_prefix:
clang_prefix = self.compiler_prefix
else:
candidate_dirs = [
os.path.join(YB_THIRDPARTY_DIR, 'clang-toolchain'),
'/usr'
]
for dir in candidate_dirs:
bin_dir = os.path.join(dir, 'bin')
if os.path.exists(os.path.join(bin_dir, 'clang' + self.compiler_suffix)):
clang_prefix = dir
break
if clang_prefix is None:
fatal("Failed to find clang at the following locations: {}".format(candidate_dirs))
assert clang_prefix is not None
clang_bin_dir = os.path.join(clang_prefix, 'bin')
return (os.path.join(clang_bin_dir, 'clang') + self.compiler_suffix,
os.path.join(clang_bin_dir, 'clang++') + self.compiler_suffix)
def extract_archive(
self,
archive_file_name: str,
out_dir: str,
out_name: Optional[str] = None) -> None:
"""
Extract the given archive into a subdirectory of out_dir, optionally renaming it to
the specified name out_name. The archive is expected to contain exactly one directory.
If out_name is not specified, the name of the directory inside the archive becomes
the name of the destination directory.
out_dir is the parent directory that should contain the extracted directory when the
function returns.
"""
def dest_dir_already_exists(full_out_path: str) -> bool:
if os.path.exists(full_out_path):
log("Directory already exists: %s, skipping extracting %s" % (
full_out_path, archive_file_name))
return True
return False
full_out_path = None
if out_name:
full_out_path = os.path.join(out_dir, out_name)
if dest_dir_already_exists(full_out_path):
return
# Extract the archive into a temporary directory.
tmp_out_dir = os.path.join(
out_dir, 'tmp-extract-%s-%s' % (
os.path.basename(archive_file_name),
get_temporal_randomized_file_name_suffix()
))
if os.path.exists(tmp_out_dir):
raise IOError("Just-generated unique directory name already exists: %s" % tmp_out_dir)
os.makedirs(tmp_out_dir)
archive_extension = None
for ext in ARCHIVE_TYPES:
if archive_file_name.endswith(ext):
archive_extension = ext
break
if not archive_extension:
fatal("Unknown archive type for: {}".format(archive_file_name))
assert archive_extension is not None
try:
with PushDir(tmp_out_dir):
cmd = ARCHIVE_TYPES[archive_extension].format(archive_file_name)
log("Extracting %s in temporary directory %s", cmd, tmp_out_dir)
subprocess.check_call(cmd, shell=True)
extracted_subdirs = [
subdir_name for subdir_name in os.listdir(tmp_out_dir)
if not subdir_name.startswith('.')
]
if len(extracted_subdirs) != 1:
raise IOError(
"Expected the extracted archive %s to contain exactly one "
"subdirectory and no files, found: %s" % (
archive_file_name, extracted_subdirs))
extracted_subdir_basename = extracted_subdirs[0]
extracted_subdir_path = os.path.join(tmp_out_dir, extracted_subdir_basename)
if not os.path.isdir(extracted_subdir_path):
raise IOError(
"This is a file, expected it to be a directory: %s" %
extracted_subdir_path)
if not full_out_path:
full_out_path = os.path.join(out_dir, extracted_subdir_basename)
if dest_dir_already_exists(full_out_path):
return
log("Moving %s to %s", extracted_subdir_path, full_out_path)
shutil.move(extracted_subdir_path, full_out_path)
finally:
log("Removing temporary directory: %s", tmp_out_dir)
shutil.rmtree(tmp_out_dir)
def download_dependency(
self,
dep: Dependency,
src_path: str,
archive_path: Optional[str]) -> None:
patch_level_path = os.path.join(src_path, 'patchlevel-{}'.format(dep.patch_version))
if os.path.exists(patch_level_path):
return
download_url = dep.download_url
assert download_url is not None, "Download URL not specified for dependency %s" % dep.name
remove_path(src_path)
# If download_url is "mkdir" then we just create empty directory with specified name.
if download_url != 'mkdir':
if archive_path is None:
return
self.ensure_file_downloaded(
url=download_url,
file_path=archive_path,
enable_using_alternative_url=True)
self.extract_archive(archive_path,
os.path.dirname(src_path),
os.path.basename(src_path))
else:
log("Creating %s", src_path)
mkdir_if_missing(src_path)
if hasattr(dep, 'extra_downloads'):
for extra in dep.extra_downloads:
assert extra.archive_name is not None
archive_path = os.path.join(self.download_dir, extra.archive_name)
log("Downloading %s from %s", extra.archive_name, extra.download_url)
self.ensure_file_downloaded(
url=extra.download_url,
file_path=archive_path,
enable_using_alternative_url=True)
output_path = os.path.join(src_path, extra.dir_name)
self.extract_archive(archive_path, output_path)
if extra.post_exec is not None:
with PushDir(output_path):
assert isinstance(extra.post_exec, list)
if isinstance(extra.post_exec[0], str):
subprocess.check_call(cast(List[str], extra.post_exec))
else:
for command in extra.post_exec:
subprocess.check_call(command)
if hasattr(dep, 'patches'):
with PushDir(src_path):
for patch in dep.patches:
log("Applying patch: %s", patch)
process = subprocess.Popen(['patch', '-p{}'.format(dep.patch_strip)],
stdin=subprocess.PIPE)
with open(os.path.join(YB_THIRDPARTY_DIR, 'patches', patch), 'rt') as inp:
patch = inp.read()
assert process.stdin is not None
process.stdin.write(patch.encode('utf-8'))
process.stdin.close()
exit_code = process.wait()
if exit_code:
fatal("Patch {} failed with code: {}".format(dep.name, exit_code))
if dep.post_patch:
subprocess.check_call(dep.post_patch)
with open(patch_level_path, 'wb') as out:
# Just create an empty file.
pass
def ensure_file_downloaded(
self,
url: str,
file_path: str,
enable_using_alternative_url: bool,
expected_checksum: Optional[str] = None,
verify_checksum: bool = True) -> None:
log(f"Ensuring {url} is downloaded to path {file_path}")
file_name = os.path.basename(file_path)
mkdir_if_missing(self.download_dir)
if os.path.exists(file_path) and verify_checksum:
# We check the filename against our checksum map only if the file exists. This is done
# so that we would still download the file even if we don't know the checksum, making it
# easier to add new third-party dependencies.
if expected_checksum is None:
expected_checksum = self.get_expected_checksum_and_maybe_add_to_file(
file_name, downloaded_path=file_path)
if self.verify_checksum(file_path, expected_checksum):
log("No need to re-download %s: checksum already correct", file_name)
return
log("File %s already exists but has wrong checksum, removing", file_path)
remove_path(file_path)
log("Fetching %s from %s", file_name, url)
download_successful = False
alternative_url = ALTERNATIVE_URL_PREFIX + file_name
total_attempts = 0
url_candidates = [url]
if enable_using_alternative_url:
url_candidates += [alternative_url]
for effective_url in url_candidates:
if effective_url == alternative_url:
log("Switching to alternative download URL %s after %d attempts",
alternative_url, total_attempts)
sleep_time_sec = INITIAL_DOWNLOAD_RETRY_SLEEP_TIME_SEC
for attempt_index in range(1, MAX_FETCH_ATTEMPTS + 1):
try:
total_attempts += 1
curl_cmd_line = [
self.curl_path,
'-o',
file_path,
'-L', # follow redirects
'--silent',
'--show-error',
'--location',
effective_url]
log("Running command: %s", shlex_join(curl_cmd_line))
subprocess.check_call(curl_cmd_line)
download_successful = True
break
except subprocess.CalledProcessError as ex:
log("Error downloading %s (attempt %d for this URL, total attempts %d): %s",
self.curl_path, attempt_index, total_attempts, str(ex))
if attempt_index == MAX_FETCH_ATTEMPTS and effective_url == alternative_url:
log("Giving up after %d attempts", MAX_FETCH_ATTEMPTS)
raise ex
log("Will retry after %.1f seconds", sleep_time_sec)
time.sleep(sleep_time_sec)
sleep_time_sec += DOWNLOAD_RETRY_SLEEP_INCREASE_SEC
if download_successful:
break
if not os.path.exists(file_path):
fatal("Downloaded '%s' but but unable to find '%s'", url, file_path)
if verify_checksum:
if expected_checksum is None:
expected_checksum = self.get_expected_checksum_and_maybe_add_to_file(
file_name, downloaded_path=file_path)
if not self.verify_checksum(file_path, expected_checksum):
fatal("File '%s' has wrong checksum after downloading from '%s'. "
"Has %s, but expected: %s",
file_path,
url,
compute_file_sha256(file_path),
expected_checksum)
