def get_lib_tester() -> LibTestBase: if is_mac(): return LibTestMac() if is_linux(): return LibTestLinux() fatal(f"Unsupported platform: {platform.system()}")
def run(self) -> None: self.init_regex() heading("Scanning installed executables and libraries...") test_pass = True # files to examine are much reduced if we look only at bin and lib directories dir_pattern = re.compile('^(lib|libcxx|[s]bin)$') dirs = [os.path.join(self.tp_installed_dir, type) for type in BUILD_TYPES] for installed_dir in dirs: if not os.path.isdir(installed_dir): logging.info("Directory %s does not exist, skipping", installed_dir) continue with os.scandir(installed_dir) as candidate_dirs: for candidate in candidate_dirs: if dir_pattern.match(candidate.name): examine_path = os.path.join(installed_dir, candidate.name) for dirpath, dirnames, files in os.walk(examine_path): for file_name in files: full_path = os.path.join(dirpath, file_name) if os.path.islink(full_path): continue if not self.good_libs(full_path): test_pass = False if not test_pass: fatal(f"Found problematic library dependencies, using tool: {self.tool}") else: log("No problems found with library dependencies.")
def select_dependencies_to_build(self) -> None: self.selected_dependencies = [] if self.args.dependencies: names = set([dep.name for dep in self.dependencies]) for dep in self.args.dependencies: if dep not in names: fatal( "Unknown dependency name: %s. Valid dependency names:\n%s", dep, (" " * 4 + ("\n" + " " * 4).join(sorted(names)))) for dep in self.dependencies: if dep.name in self.args.dependencies: self.selected_dependencies.append(dep) elif self.args.skip: skipped = set(self.args.skip.split(',')) log("Skipping dependencies: %s", sorted(skipped)) self.selected_dependencies = [] for dependency in self.dependencies: if dependency.name in skipped: skipped.remove(dependency.name) else: self.selected_dependencies.append(dependency) if skipped: raise ValueError("Unknown dependencies, cannot skip: %s" % sorted(skipped)) else: self.selected_dependencies = self.dependencies
def get_build_stamp_for_dependency(self, dep: Dependency) -> str: module_name = dep.__class__.__module__ assert isinstance(module_name, str), "Dependency's module is not a string: %s" % module_name assert module_name.startswith('build_definitions.'), "Invalid module name: %s" % module_name module_name_components = module_name.split('.') assert len(module_name_components) == 2, ( "Expected two components: %s" % module_name_components) module_name_final = module_name_components[-1] input_files_for_stamp = [ 'python/yugabyte_db_thirdparty/yb_build_thirdparty_main.py', 'build_thirdparty.sh', os.path.join('python', 'build_definitions', '%s.py' % module_name_final) ] for path in input_files_for_stamp: abs_path = os.path.join(YB_THIRDPARTY_DIR, path) if not os.path.exists(abs_path): fatal("File '%s' does not exist -- expecting it to exist when creating a 'stamp' " "for the build configuration of '%s'.", abs_path, dep.name) with PushDir(YB_THIRDPARTY_DIR): git_commit_sha1 = subprocess.check_output( ['git', 'log', '--pretty=%H', '-n', '1'] + input_files_for_stamp ).strip().decode('utf-8') build_stamp = 'git_commit_sha1={}\n'.format(git_commit_sha1) for git_extra_arg in (None, '--cached'): git_extra_args = [git_extra_arg] if git_extra_arg else [] git_diff = subprocess.check_output( ['git', 'diff'] + git_extra_args + input_files_for_stamp) git_diff_sha256 = hashlib.sha256(git_diff).hexdigest() build_stamp += 'git_diff_sha256{}={}\n'.format( '_'.join(git_extra_args).replace('--', '_'), git_diff_sha256) return build_stamp
def mkdir_if_missing(path: str) -> None: if os.path.exists(path): if not os.path.isdir(path): fatal("Trying to create dir {}, but file with the same path already exists" .format(path)) return os.makedirs(path)
def init_compiler_independent_flags(self, dep: Dependency) -> None: """ Initialize compiler and linker flags for a particular build type. We try to limit this function to flags that will work for most compilers we are using, which include various versions of GCC and Clang. """ self.preprocessor_flags = [] self.ld_flags = [] self.executable_only_ld_flags = [] self.compiler_flags = [] self.c_flags = [] self.cxx_flags = [] self.libs = [] self.add_linuxbrew_flags() for include_dir_component in set([BUILD_TYPE_COMMON, self.build_type]): self.add_include_path( os.path.join(self.fs_layout.tp_installed_dir, include_dir_component, 'include')) self.add_lib_dir_and_rpath( os.path.join(self.fs_layout.tp_installed_dir, include_dir_component, 'lib')) self.compiler_flags += self.preprocessor_flags # -fPIC is there to always generate position-independent code, even for static libraries. self.compiler_flags += [ '-fno-omit-frame-pointer', '-fPIC', '-O2', '-Wall' ] if is_linux(): # On Linux, ensure we set a long enough rpath so we can change it later with chrpath or # a similar tool. self.add_rpath(PLACEHOLDER_RPATH) self.dylib_suffix = "so" elif is_mac(): self.dylib_suffix = "dylib" # YugaByte builds with C++11, which on OS X requires using libc++ as the standard # library implementation. Some of the dependencies do not compile against libc++ by # default, so we specify it explicitly. self.cxx_flags.append("-stdlib=libc++") self.ld_flags += ["-lc++", "-lc++abi"] # Build for macOS Mojave or later. See https://bit.ly/37myHbk self.compiler_flags.append("-mmacosx-version-min=10.14") self.ld_flags.append("-Wl,-headerpad_max_install_names") else: fatal("Unsupported platform: {}".format(platform.system())) # The C++ standard must match CMAKE_CXX_STANDARD in the top-level CMakeLists.txt file in # the YugabyteDB source tree. self.cxx_flags.append('-std=c++14') self.cxx_flags.append('-frtti') if self.build_type == BUILD_TYPE_ASAN: self.compiler_flags += ASAN_FLAGS if self.build_type == BUILD_TYPE_TSAN: self.compiler_flags += TSAN_FLAGS
def verify_checksum(self, file_name: str, expected_checksum: Optional[str]) -> bool: real_checksum = compute_file_sha256(file_name) file_basename = os.path.basename(file_name) if expected_checksum is None: fatal( f"No expected checksum provided for file '{file_basename}'. Consider adding the " f"following line to thirdparty_src_checksums.txt (or re-run with --add-checksum):\n" f"{real_checksum} {file_basename}\n" ) return real_checksum == expected_checksum
def create_build_dir_and_prepare(self, dep: Dependency) -> str: src_dir = self.fs_layout.get_source_path(dep) if not os.path.isdir(src_dir): fatal("Directory '{}' does not exist".format(src_dir)) build_dir = self.fs_layout.get_build_dir_for_dependency(dep, self.build_type) mkdir_if_missing(build_dir) if dep.copy_sources: log("Bootstrapping %s from %s", build_dir, src_dir) subprocess.check_call(['rsync', '-a', src_dir + '/', build_dir]) return build_dir
def load_expected_checksums(self) -> None: if not os.path.exists(self.checksum_file_path): fatal("Expected checksum file not found at %s", self.checksum_file_path) self.filename2checksum = {} with open(self.checksum_file_path, 'rt') as inp: for line in inp: line = line.strip() if not line or line.startswith('#'): continue sum, fname = line.split(None, 1) if not re.match('^[0-9a-f]{64}$', sum): fatal("Invalid checksum: '%s' for archive name: '%s' in %s. Expected to be a " "SHA-256 sum (64 hex characters).", sum, fname, self.checksum_file_path) self.filename2checksum[fname] = sum
def _do_find_gcc(self, c_compiler: str, cxx_compiler: str) -> Tuple[str, str]: if self.using_linuxbrew(): gcc_dir = self.get_linuxbrew_dir() elif self.compiler_prefix: gcc_dir = self.compiler_prefix else: c_compiler_path = which_must_exist(c_compiler) cxx_compiler_path = which_must_exist(cxx_compiler) return c_compiler_path, cxx_compiler_path gcc_bin_dir = os.path.join(gcc_dir, 'bin') if not os.path.isdir(gcc_bin_dir): fatal("Directory {} does not exist".format(gcc_bin_dir)) return (os.path.join(gcc_bin_dir, 'gcc') + self.compiler_suffix, os.path.join(gcc_bin_dir, 'g++') + self.compiler_suffix)
def get_expected_checksum_and_maybe_add_to_file( self, filename: str, downloaded_path: str) -> str: if filename not in self.filename2checksum: if self.should_add_checksum: with open(self.checksum_file_path, 'rt') as inp: lines = inp.readlines() lines = [line.rstrip() for line in lines] checksum = compute_file_sha256(downloaded_path) lines.append("%s %s" % (checksum, filename)) with open(self.checksum_file_path, 'wt') as out: for line in lines: out.write(line + "\n") self.filename2checksum[filename] = checksum log("Added checksum for %s to %s: %s", filename, self.checksum_file_path, checksum) return checksum fatal("No expected checksum provided for {}".format(filename)) return self.filename2checksum[filename]
def find_compiler_by_type(self, compiler_type: str) -> None: compilers: Tuple[str, str] if compiler_type == 'gcc': if self.use_only_clang(): raise ValueError('Not allowed to use GCC') compilers = self.find_gcc() elif compiler_type == 'clang': if self.use_only_gcc(): raise ValueError('Not allowed to use Clang') compilers = self.find_clang() else: fatal("Unknown compiler type {}".format(compiler_type)) assert len(compilers) == 2 for compiler in compilers: if compiler is None or not os.path.exists(compiler): fatal("Compiler executable does not exist: {}".format(compiler)) self.cc = compilers[0] self.validate_compiler_path(self.cc) self.cxx = compilers[1] self.validate_compiler_path(self.cxx)
def find_clang(self) -> Tuple[str, str]: clang_prefix: Optional[str] = None if self.compiler_prefix: clang_prefix = self.compiler_prefix else: candidate_dirs = [ os.path.join(YB_THIRDPARTY_DIR, 'clang-toolchain'), '/usr' ] for dir in candidate_dirs: bin_dir = os.path.join(dir, 'bin') if os.path.exists(os.path.join(bin_dir, 'clang' + self.compiler_suffix)): clang_prefix = dir break if clang_prefix is None: fatal("Failed to find clang at the following locations: {}".format(candidate_dirs)) assert clang_prefix is not None clang_bin_dir = os.path.join(clang_prefix, 'bin') return (os.path.join(clang_bin_dir, 'clang') + self.compiler_suffix, os.path.join(clang_bin_dir, 'clang++') + self.compiler_suffix)
def extract_archive( self, archive_file_name: str, out_dir: str, out_name: Optional[str] = None) -> None: """ Extract the given archive into a subdirectory of out_dir, optionally renaming it to the specified name out_name. The archive is expected to contain exactly one directory. If out_name is not specified, the name of the directory inside the archive becomes the name of the destination directory. out_dir is the parent directory that should contain the extracted directory when the function returns. """ def dest_dir_already_exists(full_out_path: str) -> bool: if os.path.exists(full_out_path): log("Directory already exists: %s, skipping extracting %s" % ( full_out_path, archive_file_name)) return True return False full_out_path = None if out_name: full_out_path = os.path.join(out_dir, out_name) if dest_dir_already_exists(full_out_path): return # Extract the archive into a temporary directory. tmp_out_dir = os.path.join( out_dir, 'tmp-extract-%s-%s' % ( os.path.basename(archive_file_name), get_temporal_randomized_file_name_suffix() )) if os.path.exists(tmp_out_dir): raise IOError("Just-generated unique directory name already exists: %s" % tmp_out_dir) os.makedirs(tmp_out_dir) archive_extension = None for ext in ARCHIVE_TYPES: if archive_file_name.endswith(ext): archive_extension = ext break if not archive_extension: fatal("Unknown archive type for: {}".format(archive_file_name)) assert archive_extension is not None try: with PushDir(tmp_out_dir): cmd = ARCHIVE_TYPES[archive_extension].format(archive_file_name) log("Extracting %s in temporary directory %s", cmd, tmp_out_dir) subprocess.check_call(cmd, shell=True) extracted_subdirs = [ subdir_name for subdir_name in os.listdir(tmp_out_dir) if not subdir_name.startswith('.') ] if len(extracted_subdirs) != 1: raise IOError( "Expected the extracted archive %s to contain exactly one " "subdirectory and no files, found: %s" % ( archive_file_name, extracted_subdirs)) extracted_subdir_basename = extracted_subdirs[0] extracted_subdir_path = os.path.join(tmp_out_dir, extracted_subdir_basename) if not os.path.isdir(extracted_subdir_path): raise IOError( "This is a file, expected it to be a directory: %s" % extracted_subdir_path) if not full_out_path: full_out_path = os.path.join(out_dir, extracted_subdir_basename) if dest_dir_already_exists(full_out_path): return log("Moving %s to %s", extracted_subdir_path, full_out_path) shutil.move(extracted_subdir_path, full_out_path) finally: log("Removing temporary directory: %s", tmp_out_dir) shutil.rmtree(tmp_out_dir)
def download_dependency( self, dep: Dependency, src_path: str, archive_path: Optional[str]) -> None: patch_level_path = os.path.join(src_path, 'patchlevel-{}'.format(dep.patch_version)) if os.path.exists(patch_level_path): return download_url = dep.download_url assert download_url is not None, "Download URL not specified for dependency %s" % dep.name remove_path(src_path) # If download_url is "mkdir" then we just create empty directory with specified name. if download_url != 'mkdir': if archive_path is None: return self.ensure_file_downloaded( url=download_url, file_path=archive_path, enable_using_alternative_url=True) self.extract_archive(archive_path, os.path.dirname(src_path), os.path.basename(src_path)) else: log("Creating %s", src_path) mkdir_if_missing(src_path) if hasattr(dep, 'extra_downloads'): for extra in dep.extra_downloads: assert extra.archive_name is not None archive_path = os.path.join(self.download_dir, extra.archive_name) log("Downloading %s from %s", extra.archive_name, extra.download_url) self.ensure_file_downloaded( url=extra.download_url, file_path=archive_path, enable_using_alternative_url=True) output_path = os.path.join(src_path, extra.dir_name) self.extract_archive(archive_path, output_path) if extra.post_exec is not None: with PushDir(output_path): assert isinstance(extra.post_exec, list) if isinstance(extra.post_exec[0], str): subprocess.check_call(cast(List[str], extra.post_exec)) else: for command in extra.post_exec: subprocess.check_call(command) if hasattr(dep, 'patches'): with PushDir(src_path): for patch in dep.patches: log("Applying patch: %s", patch) process = subprocess.Popen(['patch', '-p{}'.format(dep.patch_strip)], stdin=subprocess.PIPE) with open(os.path.join(YB_THIRDPARTY_DIR, 'patches', patch), 'rt') as inp: patch = inp.read() assert process.stdin is not None process.stdin.write(patch.encode('utf-8')) process.stdin.close() exit_code = process.wait() if exit_code: fatal("Patch {} failed with code: {}".format(dep.name, exit_code)) if dep.post_patch: subprocess.check_call(dep.post_patch) with open(patch_level_path, 'wb') as out: # Just create an empty file. pass
def ensure_file_downloaded( self, url: str, file_path: str, enable_using_alternative_url: bool, expected_checksum: Optional[str] = None, verify_checksum: bool = True) -> None: log(f"Ensuring {url} is downloaded to path {file_path}") file_name = os.path.basename(file_path) mkdir_if_missing(self.download_dir) if os.path.exists(file_path) and verify_checksum: # We check the filename against our checksum map only if the file exists. This is done # so that we would still download the file even if we don't know the checksum, making it # easier to add new third-party dependencies. if expected_checksum is None: expected_checksum = self.get_expected_checksum_and_maybe_add_to_file( file_name, downloaded_path=file_path) if self.verify_checksum(file_path, expected_checksum): log("No need to re-download %s: checksum already correct", file_name) return log("File %s already exists but has wrong checksum, removing", file_path) remove_path(file_path) log("Fetching %s from %s", file_name, url) download_successful = False alternative_url = ALTERNATIVE_URL_PREFIX + file_name total_attempts = 0 url_candidates = [url] if enable_using_alternative_url: url_candidates += [alternative_url] for effective_url in url_candidates: if effective_url == alternative_url: log("Switching to alternative download URL %s after %d attempts", alternative_url, total_attempts) sleep_time_sec = INITIAL_DOWNLOAD_RETRY_SLEEP_TIME_SEC for attempt_index in range(1, MAX_FETCH_ATTEMPTS + 1): try: total_attempts += 1 curl_cmd_line = [ self.curl_path, '-o', file_path, '-L', # follow redirects '--silent', '--show-error', '--location', effective_url] log("Running command: %s", shlex_join(curl_cmd_line)) subprocess.check_call(curl_cmd_line) download_successful = True break except subprocess.CalledProcessError as ex: log("Error downloading %s (attempt %d for this URL, total attempts %d): %s", self.curl_path, attempt_index, total_attempts, str(ex)) if attempt_index == MAX_FETCH_ATTEMPTS and effective_url == alternative_url: log("Giving up after %d attempts", MAX_FETCH_ATTEMPTS) raise ex log("Will retry after %.1f seconds", sleep_time_sec) time.sleep(sleep_time_sec) sleep_time_sec += DOWNLOAD_RETRY_SLEEP_INCREASE_SEC if download_successful: break if not os.path.exists(file_path): fatal("Downloaded '%s' but but unable to find '%s'", url, file_path) if verify_checksum: if expected_checksum is None: expected_checksum = self.get_expected_checksum_and_maybe_add_to_file( file_name, downloaded_path=file_path) if not self.verify_checksum(file_path, expected_checksum): fatal("File '%s' has wrong checksum after downloading from '%s'. " "Has %s, but expected: %s", file_path, url, compute_file_sha256(file_path), expected_checksum)