def get_category(self) -> str:
"""
Categorizes binaries into a few buckets:
- yb -- built as part of YugabyteDB
- yb-thirdparty -- built as part of yugabyte-db-thirdparty
- linuxbrew -- built using Linuxbrew
- system -- a library residing in a system-wide library directory. We do not copy these.
- postgres -- libraries inside the postgres directory
"""
if self.category:
return self.category
linuxbrew_home = get_linuxbrew_home()
if linuxbrew_home is not None and linuxbrew_home.path_is_in_linuxbrew_dir(
self.target):
self.category = 'linuxbrew'
elif self.target.startswith(get_thirdparty_dir() + '/'):
self.category = 'yb-thirdparty'
elif self.target.startswith(self.context.build_dir + '/postgres/'):
self.category = 'postgres'
elif self.target.startswith(self.context.build_dir + '/'):
self.category = 'yb'
elif (self.target.startswith(YB_SCRIPT_BIN_DIR + '/')
or self.target.startswith(YB_BUILD_SUPPORT_DIR + '/')):
self.category = 'yb-scripts'
if not self.category:
for system_library_path in SYSTEM_LIBRARY_PATHS:
if self.target.startswith(system_library_path + '/'):
self.category = 'system'
break
if self.category:
if self.category not in LIBRARY_CATEGORIES:
raise RuntimeError((
"Internal error: library category computed as '{}', must be one of: {}. "
+ "Dependency: {}").format(self.category,
LIBRARY_CATEGORIES, self))
return self.category
if linuxbrew_home:
linuxbrew_dir_str = linuxbrew_home.get_human_readable_dirs()
else:
linuxbrew_dir_str = 'N/A'
raise RuntimeError((
"Could not determine the category of this binary "
"(yugabyte / yb-thirdparty / linuxbrew / system): '{}'. "
"Does not reside in the Linuxbrew directory ({}), "
"YB third-party directory ('{}'), "
"YB build directory ('{}'), "
"YB general-purpose script directory ('{}'), "
"YB build support script directory ('{}'), "
"and does not appear to be a system library (does not start with any of {})."
).format(self.target, linuxbrew_dir_str, get_thirdparty_dir(),
self.context.build_dir, YB_SCRIPT_BIN_DIR,
YB_BUILD_SUPPORT_DIR, SYSTEM_LIBRARY_PATHS))
def find_elf_dependencies(self, elf_file_path: str) -> Set[Dependency]:
"""
Run ldd on the given ELF file and find libraries that it depends on. Also run patchelf and
get the dynamic linker used by the file.
@param elf_file_path: ELF file (executable/library) path
"""
linuxbrew_home: Optional[LinuxbrewHome] = get_linuxbrew_home()
elf_file_path = os.path.realpath(elf_file_path)
if SYSTEM_LIBRARY_PATH_RE.match(
elf_file_path) or not using_linuxbrew():
ldd_path = '/usr/bin/ldd'
else:
assert linuxbrew_home is not None
assert linuxbrew_home.ldd_path is not None
ldd_path = linuxbrew_home.ldd_path
ldd_result = run_program([ldd_path, elf_file_path], error_ok=True)
dependencies: Set[Dependency] = set()
ldd_result_stdout_str = ldd_result.stdout
ldd_result_stderr_str = ldd_result.stderr
if ldd_result.returncode != 0:
# The below error message is printed to stdout on some platforms (CentOS) and
# stderr on other platforms (Ubuntu).
if 'not a dynamic executable' in (ldd_result_stdout_str,
ldd_result_stderr_str):
logging.debug(
"Not a dynamic executable: {}, ignoring dependency tracking"
.format(elf_file_path))
return dependencies
raise RuntimeError(ldd_result.error_msg)
for ldd_output_line in ldd_result_stdout_str.split("\n"):
resolved_dep_match = RESOLVED_DEP_RE.match(ldd_output_line)
if resolved_dep_match:
lib_name = resolved_dep_match.group(1)
lib_resolved_path = os.path.realpath(
resolved_dep_match.group(2))
dependencies.add(
Dependency(lib_name, lib_resolved_path, elf_file_path,
self.context))
tokens = ldd_output_line.split()
if len(tokens) >= 4 and tokens[1:4] == ['=>', 'not', 'found']:
missing_lib_name = tokens[0]
raise RuntimeError("Library not found for '{}': {}".format(
elf_file_path, missing_lib_name))
# If we matched neither RESOLVED_DEP_RE or the "not found" case, that is still fine,
# e.g. there could be a line of the following form in the ldd output:
# linux-vdso.so.1 => (0x00007ffc0f9d2000)
return dependencies
def get_patchelf_path() -> Optional[str]:
global g_patchelf_path_initialized, g_patchelf_path
if g_patchelf_path_initialized:
return g_patchelf_path
linuxbrew_home: Optional[LinuxbrewHome] = get_linuxbrew_home()
if linuxbrew_home is not None:
g_patchelf_path = linuxbrew_home.patchelf_path
if not g_patchelf_path:
g_patchelf_path = find_executable('patchelf')
g_patchelf_path_initialized = True
return g_patchelf_path
def package_binaries(self) -> None:
"""
The main entry point to this class. Arranges binaries (executables and shared libraries),
starting with the given set of "seed executables", in the destination directory so that
the executables can find all of their dependencies.
"""
linuxbrew_home = get_linuxbrew_home()
all_deps: List[Dependency] = []
dest_lib_dir = os.path.join(self.dest_dir, 'lib')
mkdir_p(dest_lib_dir)
mkdir_p(self.main_dest_bin_dir)
mkdir_p(self.postgres_dest_bin_dir)
main_elf_names_to_patch = []
postgres_elf_names_to_patch = []
for seed_executable_glob in self.seed_executable_patterns:
glob_results = glob.glob(seed_executable_glob)
if not glob_results:
raise RuntimeError(
"No files found matching the pattern '{}'".format(
seed_executable_glob))
for executable in glob_results:
deps = self.find_elf_dependencies(executable)
all_deps += deps
dest_bin_dir = self.get_dest_bin_dir_for_executable(executable)
if deps:
self.install_dyn_linked_binary(executable, dest_bin_dir)
executable_basename = os.path.basename(executable)
if self.is_postgres_binary(executable):
postgres_elf_names_to_patch.append(executable_basename)
else:
main_elf_names_to_patch.append(executable_basename)
else:
# This is probably a script.
shutil.copy(executable, dest_bin_dir)
if using_linuxbrew():
# Not using the install_dyn_linked_binary method for copying patchelf and ld.so as we
# won't need to do any post-processing on these two later.
assert linuxbrew_home is not None
assert linuxbrew_home.patchelf_path is not None
assert linuxbrew_home.ld_so_path is not None
shutil.copy(linuxbrew_home.patchelf_path, self.main_dest_bin_dir)
shutil.copy(linuxbrew_home.ld_so_path, dest_lib_dir)
all_deps = sorted(set(all_deps))
deps_sorted_by_name: List[Tuple[str,
List[Dependency]]] = sorted_grouped_by(
all_deps, lambda dep: dep.name)
deps_with_same_name: List[Dependency]
for dep_name, deps_with_same_name in deps_sorted_by_name:
targets = sorted(set([dep.target for dep in deps_with_same_name]))
if len(targets) > 1:
raise RuntimeError(
"Multiple dependencies with the same name {} but different targets: {}"
.format(dep_name, deps_with_same_name))
linuxbrew_dest_dir = os.path.join(self.dest_dir, 'linuxbrew')
linuxbrew_lib_dest_dir = os.path.join(linuxbrew_dest_dir, 'lib')
# Add libresolv and libnss_* libs explicitly because they are loaded by glibc at runtime.
additional_libs: Set[str] = set()
if using_linuxbrew():
for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS:
assert linuxbrew_home is not None
assert linuxbrew_home.cellar_glibc_dir is not None
additional_libs.update(lib_path for lib_path in glob.glob(
os.path.join(linuxbrew_home.cellar_glibc_dir, '*', 'lib',
additional_lib_name_glob))
if not lib_path.endswith('.a'))
for category, deps_in_category in sorted_grouped_by(
all_deps, lambda dep: dep.get_category()):
logging.info("Found {} dependencies in category '{}':".format(
len(deps_in_category), category))
max_name_len = max([len(dep.name) for dep in deps_in_category])
for dep in sorted(deps_in_category, key=lambda dep: dep.target):
logging.info(" {} -> {}".format(
dep.name + ' ' * (max_name_len - len(dep.name)),
dep.target))
if category == 'system':
logging.info(
"Not packaging any of the above dependencies from a system-wide directory."
)
continue
if category == 'postgres' and not using_linuxbrew():
# Only avoid copying postgres libraries into the lib/yb directory in non-Linuxbrew
# mode. In Linuxbrew mode, post_install.sh has complex logic for changing rpaths
# and that is not updated to allow us to find libraries in postgres/lib.
logging.info(
"Not installing any of the above YSQL libraries because they will be "
"installed as part of copying the entire postgres directory."
)
continue
if category == 'linuxbrew':
category_dest_dir = linuxbrew_lib_dest_dir
else:
category_dest_dir = os.path.join(dest_lib_dir, category)
mkdir_p(category_dest_dir)
for dep in deps_in_category:
additional_libs.discard(dep.target)
self.install_dyn_linked_binary(dep.target, category_dest_dir)
target_name = os.path.basename(dep.target)
if target_name != dep.name:
target_src = os.path.join(os.path.dirname(dep.target),
dep.name)
additional_libs.discard(target_src)
symlink(target_name,
os.path.join(category_dest_dir, dep.name))
for lib_path in additional_libs:
if os.path.isfile(lib_path):
self.install_dyn_linked_binary(lib_path,
linuxbrew_lib_dest_dir)
logging.info("Installed additional lib: " + lib_path)
elif os.path.islink(lib_path):
link_target_basename = os.path.basename(os.readlink(lib_path))
logging.info("Installed additional symlink: " + lib_path)
symlink(
link_target_basename,
os.path.join(linuxbrew_lib_dest_dir,
os.path.basename(lib_path)))
else:
raise RuntimeError(
"Expected '{}' to be a file or a symlink".format(lib_path))
for installed_binary in self.installed_dyn_linked_binaries:
# Sometimes files that we copy from other locations are not even writable by user!
subprocess.check_call(['chmod', 'u+w', installed_binary])
if using_linuxbrew():
# Remove rpath (we will set it appropriately in post_install.sh).
remove_rpath(installed_binary)
post_install_path = os.path.join(self.main_dest_bin_dir,
'post_install.sh')
if using_linuxbrew():
# Add other files used by glibc at runtime.
assert linuxbrew_home is not None
linuxbrew_dir = linuxbrew_home.linuxbrew_dir
assert linuxbrew_dir is not None
assert linuxbrew_home.ldd_path is not None
linuxbrew_glibc_real_path = os.path.normpath(
os.path.join(os.path.realpath(linuxbrew_home.ldd_path), '..',
'..'))
assert linuxbrew_home is not None
linuxbrew_glibc_rel_path = os.path.relpath(
linuxbrew_glibc_real_path, os.path.realpath(linuxbrew_dir))
# We expect glibc to live under a path like "Cellar/glibc/3.23" in
# the Linuxbrew directory.
if not linuxbrew_glibc_rel_path.startswith('Cellar/glibc/'):
raise ValueError(
"Expected to find glibc under Cellar/glibc/<version> in Linuxbrew, but found it"
" at: '%s'" % linuxbrew_glibc_rel_path)
rel_paths = []
for glibc_rel_path in [
'etc/ld.so.cache',
'etc/localtime',
'lib/locale/locale-archive',
'lib/gconv',
'libexec/getconf',
'share/locale',
'share/zoneinfo',
]:
rel_paths.append(
os.path.join(linuxbrew_glibc_rel_path, glibc_rel_path))
terminfo_glob_pattern = os.path.join(
linuxbrew_dir, 'Cellar/ncurses/*/share/terminfo')
terminfo_paths = glob.glob(terminfo_glob_pattern)
if len(terminfo_paths) != 1:
raise ValueError(
"Failed to find the terminfo directory using glob pattern %s. "
"Found: %s" % (terminfo_glob_pattern, terminfo_paths))
terminfo_rel_path = os.path.relpath(terminfo_paths[0],
linuxbrew_dir)
rel_paths.append(terminfo_rel_path)
for rel_path in rel_paths:
src = os.path.join(linuxbrew_dir, rel_path)
dst = os.path.join(linuxbrew_dest_dir, rel_path)
copy_deep(src, dst, create_dst_dir=True)
with open(post_install_path) as post_install_script_input:
post_install_script = post_install_script_input.read()
new_post_install_script = post_install_script
replacements = [
("original_linuxbrew_path_to_patch", linuxbrew_dir),
("original_linuxbrew_path_length", len(linuxbrew_dir)),
]
for macro_var_name, list_of_binary_names in [
("main_elf_names_to_patch", main_elf_names_to_patch),
("postgres_elf_names_to_patch", postgres_elf_names_to_patch),
]:
replacements.append(
(macro_var_name,
self.join_binary_names_for_bash(list_of_binary_names)))
for macro_var_name, value in replacements:
new_post_install_script = new_post_install_script.replace(
'${%s}' % macro_var_name, str(value))
else:
new_post_install_script = (
'#!/usr/bin/env bash\n'
'# For backward-compatibility only. We do not need this script anymore.\n'
)
with open(post_install_path, 'w') as post_install_script_output:
post_install_script_output.write(new_post_install_script)