def build(self):
compile_commands_path = os.path.join(self.conf.build_root_make, 'compile_commands.json')
if not os.path.exists(compile_commands_path):
# This is mostly useful during testing. We don't want to generate the list of compile
# commands by default because it takes a while, so only generate it on demand.
os.environ['CMAKE_EXPORT_COMPILE_COMMANDS'] = '1'
mkdir_p(self.conf.build_root_make)
os.environ['YB_USE_NINJA'] = '0'
subprocess.check_call(
[os.path.join(self.conf.yb_src_root, 'yb_build.sh'),
self.conf.build_type,
'--cmake-only',
'--no-rebuild-thirdparty',
'--build-root', self.conf.build_root_make])
logging.info("Loading compile commands from '{}'".format(compile_commands_path))
with open(compile_commands_path) as commands_file:
self.compile_commands = json.load(commands_file)
for entry in self.compile_commands:
self.compile_dirs.add(entry['directory'])
self.parse_link_and_depend_files()
self.find_proto_files()
self.dep_graph.validate_node_existence()
self.load_cmake_deps()
self.match_cmake_targets_with_files()
return self.dep_graph
def create_distribution(self, distribution_dir):
"""This method would read the release_manifest and traverse through the
build directory and copy necessary files/symlinks into the distribution_dir
Args:
distribution_dir (string): Directory to create the distribution
"""
for dir_from_manifest in self.release_manifest:
if dir_from_manifest == '%symlinks%':
for dst, target in self.release_manifest[
dir_from_manifest].items():
dst = os.path.join(distribution_dir, dst)
logging.debug("Creating symlink {} -> {}".format(
dst, target))
mkdir_p(os.path.dirname(dst))
os.symlink(target, dst)
continue
current_dest_dir = os.path.join(distribution_dir,
dir_from_manifest)
mkdir_p(current_dest_dir)
for elem in self.release_manifest[dir_from_manifest]:
elem = self.repo_expand_path(elem)
files = glob.glob(elem)
for file_path in files:
copy_deep(
file_path,
os.path.join(current_dest_dir,
os.path.basename(file_path)))
logging.info(
"Created the distribution at '{}'".format(distribution_dir))
def build(self):
compile_commands_path = os.path.join(self.conf.build_root,
'compile_commands.json')
if not os.path.exists(compile_commands_path):
# This is mostly useful during testing. We don't want to generate the list of compile
# commands by default because it takes a while, so only generate it on demand.
os.environ['CMAKE_EXPORT_COMPILE_COMMANDS'] = '1'
mkdir_p(self.conf.build_root)
subprocess.check_call([
os.path.join(self.conf.yb_src_root, 'yb_build.sh'),
self.conf.build_type, '--cmake-only',
'--no-rebuild-thirdparty', '--build-root', self.conf.build_root
])
logging.info(
"Loading compile commands from '{}'".format(compile_commands_path))
with open(compile_commands_path) as commands_file:
self.compile_commands = json.load(commands_file)
for entry in self.compile_commands:
self.compile_dirs.add(entry['directory'])
if self.conf.is_ninja:
self.parse_ninja_metadata()
else:
self.parse_link_and_depend_files_for_make()
self.find_proto_files()
self.dep_graph.validate_node_existence()
self.load_cmake_deps()
self.match_cmake_targets_with_files()
return self.dep_graph
def build(self) -> 'DependencyGraph':
compile_commands_path = os.path.join(self.conf.build_root,
'compile_commands.json')
cmake_deps_path = os.path.join(self.conf.build_root,
'yb_cmake_deps.txt')
run_build = False
for file_path in [compile_commands_path, cmake_deps_path]:
if not os.path.exists(file_path):
logging.info("File %s does not exist, will re-run the build.",
file_path)
run_build = True
if run_build:
# This is mostly useful during testing. We don't want to generate the list of compile
# commands by default because it takes a while, so only generate it on demand.
os.environ['YB_EXPORT_COMPILE_COMMANDS'] = '1'
mkdir_p(self.conf.build_root)
build_cmd = [
os.path.join(self.conf.yb_src_root, 'yb_build.sh'),
self.conf.build_type, '--cmake-only',
'--no-rebuild-thirdparty', '--build-root', self.conf.build_root
]
if self.conf.build_args:
# This is not ideal because it does not give a way to specify arguments with
# embedded spaces but is OK for our needs here.
logging.info("Running the build: %s", shlex_join(build_cmd))
build_cmd.extend(self.conf.build_args.strip().split())
subprocess.check_call(build_cmd)
logging.info(
"Loading compile commands from '{}'".format(compile_commands_path))
with open(compile_commands_path) as commands_file:
self.compile_commands = json.load(commands_file)
for entry in self.compile_commands:
self.compile_dirs.add(cast(Dict[str, Any], entry)['directory'])
if self.conf.is_ninja:
self.parse_ninja_metadata()
else:
self.parse_link_and_depend_files_for_make()
self.find_proto_files()
self.find_flex_bison_files()
self.dep_graph.validate_node_existence()
self.load_cmake_deps()
self.match_cmake_targets_with_files()
self.dep_graph._add_proto_generation_deps()
return self.dep_graph
def create_distribution(self, distribution_dir, prefix_dir=None):
"""This method would read the release_manifest and traverse through the
build directory and copy necessary files/symlinks into the distribution_dir
Args:
distribution_dir (string): Directory to create the distribution
prefix_dir (string): Only add entries that have a manifest key prefixed with this
"""
if prefix_dir:
full_path_prefix_dir = os.path.join(distribution_dir, prefix_dir)
if os.path.exists(full_path_prefix_dir):
logging.info(
"Found data at {}, removing before adding to distribution."
.format(full_path_prefix_dir))
# Ignore errors so that it recurses even if dir is not empty.
shutil.rmtree(full_path_prefix_dir, ignore_errors=True)
for dir_from_manifest in self.release_manifest:
if dir_from_manifest == '%symlinks%':
for dst, target in self.release_manifest[
dir_from_manifest].iteritems():
if prefix_dir is not None and not dst.startswith(
prefix_dir):
continue
dst = os.path.join(distribution_dir, dst)
logging.debug("Creating symlink {} -> {}".format(
dst, target))
mkdir_p(os.path.dirname(dst))
os.symlink(target, dst)
continue
# If we're using a prefix_dir, skip all manifest keys not starting with this.
if prefix_dir is not None and not dir_from_manifest.startswith(
prefix_dir):
continue
current_dest_dir = os.path.join(distribution_dir,
dir_from_manifest)
mkdir_p(current_dest_dir)
for elem in self.release_manifest[dir_from_manifest]:
elem = self.repo_expand_path(elem)
files = glob.glob(elem)
for file_path in files:
copy_deep(
file_path,
os.path.join(current_dest_dir,
os.path.basename(file_path)))
logging.info(
"Created the distribution at '{}'".format(distribution_dir))
def copy_file_or_tree(self, src_dir, src_rel_path, dest_dir):
"""
Does recursive copy of <src_dir>/<src_rel_path> to <dest_dir>/<src_rel_path>. Expects
<dest_dir> to be already existing directory. If <src_dir>/<src_rel_path> is a symlink
it will be copied as a symlink with the same target.
"""
if not os.path.isdir(dest_dir):
raise RuntimeError("Not a directory: '{}'".format(dest_dir))
src_path = os.path.join(src_dir, src_rel_path)
dst_path = os.path.join(dest_dir, src_rel_path)
src_is_link = os.path.islink(src_path)
if os.path.isdir(src_path) and not src_is_link:
shutil.copytree(src_path, dst_path)
else:
mkdir_p(os.path.dirname(dst_path))
if src_is_link:
os.symlink(os.readlink(src_path), dst_path)
else:
shutil.copy(src_path, dst_path)
def package_binaries(self):
"""
The main entry point to this class. Arranges binaries (executables and shared libraries),
starting with the given set of "seed executables", in the destination directory so that
the executables can find all of their dependencies.
"""
all_deps = []
executables = []
dest_bin_dir = os.path.join(self.dest_dir, 'bin')
mkdir_p(dest_bin_dir)
dest_lib_dir = os.path.join(self.dest_dir, 'lib')
mkdir_p(dest_lib_dir)
unwrapped_bin_dir = os.path.join(dest_lib_dir, 'unwrapped')
mkdir_p(unwrapped_bin_dir)
# We still need to use LD_LIBRARY_PATH because otherwise libc fails to find libresolv at
# runtime.
ld_library_path = ':'.join([
"${BASH_SOURCE%/*}/../lib/" + category
for category in LIBRARY_CATEGORIES
])
elf_names_to_set_interpreter = []
for seed_executable_glob in self.seed_executable_patterns:
glob_results = glob.glob(seed_executable_glob)
if not glob_results:
raise RuntimeError(
"No files found matching the pattern '{}'".format(
seed_executable_glob))
for executable in glob_results:
deps = self.find_elf_dependencies(executable)
all_deps += deps
if deps:
installed_binary_path = self.install_dyn_linked_binary(
executable, unwrapped_bin_dir)
executable_basename = os.path.basename(executable)
wrapper_script_path = os.path.join(dest_bin_dir,
executable_basename)
with open(wrapper_script_path, 'w') as wrapper_script_file:
wrapper_script_file.write(
"#!/usr/bin/env bash\n"
"LD_LIBRARY_PATH=" + ld_library_path + " exec "
"${BASH_SOURCE%/*}/../lib/unwrapped/" +
executable_basename + ' "$@"')
os.chmod(wrapper_script_path, 0755)
elf_names_to_set_interpreter.append(executable_basename)
else:
# This is probably a script.
shutil.copy(executable, dest_bin_dir)
# Not using the install_dyn_linked_binary method for copying patchelf and ld.so as we won't
# need to do any post-processing on these two later.
shutil.copy(PATCHELF_PATH, dest_bin_dir)
ld_path = os.path.join(LINUXBREW_HOME, 'lib', 'ld.so')
shutil.copy(ld_path, dest_lib_dir)
all_deps = sorted(set(all_deps))
for dep_name, deps in sorted_grouped_by(all_deps,
lambda dep: dep.name):
targets = sorted(set([dep.target for dep in deps]))
if len(targets) > 1:
raise RuntimeError(
"Multiple dependencies with the same name {} but different targets: {}"
.format(dep_name, deps))
categories = sorted(set([dep.get_category() for dep in all_deps]))
for category, deps_in_category in sorted_grouped_by(
all_deps, lambda dep: dep.get_category()):
logging.info("Found {} dependencies in category '{}':".format(
len(deps_in_category), category))
max_name_len = max([len(dep.name) for dep in deps_in_category])
for dep in sorted(deps_in_category, key=lambda dep: dep.target):
logging.info(" {} -> {}".format(
dep.name + ' ' * (max_name_len - len(dep.name)),
dep.target))
category_dest_dir = os.path.join(dest_lib_dir, category)
mkdir_p(category_dest_dir)
for dep in deps_in_category:
self.install_dyn_linked_binary(dep.target, category_dest_dir)
target_basename = os.path.basename(dep.target)
if os.path.basename(dep.target) != dep.name:
symlink(os.path.basename(dep.target),
os.path.join(category_dest_dir, dep.name))
linuxbrew_lib_dest_dir = os.path.join(dest_lib_dir, 'linuxbrew')
# Add libresolv and libnss_* libraries explicitly because they are loaded by glibc at
# runtime and will not be discovered automatically using ldd.
for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS:
for lib_path in glob.glob(
os.path.join(LINUXBREW_CELLAR_GLIBC_DIR, '*', 'lib',
additional_lib_name_glob)):
lib_basename = os.path.basename(lib_path)
if lib_basename.endswith('.a'):
continue
if os.path.isfile(lib_path):
self.install_dyn_linked_binary(lib_path,
linuxbrew_lib_dest_dir)
elif os.path.islink(lib_path):
link_target_basename = os.path.basename(
os.readlink(lib_path))
symlink(link_target_basename,
os.path.join(linuxbrew_lib_dest_dir, lib_basename))
else:
raise RuntimeError(
"Expected '{}' to be a file or a symlink".format(
lib_path))
for installed_binary in self.installed_dyn_linked_binaries:
# Sometimes files that we copy from other locations are not even writable by user!
subprocess.check_call(['chmod', 'u+w', installed_binary])
# Remove rpath so that it does not interfere with the LD_LIBRARY_PATH mechanism,
# and because some libraries might also have system library directories on their rpath.
run_patchelf('--remove-rpath', installed_binary)
post_install_path = os.path.join(dest_bin_dir, 'post_install.sh')
with open(post_install_path) as post_install_script_input:
post_install_script = post_install_script_input.read()
binary_names_to_patch = ' '.join(
['"{}"'.format(name) for name in elf_names_to_set_interpreter])
new_post_install_script = post_install_script.replace(
'${elf_names_to_set_interpreter}', binary_names_to_patch)
with open(post_install_path, 'w') as post_install_script_output:
post_install_script_output.write(new_post_install_script)
def package_binaries(self):
"""
The main entry point to this class. Arranges binaries (executables and shared libraries),
starting with the given set of "seed executables", in the destination directory so that
the executables can find all of their dependencies.
"""
all_deps = []
executables = []
dest_bin_dir = os.path.join(self.dest_dir, 'bin')
mkdir_p(dest_bin_dir)
dest_lib_dir = os.path.join(self.dest_dir, 'lib')
mkdir_p(dest_lib_dir)
elf_names_to_set_interpreter = []
for seed_executable_glob in self.seed_executable_patterns:
glob_results = glob.glob(seed_executable_glob)
if not glob_results:
raise RuntimeError("No files found matching the pattern '{}'".format(
seed_executable_glob))
for executable in glob_results:
deps = self.find_elf_dependencies(executable)
all_deps += deps
if deps:
self.install_dyn_linked_binary(executable, dest_bin_dir)
executable_basename = os.path.basename(executable)
elf_names_to_set_interpreter.append(executable_basename)
else:
# This is probably a script.
shutil.copy(executable, dest_bin_dir)
# Not using the install_dyn_linked_binary method for copying patchelf and ld.so as we won't
# need to do any post-processing on these two later.
shutil.copy(PATCHELF_PATH, dest_bin_dir)
ld_path = os.path.join(LINUXBREW_HOME, 'lib', 'ld.so')
shutil.copy(ld_path, dest_lib_dir)
all_deps = sorted(set(all_deps))
for dep_name, deps in sorted_grouped_by(all_deps, lambda dep: dep.name):
targets = sorted(set([dep.target for dep in deps]))
if len(targets) > 1:
raise RuntimeError(
"Multiple dependencies with the same name {} but different targets: {}".format(
dep_name, deps
))
categories = sorted(set([dep.get_category() for dep in all_deps]))
for category, deps_in_category in sorted_grouped_by(all_deps,
lambda dep: dep.get_category()):
logging.info("Found {} dependencies in category '{}':".format(
len(deps_in_category), category))
max_name_len = max([len(dep.name) for dep in deps_in_category])
for dep in sorted(deps_in_category, key=lambda dep: dep.target):
logging.info(" {} -> {}".format(
dep.name + ' ' * (max_name_len - len(dep.name)), dep.target))
category_dest_dir = os.path.join(dest_lib_dir, category)
mkdir_p(category_dest_dir)
for dep in deps_in_category:
self.install_dyn_linked_binary(dep.target, category_dest_dir)
target_basename = os.path.basename(dep.target)
if os.path.basename(dep.target) != dep.name:
symlink(os.path.basename(dep.target),
os.path.join(category_dest_dir, dep.name))
linuxbrew_lib_dest_dir = os.path.join(dest_lib_dir, 'linuxbrew')
# Add libresolv and libnss_* libraries explicitly because they are loaded by glibc at
# runtime and will not be discovered automatically using ldd.
for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS:
for lib_path in glob.glob(os.path.join(LINUXBREW_CELLAR_GLIBC_DIR, '*', 'lib',
additional_lib_name_glob)):
lib_basename = os.path.basename(lib_path)
if lib_basename.endswith('.a'):
continue
if os.path.isfile(lib_path):
self.install_dyn_linked_binary(lib_path, linuxbrew_lib_dest_dir)
elif os.path.islink(lib_path):
link_target_basename = os.path.basename(os.readlink(lib_path))
symlink(link_target_basename,
os.path.join(linuxbrew_lib_dest_dir, lib_basename))
else:
raise RuntimeError(
"Expected '{}' to be a file or a symlink".format(lib_path))
for installed_binary in self.installed_dyn_linked_binaries:
# Sometimes files that we copy from other locations are not even writable by user!
subprocess.check_call(['chmod', 'u+w', installed_binary])
# Remove rpath (we will set it appropriately in post_install.sh).
run_patchelf('--remove-rpath', installed_binary)
post_install_path = os.path.join(dest_bin_dir, 'post_install.sh')
with open(post_install_path) as post_install_script_input:
post_install_script = post_install_script_input.read()
binary_names_to_patch = ' '.join([
'"{}"'.format(name) for name in elf_names_to_set_interpreter])
new_post_install_script = post_install_script.replace('${elf_names_to_set_interpreter}',
binary_names_to_patch)
with open(post_install_path, 'w') as post_install_script_output:
post_install_script_output.write(new_post_install_script)
def package_binaries(self):
"""
The main entry point to this class. Arranges binaries (executables and shared libraries),
starting with the given set of "seed executables", in the destination directory so that
the executables can find all of their dependencies.
"""
all_deps = []
executables = []
dest_bin_dir = os.path.join(self.dest_dir, 'bin')
mkdir_p(dest_bin_dir)
dest_lib_dir = os.path.join(self.dest_dir, 'lib')
mkdir_p(dest_lib_dir)
unwrapped_bin_dir = os.path.join(dest_lib_dir, 'unwrapped')
mkdir_p(unwrapped_bin_dir)
unwrapped_executables = []
ld_library_path = ":".join([
"${BASH_SOURCE%/*}/../lib/" + category
for category in ['system', 'yb', 'yb-thirdparty', 'linuxbrew']
] + SYSTEM_LIBRARY_PATHS)
for seed_executable_glob in self.seed_executable_patterns:
re_match = re.match(r'^build/latest/(.*)$', seed_executable_glob)
if re_match:
updated_glob = os.path.join(self.context.build_dir,
re_match.group(1))
logging.info(
"Automatically updating seed glob to be relative to build dir: {} -> {}"
.format(seed_executable_glob, updated_glob))
seed_executable_glob = updated_glob
for executable in glob.glob(seed_executable_glob):
deps = self.find_elf_dependencies(executable)
all_deps += deps
if deps:
executables.append(executable)
shutil.copy(executable, unwrapped_bin_dir)
executable_basename = os.path.basename(executable)
unwrapped_executables.append(
os.path.join(unwrapped_bin_dir, executable_basename))
wrapper_script_path = os.path.join(dest_bin_dir,
executable_basename)
with open(wrapper_script_path, 'w') as wrapper_script_file:
wrapper_script_file.write(
"#!/usr/bin/env bash\n"
"LD_LIBRARY_PATH=" + ld_library_path + " exec "
"${BASH_SOURCE%/*}/../lib/unwrapped/" +
executable_basename + ' "$@"')
os.chmod(wrapper_script_path, 0755)
else:
# This is probably a script.
shutil.copy(executable, dest_bin_dir)
shutil.copy(PATCHELF_PATH, dest_bin_dir)
ld_path = os.path.join(LINUXBREW_HOME, 'lib', 'ld.so')
shutil.copy(ld_path, dest_lib_dir)
all_deps = sorted(set(all_deps))
for dep_name, deps in group_by(all_deps, lambda dep: dep.name):
targets = sorted(set([dep.target for dep in deps]))
if len(targets) > 1:
raise RuntimeException(
"Multiple dependencies with the same name {} but different targets: {}"
.format(dep_name, targets))
categories = sorted(set([dep.get_category() for dep in all_deps]))
for category, deps_in_category in group_by(
all_deps, lambda dep: dep.get_category()):
if category == 'system':
# We completely skip all system dependencies. They are supposed to be installed
# separately on the target system.
continue
logging.info("Found {} dependencies in category '{}':".format(
len(deps_in_category), category))
max_name_len = max([len(dep.name) for dep in deps_in_category])
for dep in sorted(deps_in_category, key=lambda dep: dep.target):
logging.info(" {} -> {}".format(
dep.name + ' ' * (max_name_len - len(dep.name)),
dep.target))
category_dest_dir = os.path.join(dest_lib_dir, category)
mkdir_p(category_dest_dir)
for dep in deps_in_category:
shutil.copy(dep.target, category_dest_dir)
target_basename = os.path.basename(dep.target)
if os.path.basename(dep.target) != dep.name:
symlink(os.path.basename(dep.target),
os.path.join(category_dest_dir, dep.name))
linuxbrew_lib_dest_dir = os.path.join(dest_lib_dir, 'linuxbrew')
# Add libnss_* libraries explicitly because they are loaded by glibc at runtime and will not
# be discovered automatically using ldd.
for libnss_path in glob.glob(
os.path.join(LINUXBREW_CELLAR_GLIBC_DIR, '*', 'lib',
'libnss_*')):
lib_basename = os.path.basename(libnss_path)
if os.path.isfile(libnss_path):
shutil.copy(libnss_path, linuxbrew_lib_dest_dir)
elif os.path.islink(libnss_path):
link_target_basename = os.path.basename(
os.readlink(libnss_path))
symlink(link_target_basename,
os.path.join(linuxbrew_lib_dest_dir, lib_basename))
else:
raise RuntimeError(
"Expected '{}' to be a file or a symlink".format(
libnss_path))
# Remove rpath as we're using LD_LIBRARY_PATH in wrapper scripts.
for unwrapped_executable_path in unwrapped_executables:
run_patchelf('--remove-rpath', unwrapped_executable_path)
def package_binaries(self):
"""
The main entry point to this class. Arranges binaries (executables and shared libraries),
starting with the given set of "seed executables", in the destination directory so that
the executables can find all of their dependencies.
"""
all_deps = []
dest_lib_dir = os.path.join(self.dest_dir, 'lib')
mkdir_p(dest_lib_dir)
mkdir_p(self.main_dest_bin_dir)
mkdir_p(self.postgres_dest_bin_dir)
main_elf_names_to_patch = []
postgres_elf_names_to_patch = []
for seed_executable_glob in self.seed_executable_patterns:
glob_results = glob.glob(seed_executable_glob)
if not glob_results:
raise RuntimeError(
"No files found matching the pattern '{}'".format(
seed_executable_glob))
for executable in glob_results:
deps = self.find_elf_dependencies(executable)
all_deps += deps
dest_bin_dir = self.get_dest_bin_dir_for_executable(executable)
if deps:
self.install_dyn_linked_binary(executable, dest_bin_dir)
executable_basename = os.path.basename(executable)
if self.is_postgres_binary(executable):
postgres_elf_names_to_patch.append(executable_basename)
else:
main_elf_names_to_patch.append(executable_basename)
else:
# This is probably a script.
shutil.copy(executable, dest_bin_dir)
# Not using the install_dyn_linked_binary method for copying patchelf and ld.so as we won't
# need to do any post-processing on these two later.
shutil.copy(PATCHELF_PATH, self.main_dest_bin_dir)
ld_path = os.path.join(LINUXBREW_HOME, 'lib', 'ld.so')
shutil.copy(ld_path, dest_lib_dir)
all_deps = sorted(set(all_deps))
for dep_name, deps in sorted_grouped_by(all_deps,
lambda dep: dep.name):
targets = sorted(set([dep.target for dep in deps]))
if len(targets) > 1:
raise RuntimeError(
"Multiple dependencies with the same name {} but different targets: {}"
.format(dep_name, deps))
for category, deps_in_category in sorted_grouped_by(
all_deps, lambda dep: dep.get_category()):
logging.info("Found {} dependencies in category '{}':".format(
len(deps_in_category), category))
max_name_len = max([len(dep.name) for dep in deps_in_category])
for dep in sorted(deps_in_category, key=lambda dep: dep.target):
logging.info(" {} -> {}".format(
dep.name + ' ' * (max_name_len - len(dep.name)),
dep.target))
category_dest_dir = os.path.join(dest_lib_dir, category)
mkdir_p(category_dest_dir)
for dep in deps_in_category:
self.install_dyn_linked_binary(dep.target, category_dest_dir)
if os.path.basename(dep.target) != dep.name:
symlink(os.path.basename(dep.target),
os.path.join(category_dest_dir, dep.name))
linuxbrew_lib_dest_dir = os.path.join(dest_lib_dir, 'linuxbrew')
# Add libresolv and libnss_* libraries explicitly because they are loaded by glibc at
# runtime and will not be discovered automatically using ldd.
for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS:
for lib_path in glob.glob(
os.path.join(LINUXBREW_CELLAR_GLIBC_DIR, '*', 'lib',
additional_lib_name_glob)):
lib_basename = os.path.basename(lib_path)
if lib_basename.endswith('.a'):
continue
if os.path.isfile(lib_path):
self.install_dyn_linked_binary(lib_path,
linuxbrew_lib_dest_dir)
elif os.path.islink(lib_path):
link_target_basename = os.path.basename(
os.readlink(lib_path))
symlink(link_target_basename,
os.path.join(linuxbrew_lib_dest_dir, lib_basename))
else:
raise RuntimeError(
"Expected '{}' to be a file or a symlink".format(
lib_path))
for installed_binary in self.installed_dyn_linked_binaries:
# Sometimes files that we copy from other locations are not even writable by user!
subprocess.check_call(['chmod', 'u+w', installed_binary])
# Remove rpath (we will set it appropriately in post_install.sh).
run_patchelf('--remove-rpath', installed_binary)
post_install_path = os.path.join(self.main_dest_bin_dir,
'post_install.sh')
with open(post_install_path) as post_install_script_input:
post_install_script = post_install_script_input.read()
new_post_install_script = post_install_script
for macro_var_name, list_of_binary_names in [
("main_elf_names_to_patch", main_elf_names_to_patch),
("postgres_elf_names_to_patch", postgres_elf_names_to_patch)
]:
new_post_install_script = new_post_install_script.replace(
'${%s}' % macro_var_name,
self.join_binary_names_for_bash(list_of_binary_names))
with open(post_install_path, 'w') as post_install_script_output:
post_install_script_output.write(new_post_install_script)
def package_binaries(self) -> None:
"""
The main entry point to this class. Arranges binaries (executables and shared libraries),
starting with the given set of "seed executables", in the destination directory so that
the executables can find all of their dependencies.
"""
all_deps: List[Dependency] = []
dest_lib_dir = os.path.join(self.dest_dir, 'lib')
mkdir_p(dest_lib_dir)
mkdir_p(self.main_dest_bin_dir)
mkdir_p(self.postgres_dest_bin_dir)
main_elf_names_to_patch = []
postgres_elf_names_to_patch = []
for seed_executable_glob in self.seed_executable_patterns:
glob_results = glob.glob(seed_executable_glob)
if not glob_results:
raise RuntimeError(
"No files found matching the pattern '{}'".format(
seed_executable_glob))
for executable in glob_results:
deps = self.find_elf_dependencies(executable)
all_deps += deps
dest_bin_dir = self.get_dest_bin_dir_for_executable(executable)
if deps:
self.install_dyn_linked_binary(executable, dest_bin_dir)
executable_basename = os.path.basename(executable)
if self.is_postgres_binary(executable):
postgres_elf_names_to_patch.append(executable_basename)
else:
main_elf_names_to_patch.append(executable_basename)
else:
# This is probably a script.
shutil.copy(executable, dest_bin_dir)
if using_linuxbrew():
# Not using the install_dyn_linked_binary method for copying patchelf and ld.so as we
# won't need to do any post-processing on these two later.
assert linuxbrew_home is not None
shutil.copy(linuxbrew_home.patchelf_path, self.main_dest_bin_dir)
shutil.copy(linuxbrew_home.ld_so_path, dest_lib_dir)
all_deps = sorted(set(all_deps))
deps_sorted_by_name: List[Tuple[str,
List[Dependency]]] = sorted_grouped_by(
all_deps, lambda dep: dep.name)
deps_with_same_name: List[Dependency]
for dep_name, deps_with_same_name in deps_sorted_by_name:
targets = sorted(set([dep.target for dep in deps_with_same_name]))
if len(targets) > 1:
raise RuntimeError(
"Multiple dependencies with the same name {} but different targets: {}"
.format(dep_name, deps_with_same_name))
linuxbrew_dest_dir = os.path.join(self.dest_dir, 'linuxbrew')
linuxbrew_lib_dest_dir = os.path.join(linuxbrew_dest_dir, 'lib')
# Add libresolv and libnss_* libs explicitly because they are loaded by glibc at runtime.
additional_libs = set()
if using_linuxbrew():
for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS:
assert linuxbrew_home is not None
additional_libs.update(lib_path for lib_path in glob.glob(
os.path.join(linuxbrew_home.cellar_glibc_dir, '*', 'lib',
additional_lib_name_glob))
if not lib_path.endswith('.a'))
for category, deps_in_category in sorted_grouped_by(
all_deps, lambda dep: dep.get_category()):
logging.info("Found {} dependencies in category '{}':".format(
len(deps_in_category), category))
max_name_len = max([len(dep.name) for dep in deps_in_category])
for dep in sorted(deps_in_category, key=lambda dep: dep.target):
logging.info(" {} -> {}".format(
dep.name + ' ' * (max_name_len - len(dep.name)),
dep.target))
if category == 'system':
logging.info(
"Not packaging any of the above dependencies from a system-wide directory."
)
continue
if category == 'postgres' and not using_linuxbrew():
# Only avoid copying postgres libraries into the lib/yb directory in non-Linuxbrew
# mode. In Linuxbrew mode, post_install.sh has complex logic for changing rpaths
# and that is not updated to allow us to find libraries in postgres/lib.
logging.info(
"Not installing any of the above YSQL libraries because they will be "
"installed as part of copying the entire postgres directory."
)
continue
if category == 'linuxbrew':
category_dest_dir = linuxbrew_lib_dest_dir
else:
category_dest_dir = os.path.join(dest_lib_dir, category)
mkdir_p(category_dest_dir)
for dep in deps_in_category:
additional_libs.discard(dep.target)
self.install_dyn_linked_binary(dep.target, category_dest_dir)
target_name = os.path.basename(dep.target)
if target_name != dep.name:
target_src = os.path.join(os.path.dirname(dep.target),
dep.name)
additional_libs.discard(target_src)
symlink(target_name,
os.path.join(category_dest_dir, dep.name))
for lib_path in additional_libs:
if os.path.isfile(lib_path):
self.install_dyn_linked_binary(lib_path,
linuxbrew_lib_dest_dir)
logging.info("Installed additional lib: " + lib_path)
elif os.path.islink(lib_path):
link_target_basename = os.path.basename(os.readlink(lib_path))
logging.info("Installed additional symlink: " + lib_path)
symlink(
link_target_basename,
os.path.join(linuxbrew_lib_dest_dir,
os.path.basename(lib_path)))
else:
raise RuntimeError(
"Expected '{}' to be a file or a symlink".format(lib_path))
for installed_binary in self.installed_dyn_linked_binaries:
# Sometimes files that we copy from other locations are not even writable by user!
subprocess.check_call(['chmod', 'u+w', installed_binary])
if using_linuxbrew():
# Remove rpath (we will set it appropriately in post_install.sh).
run_patchelf(['--remove-rpath', installed_binary])
post_install_path = os.path.join(self.main_dest_bin_dir,
'post_install.sh')
if using_linuxbrew():
# Add other files used by glibc at runtime.
assert linuxbrew_home is not None
linuxbrew_glibc_real_path = os.path.normpath(
os.path.join(os.path.realpath(linuxbrew_home.ldd_path), '..',
'..'))
assert linuxbrew_home is not None
linuxbrew_glibc_rel_path = os.path.relpath(
linuxbrew_glibc_real_path,
os.path.realpath(linuxbrew_home.linuxbrew_dir))
# We expect glibc to live under a path like "Cellar/glibc/3.23" in
# the Linuxbrew directory.
if not linuxbrew_glibc_rel_path.startswith('Cellar/glibc/'):
raise ValueError(
"Expected to find glibc under Cellar/glibc/<version> in Linuxbrew, but found it"
" at: '%s'" % linuxbrew_glibc_rel_path)
rel_paths = []
for glibc_rel_path in [
'etc/ld.so.cache',
'etc/localtime',
'lib/locale/locale-archive',
'lib/gconv',
'libexec/getconf',
'share/locale',
'share/zoneinfo',
]:
rel_paths.append(
os.path.join(linuxbrew_glibc_rel_path, glibc_rel_path))
terminfo_glob_pattern = os.path.join(
linuxbrew_home.linuxbrew_dir,
'Cellar/ncurses/*/share/terminfo')
terminfo_paths = glob.glob(terminfo_glob_pattern)
if len(terminfo_paths) != 1:
raise ValueError(
"Failed to find the terminfo directory using glob pattern %s. "
"Found: %s" % (terminfo_glob_pattern, terminfo_paths))
terminfo_rel_path = os.path.relpath(terminfo_paths[0],
linuxbrew_home.linuxbrew_dir)
rel_paths.append(terminfo_rel_path)
for rel_path in rel_paths:
src = os.path.join(linuxbrew_home.linuxbrew_dir, rel_path)
dst = os.path.join(linuxbrew_dest_dir, rel_path)
copy_deep(src, dst, create_dst_dir=True)
with open(post_install_path) as post_install_script_input:
post_install_script = post_install_script_input.read()
new_post_install_script = post_install_script
replacements = [
("original_linuxbrew_path_to_patch",
linuxbrew_home.linuxbrew_dir),
("original_linuxbrew_path_length",
len(linuxbrew_home.linuxbrew_dir)),
]
for macro_var_name, list_of_binary_names in [
("main_elf_names_to_patch", main_elf_names_to_patch),
("postgres_elf_names_to_patch", postgres_elf_names_to_patch),
]:
replacements.append(
(macro_var_name,
self.join_binary_names_for_bash(list_of_binary_names)))
for macro_var_name, value in replacements:
new_post_install_script = new_post_install_script.replace(
'${%s}' % macro_var_name, str(value))
else:
new_post_install_script = (
'#!/usr/bin/env bash\n'
'# For backward-compatibility only. We do not need this script anymore.\n'
)
with open(post_install_path, 'w') as post_install_script_output:
post_install_script_output.write(new_post_install_script)
def package_binaries(self):
"""
The main entry point to this class. Arranges binaries (executables and shared libraries),
starting with the given set of "seed executables", in the destination directory so that
the executables can find all of their dependencies.
"""
all_deps = []
dest_lib_dir = os.path.join(self.dest_dir, 'lib')
mkdir_p(dest_lib_dir)
mkdir_p(self.main_dest_bin_dir)
mkdir_p(self.postgres_dest_bin_dir)
main_elf_names_to_patch = []
postgres_elf_names_to_patch = []
for seed_executable_glob in self.seed_executable_patterns:
glob_results = glob.glob(seed_executable_glob)
if not glob_results:
raise RuntimeError("No files found matching the pattern '{}'".format(
seed_executable_glob))
for executable in glob_results:
deps = self.find_elf_dependencies(executable)
all_deps += deps
dest_bin_dir = self.get_dest_bin_dir_for_executable(executable)
if deps:
self.install_dyn_linked_binary(executable, dest_bin_dir)
executable_basename = os.path.basename(executable)
if self.is_postgres_binary(executable):
postgres_elf_names_to_patch.append(executable_basename)
else:
main_elf_names_to_patch.append(executable_basename)
else:
# This is probably a script.
shutil.copy(executable, dest_bin_dir)
# Not using the install_dyn_linked_binary method for copying patchelf and ld.so as we won't
# need to do any post-processing on these two later.
shutil.copy(linuxbrew_home.patchelf_path, self.main_dest_bin_dir)
ld_path = linuxbrew_home.ld_so_path
shutil.copy(ld_path, dest_lib_dir)
all_deps = sorted(set(all_deps))
for dep_name, deps in sorted_grouped_by(all_deps, lambda dep: dep.name):
targets = sorted(set([dep.target for dep in deps]))
if len(targets) > 1:
raise RuntimeError(
"Multiple dependencies with the same name {} but different targets: {}".format(
dep_name, deps
))
linuxbrew_dest_dir = os.path.join(self.dest_dir, 'linuxbrew')
linuxbrew_lib_dest_dir = os.path.join(linuxbrew_dest_dir, 'lib')
for category, deps_in_category in sorted_grouped_by(all_deps,
lambda dep: dep.get_category()):
logging.info("Found {} dependencies in category '{}':".format(
len(deps_in_category), category))
max_name_len = max([len(dep.name) for dep in deps_in_category])
for dep in sorted(deps_in_category, key=lambda dep: dep.target):
logging.info(" {} -> {}".format(
dep.name + ' ' * (max_name_len - len(dep.name)), dep.target))
if category == 'linuxbrew':
category_dest_dir = linuxbrew_lib_dest_dir
else:
category_dest_dir = os.path.join(dest_lib_dir, category)
mkdir_p(category_dest_dir)
for dep in deps_in_category:
self.install_dyn_linked_binary(dep.target, category_dest_dir)
if os.path.basename(dep.target) != dep.name:
symlink(os.path.basename(dep.target),
os.path.join(category_dest_dir, dep.name))
# Add libresolv and libnss_* libraries explicitly because they are loaded by glibc at
# runtime and will not be discovered automatically using ldd.
for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS:
for lib_path in glob.glob(os.path.join(linuxbrew_home.cellar_glibc_dir, '*', 'lib',
additional_lib_name_glob)):
lib_basename = os.path.basename(lib_path)
if lib_basename.endswith('.a'):
continue
if os.path.isfile(lib_path):
self.install_dyn_linked_binary(lib_path, linuxbrew_lib_dest_dir)
elif os.path.islink(lib_path):
link_target_basename = os.path.basename(os.readlink(lib_path))
symlink(link_target_basename,
os.path.join(linuxbrew_lib_dest_dir, lib_basename))
else:
raise RuntimeError(
"Expected '{}' to be a file or a symlink".format(lib_path))
for installed_binary in self.installed_dyn_linked_binaries:
# Sometimes files that we copy from other locations are not even writable by user!
subprocess.check_call(['chmod', 'u+w', installed_binary])
# Remove rpath (we will set it appropriately in post_install.sh).
run_patchelf('--remove-rpath', installed_binary)
# Add other files used by glibc at runtime.
linuxbrew_glibc_real_path = os.path.normpath(
os.path.join(os.path.realpath(linuxbrew_home.ldd_path), '..', '..'))
linuxbrew_glibc_rel_path = os.path.relpath(
linuxbrew_glibc_real_path, os.path.realpath(linuxbrew_home.linuxbrew_dir))
# We expect glibc to live under a path like "Cellar/glibc/2.23" in the Linuxbrew directory.
if not linuxbrew_glibc_rel_path.startswith('Cellar/glibc/'):
raise ValueError(
"Expected to find glibc under Cellar/glibc/<version> in Linuxbrew, but found it "
"at: '%s'" % linuxbrew_glibc_rel_path)
rel_paths = []
for glibc_rel_path in [
'etc/ld.so.cache',
'etc/localtime',
'lib/locale/locale-archive',
'lib/gconv',
'libexec/getconf',
'share/locale',
'share/zoneinfo',
]:
rel_paths.append(os.path.join(linuxbrew_glibc_rel_path, glibc_rel_path))
terminfo_glob_pattern = os.path.join(
linuxbrew_home.linuxbrew_dir, 'Cellar/ncurses/*/share/terminfo')
terminfo_paths = glob.glob(terminfo_glob_pattern)
if len(terminfo_paths) != 1:
raise ValueError(
"Failed to find the terminfo directory using glob pattern %s. "
"Found: %s" % (terminfo_glob_pattern, terminfo_paths))
terminfo_rel_path = os.path.relpath(terminfo_paths[0], linuxbrew_home.linuxbrew_dir)
rel_paths.append(terminfo_rel_path)
for rel_path in rel_paths:
src = os.path.join(linuxbrew_home.linuxbrew_dir, rel_path)
dst = os.path.join(linuxbrew_dest_dir, rel_path)
copy_deep(src, dst, create_dst_dir=True)
post_install_path = os.path.join(self.main_dest_bin_dir, 'post_install.sh')
with open(post_install_path) as post_install_script_input:
post_install_script = post_install_script_input.read()
new_post_install_script = post_install_script
replacements = [
("original_linuxbrew_path_to_patch", linuxbrew_home.linuxbrew_dir),
("original_linuxbrew_path_length", len(linuxbrew_home.linuxbrew_dir)),
]
for macro_var_name, list_of_binary_names in [
("main_elf_names_to_patch", main_elf_names_to_patch),
("postgres_elf_names_to_patch", postgres_elf_names_to_patch),
]:
replacements.append(
(macro_var_name, self.join_binary_names_for_bash(list_of_binary_names)))
for macro_var_name, value in replacements:
new_post_install_script = new_post_install_script.replace(
'${%s}' % macro_var_name, str(value))
with open(post_install_path, 'w') as post_install_script_output:
post_install_script_output.write(new_post_install_script)
