def create_distribution(self, distribution_dir): """This method would read the release_manifest and traverse through the build directory and copy necessary files/symlinks into the distribution_dir Args: distribution_dir (string): Directory to create the distribution """ for dir_from_manifest in self.release_manifest: if dir_from_manifest == '%symlinks%': for dst, target in self.release_manifest[ dir_from_manifest].items(): dst = os.path.join(distribution_dir, dst) logging.debug("Creating symlink {} -> {}".format( dst, target)) mkdir_p(os.path.dirname(dst)) os.symlink(target, dst) continue current_dest_dir = os.path.join(distribution_dir, dir_from_manifest) mkdir_p(current_dest_dir) for elem in self.release_manifest[dir_from_manifest]: elem = self.repo_expand_path(elem) files = glob.glob(elem) for file_path in files: copy_deep( file_path, os.path.join(current_dest_dir, os.path.basename(file_path))) logging.info( "Created the distribution at '{}'".format(distribution_dir))
def create_distribution(self, distribution_dir, prefix_dir=None): """This method would read the release_manifest and traverse through the build directory and copy necessary files/symlinks into the distribution_dir Args: distribution_dir (string): Directory to create the distribution prefix_dir (string): Only add entries that have a manifest key prefixed with this """ if prefix_dir: full_path_prefix_dir = os.path.join(distribution_dir, prefix_dir) if os.path.exists(full_path_prefix_dir): logging.info( "Found data at {}, removing before adding to distribution." .format(full_path_prefix_dir)) # Ignore errors so that it recurses even if dir is not empty. shutil.rmtree(full_path_prefix_dir, ignore_errors=True) for dir_from_manifest in self.release_manifest: if dir_from_manifest == '%symlinks%': for dst, target in self.release_manifest[ dir_from_manifest].iteritems(): if prefix_dir is not None and not dst.startswith( prefix_dir): continue dst = os.path.join(distribution_dir, dst) logging.debug("Creating symlink {} -> {}".format( dst, target)) mkdir_p(os.path.dirname(dst)) os.symlink(target, dst) continue # If we're using a prefix_dir, skip all manifest keys not starting with this. if prefix_dir is not None and not dir_from_manifest.startswith( prefix_dir): continue current_dest_dir = os.path.join(distribution_dir, dir_from_manifest) mkdir_p(current_dest_dir) for elem in self.release_manifest[dir_from_manifest]: elem = self.repo_expand_path(elem) files = glob.glob(elem) for file_path in files: copy_deep( file_path, os.path.join(current_dest_dir, os.path.basename(file_path))) logging.info( "Created the distribution at '{}'".format(distribution_dir))
def package_binaries(self) -> None: """ The main entry point to this class. Arranges binaries (executables and shared libraries), starting with the given set of "seed executables", in the destination directory so that the executables can find all of their dependencies. """ all_deps: List[Dependency] = [] dest_lib_dir = os.path.join(self.dest_dir, 'lib') mkdir_p(dest_lib_dir) mkdir_p(self.main_dest_bin_dir) mkdir_p(self.postgres_dest_bin_dir) main_elf_names_to_patch = [] postgres_elf_names_to_patch = [] for seed_executable_glob in self.seed_executable_patterns: glob_results = glob.glob(seed_executable_glob) if not glob_results: raise RuntimeError( "No files found matching the pattern '{}'".format( seed_executable_glob)) for executable in glob_results: deps = self.find_elf_dependencies(executable) all_deps += deps dest_bin_dir = self.get_dest_bin_dir_for_executable(executable) if deps: self.install_dyn_linked_binary(executable, dest_bin_dir) executable_basename = os.path.basename(executable) if self.is_postgres_binary(executable): postgres_elf_names_to_patch.append(executable_basename) else: main_elf_names_to_patch.append(executable_basename) else: # This is probably a script. shutil.copy(executable, dest_bin_dir) if using_linuxbrew(): # Not using the install_dyn_linked_binary method for copying patchelf and ld.so as we # won't need to do any post-processing on these two later. assert linuxbrew_home is not None shutil.copy(linuxbrew_home.patchelf_path, self.main_dest_bin_dir) shutil.copy(linuxbrew_home.ld_so_path, dest_lib_dir) all_deps = sorted(set(all_deps)) deps_sorted_by_name: List[Tuple[str, List[Dependency]]] = sorted_grouped_by( all_deps, lambda dep: dep.name) deps_with_same_name: List[Dependency] for dep_name, deps_with_same_name in deps_sorted_by_name: targets = sorted(set([dep.target for dep in deps_with_same_name])) if len(targets) > 1: raise RuntimeError( "Multiple dependencies with the same name {} but different targets: {}" .format(dep_name, deps_with_same_name)) linuxbrew_dest_dir = os.path.join(self.dest_dir, 'linuxbrew') linuxbrew_lib_dest_dir = os.path.join(linuxbrew_dest_dir, 'lib') # Add libresolv and libnss_* libs explicitly because they are loaded by glibc at runtime. additional_libs = set() if using_linuxbrew(): for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS: assert linuxbrew_home is not None additional_libs.update(lib_path for lib_path in glob.glob( os.path.join(linuxbrew_home.cellar_glibc_dir, '*', 'lib', additional_lib_name_glob)) if not lib_path.endswith('.a')) for category, deps_in_category in sorted_grouped_by( all_deps, lambda dep: dep.get_category()): logging.info("Found {} dependencies in category '{}':".format( len(deps_in_category), category)) max_name_len = max([len(dep.name) for dep in deps_in_category]) for dep in sorted(deps_in_category, key=lambda dep: dep.target): logging.info(" {} -> {}".format( dep.name + ' ' * (max_name_len - len(dep.name)), dep.target)) if category == 'system': logging.info( "Not packaging any of the above dependencies from a system-wide directory." ) continue if category == 'postgres' and not using_linuxbrew(): # Only avoid copying postgres libraries into the lib/yb directory in non-Linuxbrew # mode. In Linuxbrew mode, post_install.sh has complex logic for changing rpaths # and that is not updated to allow us to find libraries in postgres/lib. logging.info( "Not installing any of the above YSQL libraries because they will be " "installed as part of copying the entire postgres directory." ) continue if category == 'linuxbrew': category_dest_dir = linuxbrew_lib_dest_dir else: category_dest_dir = os.path.join(dest_lib_dir, category) mkdir_p(category_dest_dir) for dep in deps_in_category: additional_libs.discard(dep.target) self.install_dyn_linked_binary(dep.target, category_dest_dir) target_name = os.path.basename(dep.target) if target_name != dep.name: target_src = os.path.join(os.path.dirname(dep.target), dep.name) additional_libs.discard(target_src) symlink(target_name, os.path.join(category_dest_dir, dep.name)) for lib_path in additional_libs: if os.path.isfile(lib_path): self.install_dyn_linked_binary(lib_path, linuxbrew_lib_dest_dir) logging.info("Installed additional lib: " + lib_path) elif os.path.islink(lib_path): link_target_basename = os.path.basename(os.readlink(lib_path)) logging.info("Installed additional symlink: " + lib_path) symlink( link_target_basename, os.path.join(linuxbrew_lib_dest_dir, os.path.basename(lib_path))) else: raise RuntimeError( "Expected '{}' to be a file or a symlink".format(lib_path)) for installed_binary in self.installed_dyn_linked_binaries: # Sometimes files that we copy from other locations are not even writable by user! subprocess.check_call(['chmod', 'u+w', installed_binary]) if using_linuxbrew(): # Remove rpath (we will set it appropriately in post_install.sh). run_patchelf(['--remove-rpath', installed_binary]) post_install_path = os.path.join(self.main_dest_bin_dir, 'post_install.sh') if using_linuxbrew(): # Add other files used by glibc at runtime. assert linuxbrew_home is not None linuxbrew_glibc_real_path = os.path.normpath( os.path.join(os.path.realpath(linuxbrew_home.ldd_path), '..', '..')) assert linuxbrew_home is not None linuxbrew_glibc_rel_path = os.path.relpath( linuxbrew_glibc_real_path, os.path.realpath(linuxbrew_home.linuxbrew_dir)) # We expect glibc to live under a path like "Cellar/glibc/3.23" in # the Linuxbrew directory. if not linuxbrew_glibc_rel_path.startswith('Cellar/glibc/'): raise ValueError( "Expected to find glibc under Cellar/glibc/<version> in Linuxbrew, but found it" " at: '%s'" % linuxbrew_glibc_rel_path) rel_paths = [] for glibc_rel_path in [ 'etc/ld.so.cache', 'etc/localtime', 'lib/locale/locale-archive', 'lib/gconv', 'libexec/getconf', 'share/locale', 'share/zoneinfo', ]: rel_paths.append( os.path.join(linuxbrew_glibc_rel_path, glibc_rel_path)) terminfo_glob_pattern = os.path.join( linuxbrew_home.linuxbrew_dir, 'Cellar/ncurses/*/share/terminfo') terminfo_paths = glob.glob(terminfo_glob_pattern) if len(terminfo_paths) != 1: raise ValueError( "Failed to find the terminfo directory using glob pattern %s. " "Found: %s" % (terminfo_glob_pattern, terminfo_paths)) terminfo_rel_path = os.path.relpath(terminfo_paths[0], linuxbrew_home.linuxbrew_dir) rel_paths.append(terminfo_rel_path) for rel_path in rel_paths: src = os.path.join(linuxbrew_home.linuxbrew_dir, rel_path) dst = os.path.join(linuxbrew_dest_dir, rel_path) copy_deep(src, dst, create_dst_dir=True) with open(post_install_path) as post_install_script_input: post_install_script = post_install_script_input.read() new_post_install_script = post_install_script replacements = [ ("original_linuxbrew_path_to_patch", linuxbrew_home.linuxbrew_dir), ("original_linuxbrew_path_length", len(linuxbrew_home.linuxbrew_dir)), ] for macro_var_name, list_of_binary_names in [ ("main_elf_names_to_patch", main_elf_names_to_patch), ("postgres_elf_names_to_patch", postgres_elf_names_to_patch), ]: replacements.append( (macro_var_name, self.join_binary_names_for_bash(list_of_binary_names))) for macro_var_name, value in replacements: new_post_install_script = new_post_install_script.replace( '${%s}' % macro_var_name, str(value)) else: new_post_install_script = ( '#!/usr/bin/env bash\n' '# For backward-compatibility only. We do not need this script anymore.\n' ) with open(post_install_path, 'w') as post_install_script_output: post_install_script_output.write(new_post_install_script)
def package_binaries(self): """ The main entry point to this class. Arranges binaries (executables and shared libraries), starting with the given set of "seed executables", in the destination directory so that the executables can find all of their dependencies. """ all_deps = [] dest_lib_dir = os.path.join(self.dest_dir, 'lib') mkdir_p(dest_lib_dir) mkdir_p(self.main_dest_bin_dir) mkdir_p(self.postgres_dest_bin_dir) main_elf_names_to_patch = [] postgres_elf_names_to_patch = [] for seed_executable_glob in self.seed_executable_patterns: glob_results = glob.glob(seed_executable_glob) if not glob_results: raise RuntimeError("No files found matching the pattern '{}'".format( seed_executable_glob)) for executable in glob_results: deps = self.find_elf_dependencies(executable) all_deps += deps dest_bin_dir = self.get_dest_bin_dir_for_executable(executable) if deps: self.install_dyn_linked_binary(executable, dest_bin_dir) executable_basename = os.path.basename(executable) if self.is_postgres_binary(executable): postgres_elf_names_to_patch.append(executable_basename) else: main_elf_names_to_patch.append(executable_basename) else: # This is probably a script. shutil.copy(executable, dest_bin_dir) # Not using the install_dyn_linked_binary method for copying patchelf and ld.so as we won't # need to do any post-processing on these two later. shutil.copy(linuxbrew_home.patchelf_path, self.main_dest_bin_dir) ld_path = linuxbrew_home.ld_so_path shutil.copy(ld_path, dest_lib_dir) all_deps = sorted(set(all_deps)) for dep_name, deps in sorted_grouped_by(all_deps, lambda dep: dep.name): targets = sorted(set([dep.target for dep in deps])) if len(targets) > 1: raise RuntimeError( "Multiple dependencies with the same name {} but different targets: {}".format( dep_name, deps )) linuxbrew_dest_dir = os.path.join(self.dest_dir, 'linuxbrew') linuxbrew_lib_dest_dir = os.path.join(linuxbrew_dest_dir, 'lib') for category, deps_in_category in sorted_grouped_by(all_deps, lambda dep: dep.get_category()): logging.info("Found {} dependencies in category '{}':".format( len(deps_in_category), category)) max_name_len = max([len(dep.name) for dep in deps_in_category]) for dep in sorted(deps_in_category, key=lambda dep: dep.target): logging.info(" {} -> {}".format( dep.name + ' ' * (max_name_len - len(dep.name)), dep.target)) if category == 'linuxbrew': category_dest_dir = linuxbrew_lib_dest_dir else: category_dest_dir = os.path.join(dest_lib_dir, category) mkdir_p(category_dest_dir) for dep in deps_in_category: self.install_dyn_linked_binary(dep.target, category_dest_dir) if os.path.basename(dep.target) != dep.name: symlink(os.path.basename(dep.target), os.path.join(category_dest_dir, dep.name)) # Add libresolv and libnss_* libraries explicitly because they are loaded by glibc at # runtime and will not be discovered automatically using ldd. for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS: for lib_path in glob.glob(os.path.join(linuxbrew_home.cellar_glibc_dir, '*', 'lib', additional_lib_name_glob)): lib_basename = os.path.basename(lib_path) if lib_basename.endswith('.a'): continue if os.path.isfile(lib_path): self.install_dyn_linked_binary(lib_path, linuxbrew_lib_dest_dir) elif os.path.islink(lib_path): link_target_basename = os.path.basename(os.readlink(lib_path)) symlink(link_target_basename, os.path.join(linuxbrew_lib_dest_dir, lib_basename)) else: raise RuntimeError( "Expected '{}' to be a file or a symlink".format(lib_path)) for installed_binary in self.installed_dyn_linked_binaries: # Sometimes files that we copy from other locations are not even writable by user! subprocess.check_call(['chmod', 'u+w', installed_binary]) # Remove rpath (we will set it appropriately in post_install.sh). run_patchelf('--remove-rpath', installed_binary) # Add other files used by glibc at runtime. linuxbrew_glibc_real_path = os.path.normpath( os.path.join(os.path.realpath(linuxbrew_home.ldd_path), '..', '..')) linuxbrew_glibc_rel_path = os.path.relpath( linuxbrew_glibc_real_path, os.path.realpath(linuxbrew_home.linuxbrew_dir)) # We expect glibc to live under a path like "Cellar/glibc/2.23" in the Linuxbrew directory. if not linuxbrew_glibc_rel_path.startswith('Cellar/glibc/'): raise ValueError( "Expected to find glibc under Cellar/glibc/<version> in Linuxbrew, but found it " "at: '%s'" % linuxbrew_glibc_rel_path) rel_paths = [] for glibc_rel_path in [ 'etc/ld.so.cache', 'etc/localtime', 'lib/locale/locale-archive', 'lib/gconv', 'libexec/getconf', 'share/locale', 'share/zoneinfo', ]: rel_paths.append(os.path.join(linuxbrew_glibc_rel_path, glibc_rel_path)) terminfo_glob_pattern = os.path.join( linuxbrew_home.linuxbrew_dir, 'Cellar/ncurses/*/share/terminfo') terminfo_paths = glob.glob(terminfo_glob_pattern) if len(terminfo_paths) != 1: raise ValueError( "Failed to find the terminfo directory using glob pattern %s. " "Found: %s" % (terminfo_glob_pattern, terminfo_paths)) terminfo_rel_path = os.path.relpath(terminfo_paths[0], linuxbrew_home.linuxbrew_dir) rel_paths.append(terminfo_rel_path) for rel_path in rel_paths: src = os.path.join(linuxbrew_home.linuxbrew_dir, rel_path) dst = os.path.join(linuxbrew_dest_dir, rel_path) copy_deep(src, dst, create_dst_dir=True) post_install_path = os.path.join(self.main_dest_bin_dir, 'post_install.sh') with open(post_install_path) as post_install_script_input: post_install_script = post_install_script_input.read() new_post_install_script = post_install_script replacements = [ ("original_linuxbrew_path_to_patch", linuxbrew_home.linuxbrew_dir), ("original_linuxbrew_path_length", len(linuxbrew_home.linuxbrew_dir)), ] for macro_var_name, list_of_binary_names in [ ("main_elf_names_to_patch", main_elf_names_to_patch), ("postgres_elf_names_to_patch", postgres_elf_names_to_patch), ]: replacements.append( (macro_var_name, self.join_binary_names_for_bash(list_of_binary_names))) for macro_var_name, value in replacements: new_post_install_script = new_post_install_script.replace( '${%s}' % macro_var_name, str(value)) with open(post_install_path, 'w') as post_install_script_output: post_install_script_output.write(new_post_install_script)