def expand_value(self, old_value): """ Expand old_value with the following changes: - Replace ${project.version} with the Java version from pom.xml. - Replace the leading "thirdparty/" with the respective YB_THIRDPARTY_DIR from the build. - Replace $BUILD_ROOT with the actual build_root. """ # Substitution for Java. new_value = old_value.replace('${project.version}', self.java_project_version) # Substitution for thirdparty. thirdparty_prefix_match = THIRDPARTY_PREFIX_RE.match(new_value) if thirdparty_prefix_match: new_value = os.path.join(get_thirdparty_dir(), thirdparty_prefix_match.group(1)) # Substitution for BUILD_ROOT. new_value = new_value.replace("$BUILD_ROOT", self.build_root) thirdparty_intrumentation = "clang_uninstrumented" if is_macos( ) else "uninstrumented" new_value = new_value.replace( "$THIRDPARTY_BUILD_SPECIFIC_DIR", os.path.join(get_thirdparty_dir(), "installed", thirdparty_intrumentation)) if new_value != old_value: logging.info("Substituting '{}' -> '{}' in manifest".format( old_value, new_value)) return new_value
def get_category(self) -> str: """ Categorizes binaries into a few buckets: - yb -- built as part of YugabyteDB - yb-thirdparty -- built as part of yugabyte-db-thirdparty - linuxbrew -- built using Linuxbrew - system -- a library residing in a system-wide library directory. We do not copy these. - postgres -- libraries inside the postgres directory """ if self.category: return self.category linuxbrew_home = get_linuxbrew_home() if linuxbrew_home is not None and linuxbrew_home.path_is_in_linuxbrew_dir( self.target): self.category = 'linuxbrew' elif self.target.startswith(get_thirdparty_dir() + '/'): self.category = 'yb-thirdparty' elif self.target.startswith(self.context.build_dir + '/postgres/'): self.category = 'postgres' elif self.target.startswith(self.context.build_dir + '/'): self.category = 'yb' elif (self.target.startswith(YB_SCRIPT_BIN_DIR + '/') or self.target.startswith(YB_BUILD_SUPPORT_DIR + '/')): self.category = 'yb-scripts' if not self.category: for system_library_path in SYSTEM_LIBRARY_PATHS: if self.target.startswith(system_library_path + '/'): self.category = 'system' break if self.category: if self.category not in LIBRARY_CATEGORIES: raise RuntimeError(( "Internal error: library category computed as '{}', must be one of: {}. " + "Dependency: {}").format(self.category, LIBRARY_CATEGORIES, self)) return self.category if linuxbrew_home: linuxbrew_dir_str = linuxbrew_home.get_human_readable_dirs() else: linuxbrew_dir_str = 'N/A' raise RuntimeError(( "Could not determine the category of this binary " "(yugabyte / yb-thirdparty / linuxbrew / system): '{}'. " "Does not reside in the Linuxbrew directory ({}), " "YB third-party directory ('{}'), " "YB build directory ('{}'), " "YB general-purpose script directory ('{}'), " "YB build support script directory ('{}'), " "and does not appear to be a system library (does not start with any of {})." ).format(self.target, linuxbrew_dir_str, get_thirdparty_dir(), self.context.build_dir, YB_SCRIPT_BIN_DIR, YB_BUILD_SUPPORT_DIR, SYSTEM_LIBRARY_PATHS))
def _rewrite_manifest(self): """ Rewrite the release manifest with the following changes: - Replace ${project.version} with the Java version from pom.xml. - Replace the leading "thirdparty/" with the respective YB_THIRDPARTY_DIR from the build. - Replace $BUILD_ROOT with the actual build_root. """ pom_file = os.path.join(self.repo, 'java', 'pom.xml') java_project_version = minidom.parse(pom_file).getElementsByTagName( 'version')[0].firstChild.nodeValue logging.info("Java project version from pom.xml: {}".format( java_project_version)) for key, value_list in self.release_manifest.iteritems(): for i in xrange(len(value_list)): old_value = value_list[i] # Substitution for Java. new_value = old_value.replace('${project.version}', java_project_version) # Substitution for thirdparty. thirdparty_prefix_match = THIRDPARTY_PREFIX_RE.match(new_value) if thirdparty_prefix_match: new_value = os.path.join(get_thirdparty_dir(), thirdparty_prefix_match.group(1)) # Substitution for BUILD_ROOT. new_value = new_value.replace("$BUILD_ROOT", self.build_root) if new_value != value_list[i]: logging.info( "Substituting '{}' -> '{}' in manifest".format( value_list[i], new_value)) value_list[i] = new_value
def rewrite_manifest(self, build_root): """ Rewrite the release manifest with the following changes: - Replace ${project.version} with the Java version from pom.xml. - Replace the leading "thirdparty/" with the respective YB_THIRDPARTY_DIR from the build. - Replace $BUILD_ROOT with the actual build_root. """ pom_file = os.path.join(self.repo, 'java', 'pom.xml') java_project_version = minidom.parse(pom_file).getElementsByTagName( 'version')[0].firstChild.nodeValue logging.info("Java project version from pom.xml: {}".format(java_project_version)) for key, value_list in self.release_manifest.iteritems(): for i in xrange(len(value_list)): old_value = value_list[i] # Substitution for Java. new_value = old_value.replace('${project.version}', java_project_version) # Substitution for thirdparty. thirdparty_prefix_match = THIRDPARTY_PREFIX_RE.match(new_value) if thirdparty_prefix_match: new_value = os.path.join(get_thirdparty_dir(), thirdparty_prefix_match.group(1)) # Substitution for BUILD_ROOT. new_value = new_value.replace("$BUILD_ROOT", build_root) if new_value != value_list[i]: logging.info("Substituting '{}' -> '{}' in manifest".format( value_list[i], new_value)) value_list[i] = new_value
def get_category(self): """ Categorizes binaries into a few buckets: - yb -- YugaByte product itself - yb-thirdparty -- built with YugaByte - linuxbrew -- built using Linuxbrew - system -- grabbed from a system-wide library directory """ if self.category: return self.category if self.target.startswith(LINUXBREW_HOME + '/'): self.category = 'linuxbrew' elif self.target.startswith(get_thirdparty_dir() + '/'): self.category = 'yb-thirdparty' elif self.target.startswith(self.context.build_dir + '/'): self.category = 'yb' elif (self.target.startswith(YB_SCRIPT_BIN_DIR + '/') or self.target.startswith(YB_BUILD_SUPPORT_DIR + '/')): self.category = 'yb-scripts' if not self.category: for system_library_path in SYSTEM_LIBRARY_PATHS: if self.target.startswith(system_library_path + '/'): self.category = 'system' break if self.category: if self.category not in LIBRARY_CATEGORIES: raise RuntimeError(( "Internal error: library category computed as '{}', must be one of: {}. " + "Dependency: {}").format(self.category, LIBRARY_CATEGORIES, self)) return self.category raise RuntimeError(( "Could not determine the category of this binary " "(yugabyte / yb-thirdparty / linuxbrew / system): '{}'. " "Does not reside in the Linuxbrew directory ('{}'), " "YB third-party directory ('{}'), " "YB build directory ('{}'), " "YB general-purpose script directory ('{}'), " "YB build support script directory ('{}'), " "and does not appear to be a system library (does not start with any of {})." ).format(self.target, LINUXBREW_HOME, get_thirdparty_dir(), self.context.build_dir, YB_SCRIPT_BIN_DIR, YB_BUILD_SUPPORT_DIR, SYSTEM_LIBRARY_PATHS))
def get_category(self): """ Categorizes binaries into a few buckets: - yb -- YugaByte product itself - yb-thirdparty -- built with YugaByte - linuxbrew -- built using Linuxbrew - system -- grabbed from a system-wide library directory """ if self.category: return self.category if self.target.startswith(LINUXBREW_HOME + '/'): self.category = 'linuxbrew' elif self.target.startswith(get_thirdparty_dir() + '/'): self.category = 'yb-thirdparty' elif self.target.startswith(self.context.build_dir + '/'): self.category = 'yb' elif (self.target.startswith(YB_SCRIPT_BIN_DIR + '/') or self.target.startswith(YB_BUILD_SUPPORT_DIR + '/')): self.category = 'yb-scripts' if not self.category: for system_library_path in SYSTEM_LIBRARY_PATHS: if self.target.startswith(system_library_path + '/'): self.category = 'system' break if self.category: if self.category not in LIBRARY_CATEGORIES: raise RuntimeError( ("Internal error: library category computed as '{}', must be one of: {}. " + "Dependency: {}").format(self.category, LIBRARY_CATEGORIES, self)) return self.category raise RuntimeError( ("Could not determine the category of this binary " "(yugabyte / yb-thirdparty / linuxbrew / system): '{}'. " "Does not reside in the Linuxbrew directory ('{}'), " "YB third-party directory ('{}'), " "YB build directory ('{}'), " "YB general-purpose script directory ('{}'), " "YB build support script directory ('{}'), " "and does not appear to be a system library (does not start with any of {})." ).format(self.target, LINUXBREW_HOME, get_thirdparty_dir(), self.context.build_dir, YB_SCRIPT_BIN_DIR, YB_BUILD_SUPPORT_DIR, SYSTEM_LIBRARY_PATHS))
def main(): parser = argparse.ArgumentParser( description='Run FOSSA analysis (open source license compliance).') parser.add_argument('--verbose', action='store_true', help='Enable verbose output') parser.add_argument( 'fossa_cli_args', nargs='*', help='These arguments are passed directly to fossa-cli') args = parser.parse_args() init_env(args.verbose) # TODO: We may also want to try using the v2 option --unpack-archives # Though that may be going to deeper level than we want. fossa_cmd_line = ['fossa', 'analyze'] fossa_cmd_line.extend(args.fossa_cli_args) should_upload = not any( arg in args.fossa_cli_args for arg in ('--show-output', '--output', '-o')) if should_upload and not os.getenv('FOSSA_API_KEY'): # --output is used for local analysis only, without uploading the results. In all other # cases we would like . raise RuntimeError('FOSSA_API_KEY must be specified in order to upload analysis results.') logging.info( f"FOSSA CLI command line: {shlex_join(fossa_cmd_line)}") fossa_version_str = subprocess.check_output(['fossa', '--version']).decode('utf-8') fossa_version_match = FOSSA_VERSION_RE.match(fossa_version_str) if not fossa_version_match: raise RuntimeError(f"Cannot parse fossa-cli version: {fossa_version_str}") fossa_version = fossa_version_match.group(1) if version.parse(fossa_version) < version.parse(MIN_FOSSA_CLI_VERSION): raise RuntimeError( f"fossa version too old: {fossa_version} " f"(expected {MIN_FOSSA_CLI_VERSION} or later)") download_cache_path = get_download_cache_dir() logging.info(f"Using the download cache directory {download_cache_path}") download_config = DownloadConfig( verbose=args.verbose, cache_dir_path=download_cache_path ) downloader = Downloader(download_config) fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa-local.yml') fossa_yml_data = load_yaml_file(fossa_yml_path) modules = fossa_yml_data['analyze']['modules'] # fossa v2.6.1 does not pick up project name from config file version 2 format. # TODO: update to config file version 3 fossa_cmd_line.extend(["--project", fossa_yml_data['cli']['project']]) thirdparty_dir = get_thirdparty_dir() fossa_modules_path = os.path.join(thirdparty_dir, 'fossa_modules.yml') seen_urls = set() start_time_sec = time.time() if os.path.exists(fossa_modules_path): thirdparty_fossa_modules_data = load_yaml_file(fossa_modules_path) for thirdparty_module_data in thirdparty_fossa_modules_data: fossa_module_data = thirdparty_module_data['fossa_module'] module_name = fossa_module_data['name'] if not should_include_fossa_module(module_name): continue fossa_module_yb_metadata = thirdparty_module_data['yb_metadata'] expected_sha256 = fossa_module_yb_metadata['sha256sum'] url = fossa_module_yb_metadata['url'] if url in seen_urls: # Due to a bug in some versions of yugabyte-db-thirdparty scripts, as of 04/20/2021 # we may include the same dependency twice in the fossa_modules.yml file. We just # skip the duplicates here. continue seen_urls.add(url) logging.info(f"Adding module from {url}") downloaded_path = downloader.download_url( url, download_parent_dir_path=None, # Download to cache directly. verify_checksum=True, expected_sha256=expected_sha256 ) fossa_module_data['target'] = downloaded_path modules.append(fossa_module_data) # TODO: Once we move to v2 fossa, we may want to use fossa-dep.yml file instead of # re-writing the main file. effective_fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa.yml') write_yaml_file(fossa_yml_data, effective_fossa_yml_path) logging.info(f"Wrote the expanded FOSSA file to {effective_fossa_yml_path}") else: logging.warning( f"File {fossa_modules_path} does not exist. Some C/C++ dependencies will be missing " f"from FOSSA analysis.") effective_fossa_yml_path = fossa_yml_path elapsed_time_sec = time.time() - start_time_sec logging.info("Generated the effective FOSSA configuration file in %.1f sec", elapsed_time_sec) logging.info(f"Running command: {shlex_join(fossa_cmd_line)})") subprocess.check_call(fossa_cmd_line)
# and because some libraries might also have system library directories on their rpath. run_patchelf('--remove-rpath', installed_binary) post_install_path = os.path.join(dest_bin_dir, 'post_install.sh') with open(post_install_path) as post_install_script_input: post_install_script = post_install_script_input.read() binary_names_to_patch = ' '.join( ['"{}"'.format(name) for name in elf_names_to_set_interpreter]) new_post_install_script = post_install_script.replace( '${elf_names_to_set_interpreter}', binary_names_to_patch) with open(post_install_path, 'w') as post_install_script_output: post_install_script_output.write(new_post_install_script) if __name__ == '__main__': if not os.path.isdir(get_thirdparty_dir()): raise RuntimeError( "Third-party dependency directory '{}' does not exist".format( get_thirdparty_dir())) parser = argparse.ArgumentParser(description=LibraryPackager.__doc__) parser.add_argument( '--build-dir', help='Build directory to pick up executables/libraries from.', required=True) parser.add_argument( '--dest-dir', help='Destination directory to save the self-sufficient directory tree ' 'of executables and libraries at.', required=True) parser.add_argument(
def package_binaries(self): src = self.build_dir dst = self.dest_dir dst_bin_dir = os.path.join(dst, 'bin') dst_lib_dir = os.path.join(dst, 'lib') try: os.makedirs(dst_bin_dir) except OSError as e: raise RuntimeError('Unable to create directory %s', dst) logging.debug('Created directory %s', dst) bin_dir_files = [] for seed_executable_glob in self.seed_executable_patterns: if seed_executable_glob.find('postgres/bin/') >= 0: # Skip postgres binaries since they are copied with the postgres root directory # which is handled below. continue if seed_executable_glob.startswith('bin/'): bin_dir_files.append(os.path.basename(seed_executable_glob)) logging.debug("Adding file '%s' to bash_scripts", seed_executable_glob) updated_glob = seed_executable_glob.replace( '$BUILD_ROOT', self.build_dir) if updated_glob != seed_executable_glob: logging.info('Substituting: {} -> {}'.format( seed_executable_glob, updated_glob)) seed_executable_glob = updated_glob glob_results = glob.glob(seed_executable_glob) if not glob_results: raise RuntimeError( "No files found matching the pattern '{}'".format( seed_executable_glob)) for executable in glob_results: shutil.copy(executable, dst_bin_dir) src_lib_dir = os.path.join(src, 'lib') yb_lib_file_for_postgres = os.path.join(src_lib_dir, 'libyb_pggate.dylib') libedit_file_for_postgres = os.path.join( get_thirdparty_dir(), 'installed/common/lib/libedit.dylib') processed_libs = [] for bin_file in os.listdir(dst_bin_dir): if bin_file.endswith('.sh') or bin_file in bin_dir_files: logging.info( "Not modifying rpath for file '%s' because it's not a binary file", bin_file) continue logging.debug('Processing binary file: %s', bin_file) libs = [] os.makedirs(os.path.join(dst, 'lib', bin_file)) libs = self.fix_load_paths( os.path.join(dst_bin_dir, bin_file), os.path.join(dst_lib_dir, bin_file), os.path.join('@loader_path/../lib/', bin_file)) # Elements in libs are absolute paths. logging.debug('library dependencies for file %s: %s', bin_file, libs) # Treat this as a special case for now (10/14/18). libs.append(yb_lib_file_for_postgres) libs.append(libedit_file_for_postgres) for lib in libs: if lib in processed_libs: continue # For each library dependency, check whether it already has its own directory (if it # does, a physical copy of this library must exist there). If it doesn't, create it # and copy the physical file there. logging.debug('Processing library: %s', lib) libname = os.path.basename(lib) lib_dir_path = os.path.join(dst, 'lib', libname) if os.path.exists(lib_dir_path): continue os.mkdir(lib_dir_path) shutil.copy(lib, lib_dir_path) lib_file_path = os.path.join(lib_dir_path, libname) new_libs = self.fix_load_paths(lib_file_path, lib_dir_path, '@loader_path') for new_lib in new_libs: if new_lib not in processed_libs and new_lib not in libs: logging.info('Adding dependency %s for library %s', new_lib, lib_file_path) libs.append(new_lib) processed_libs.append(lib) # Handle postgres as a special case for now (10/14/18). postgres_src = os.path.join(src, 'postgres') postgres_dst = os.path.join(dst, 'postgres') shutil.copytree(postgres_src, postgres_dst, symlinks=True) postgres_bin = os.path.join(postgres_dst, 'bin') postgres_lib = os.path.join(postgres_dst, 'lib') for bin_file in os.listdir(postgres_bin): self.fix_postgres_load_paths(os.path.join(postgres_bin, bin_file), dst) for lib_file in os.listdir(postgres_lib): if os.path.isdir(os.path.join(postgres_lib, lib_file)): continue logging.debug("Processing postgres library %s", lib_file) self.fix_postgres_load_paths(os.path.join(postgres_lib, lib_file), dst)
# Remove rpath (we will set it appropriately in post_install.sh). run_patchelf('--remove-rpath', installed_binary) post_install_path = os.path.join(dest_bin_dir, 'post_install.sh') with open(post_install_path) as post_install_script_input: post_install_script = post_install_script_input.read() binary_names_to_patch = ' '.join([ '"{}"'.format(name) for name in elf_names_to_set_interpreter]) new_post_install_script = post_install_script.replace('${elf_names_to_set_interpreter}', binary_names_to_patch) with open(post_install_path, 'w') as post_install_script_output: post_install_script_output.write(new_post_install_script) if __name__ == '__main__': if not os.path.isdir(get_thirdparty_dir()): raise RuntimeError("Third-party dependency directory '{}' does not exist".format( get_thirdparty_dir())) parser = argparse.ArgumentParser(description=LibraryPackager.__doc__) parser.add_argument('--build-dir', help='Build directory to pick up executables/libraries from.', required=True) parser.add_argument('--dest-dir', help='Destination directory to save the self-sufficient directory tree ' 'of executables and libraries at.', required=True) parser.add_argument('--clean-dest', help='Remove the destination directory if it already exists. Only works ' 'with directories under /tmp/... for safety.', action='store_true')