def main() -> None: args = parse_args() init_env(verbose=args.verbose) if args.update: updater = MetadataUpdater( github_token_file_path=args.github_token_file, tag_filter_regex_str=args.tag_filter_regex) updater.update_archive_metadata_file() return metadata = load_metadata() manual_metadata = load_manual_metadata() if args.get_sha1: print(metadata[SHA_FOR_LOCAL_CHECKOUT_KEY]) return metadata['archives'].extend(manual_metadata['archives']) if args.save_download_url_to_file: if not args.compiler_type: raise ValueError("Compiler type not specified") url = get_download_url(metadata=metadata, compiler_type=args.compiler_type, os_type=args.os_type, architecture=args.architecture) if url is None: raise RuntimeError("Could not determine download URL") logging.info(f"Download URL for the third-party dependencies: {url}") output_file_dir = os.path.dirname( os.path.abspath(args.save_download_url_to_file)) os.makedirs(output_file_dir, exist_ok=True) with open(args.save_download_url_to_file, 'w') as output_file: output_file.write(url)
def main() -> None: args = parse_args() init_env(verbose=args.verbose) if args.update: updater = MetadataUpdater( github_token_file_path=args.github_token_file, tag_filter_regex_str=args.tag_filter_regex) updater.update_archive_metadata_file() return metadata = load_metadata() manual_metadata = load_manual_metadata() if args.get_sha1: print(metadata[SHA_FOR_LOCAL_CHECKOUT_KEY]) return metadata_items = [ MetadataItem(item_json_data) for item_json_data in metadata['archives'] + (manual_metadata['archives'] or []) ] if args.list_compilers: compiler_list = get_compilers( metadata_items=metadata_items, os_type=args.os_type, architecture=args.architecture) for compiler in compiler_list: print(compiler) return if args.save_thirdparty_url_to_file or args.save_llvm_url_to_file: if not args.compiler_type: raise ValueError("Compiler type not specified") thirdparty_release: Optional[MetadataItem] = get_third_party_release( available_archives=metadata_items, compiler_type=args.compiler_type, os_type=args.os_type, architecture=args.architecture, is_linuxbrew=args.is_linuxbrew, lto=args.lto) if thirdparty_release is None: raise RuntimeError("Could not determine third-party archive download URL") thirdparty_url = thirdparty_release.url() logging.info(f"Download URL for the third-party dependencies: {thirdparty_url}") if args.save_thirdparty_url_to_file: make_parent_dir(args.save_thirdparty_url_to_file) write_file(thirdparty_url, args.save_thirdparty_url_to_file) if (args.save_llvm_url_to_file and thirdparty_release.compiler_type.startswith('clang') and thirdparty_release.is_linuxbrew): llvm_url = get_llvm_url(thirdparty_release.compiler_type) if llvm_url is not None: logging.info(f"Download URL for the LLVM toolchain: {llvm_url}") make_parent_dir(args.save_llvm_url_to_file) write_file(llvm_url, args.save_llvm_url_to_file) else: logging.info("Could not determine LLVM URL for compiler type %s" % thirdparty_release.compiler_type) else: logging.info("Not a Linuxbrew URL, not saving LLVM URL to file")
def main() -> None: parser = argparse.ArgumentParser(prog=sys.argv[0]) parser.add_argument('build_args', nargs=argparse.REMAINDER, help='arguments for yb_build.sh') remote.add_common_args(parser) remote.handle_yb_build_cmd_line() args = parser.parse_args() init_env(verbose=args.verbose) remote.load_profile(args, args.profile) remote.apply_default_arg_values(args) os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) remote.log_args(args) escaped_remote_path = remote.sync_changes_with_args(args) if args.skip_build: sys.exit(0) remote_args = [] if args.build_type: remote_args.append(args.build_type) if len(args.build_args) != 0 and args.build_args[0] == '--': remote_args += args.build_args[1:] else: remote_args += args.build_args if '--host-for-tests' not in remote_args and 'YB_HOST_FOR_RUNNING_TESTS' in os.environ: remote_args = add_extra_yb_build_args( remote_args, ['--host-for-tests', os.environ['YB_HOST_FOR_RUNNING_TESTS']]) remote.exec_command( host=args.host, escaped_remote_path=escaped_remote_path, script_name='yb_build.sh', script_args=remote_args, should_quote_args=True, extra_ssh_args=remote.process_extra_ssh_args(args.extra_ssh_args))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--build', help='Build type (debug/release)', dest='build_type') parser.add_argument( '--build_root', help= 'The root directory where the code is being built. If the build type ' 'is specified, it needs to be consistent with the build root.') parser.add_argument( '--build_args', default='', help='Additional arguments to pass to the build script') parser.add_argument( '--build_archive', action='store_true', help='Whether or not we should build a package. This defaults to ' 'false if --build_target is specified, true otherwise.') parser.add_argument('--destination', help='Copy release to Destination folder.') parser.add_argument('--force', help='Skip prompts', action='store_true') parser.add_argument('--commit', help='Custom specify a git commit.') parser.add_argument('--skip_build', help='Skip building the code', action='store_true') parser.add_argument( '--build_target', help='Target directory to put the YugaByte distribution into. This can ' 'be used for debugging this script without having to build the ' 'tarball. If specified, this directory must either not exist or be ' 'empty.') parser.add_argument('--keep_tmp_dir', action='store_true', help='Keep the temporary directory (for debugging).') parser.add_argument( '--save_release_path_to_file', help='Save the newly built release path to a file with this name. ' 'This allows to post-process / upload the newly generated release ' 'in an enclosing script.') parser.add_argument('--yw', action='store_true', help='Package YugaWare too.') parser.add_argument( '--no_reinitdb', action='store_true', help='Do not re-create the initial sys catalog snapshot. Useful when ' 'debugging the release process.') add_common_arguments(parser) args = parser.parse_args() # --------------------------------------------------------------------------------------------- # Processing the arguments # --------------------------------------------------------------------------------------------- init_env(args.verbose) if not args.build_target and not args.build_archive: logging.info( "Implying --build_archive (build package) because --build_target " "(a custom directory to put YB distribution files into) is not specified." ) args.build_archive = True if not args.build_archive and args.save_release_path_to_file: raise RuntimeError( '--save_release_path_to_file does not make sense without ' '--build_archive') build_root = args.build_root build_type = args.build_type tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4()))) try: os.mkdir(tmp_dir) except OSError as e: logging.error("Could not create directory at '{}'".format(tmp_dir)) raise e if not args.keep_tmp_dir: atexit.register(lambda: shutil.rmtree(tmp_dir)) yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution') os.chdir(YB_SRC_ROOT) if not args.force: check_for_local_changes() # This is not a "target" in terms of Make / CMake, but a target directory. build_target = args.build_target if build_target is None: # Use a temporary directory. build_target = os.path.join(tmp_dir, 'tmp_yb_distribution') if build_root: build_type_from_build_root = get_build_type_from_build_root(build_root) if build_type: if build_type != build_type_from_build_root: raise RuntimeError(( "Specified build type ('{}') is inconsistent with the specified build " "root ('{}')").format(build_type, build_root)) else: build_type = build_type_from_build_root if not build_type: build_type = 'release' logging.info("Building YugabyteDB {} build".format(build_type)) build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml') build_cmd_list = [ "./yb_build.sh", "--write-build-descriptor", build_desc_path, build_type ] if args.force: build_cmd_list.append("--force") if build_root: # This will force yb_build.sh to use this build directory, and detect build type, # compiler type, etc. based on that. build_cmd_list += ["--build-root", build_root] build_cmd_list += [ # This will build the exact set of targets that are needed for the release. "packaged_targets", # We do not need java code built for a release package "--skip-java" ] if args.skip_build: build_cmd_list += ["--skip-build"] if args.build_args: # TODO: run with shell=True and append build_args as is. build_cmd_list += args.build_args.strip().split() # --------------------------------------------------------------------------------------------- # Perform the build # --------------------------------------------------------------------------------------------- build_cmd_line = " ".join(build_cmd_list).strip() logging.info("Build command line: {}".format(build_cmd_line)) if not args.skip_build: # TODO: figure out the dependency issues in our CMake build instead. # TODO: move this into yb_build.sh itself. for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']: preliminary_step_cmd_list = [ arg for arg in build_cmd_list if arg != 'packaged_targets' ] + ['--target', preliminary_target, '--skip-java'] logging.info("Running a preliminary step to build target %s: %s", preliminary_target, preliminary_step_cmd_list) subprocess.check_call(preliminary_step_cmd_list) # We still need to call this even when --skip-build is specified to generate the "build # descriptor" YAML file. final_build_cmd_list = build_cmd_list + ([] if args.no_reinitdb else ['reinitdb']) logging.info("Running final build step: %s", final_build_cmd_list) subprocess.check_call(final_build_cmd_list) if not os.path.exists(build_desc_path): raise IOError( "The build script failed to generate build descriptor file at '{}'" .format(build_desc_path)) with open(build_desc_path) as build_desc_file: build_desc = yaml.load(build_desc_file) logging.info("Build descriptor: {}".format(build_desc)) build_root_from_build_desc = build_desc['build_root'] if not build_root: build_root = build_root_from_build_desc build_root = os.path.realpath(build_root) if build_root != os.path.realpath(build_root_from_build_desc): raise RuntimeError( "Build root from the build descriptor file (see above) is inconsistent with that " "specified on the command line ('{}')".format(build_root)) # We are guaranteed to have a build_root by now. library_packager_module.set_build_root(build_root) thirdparty_dir = build_desc["thirdparty_dir"] thirdparty_dir_from_env = os.environ.get("YB_THIRDPARTY_DIR", thirdparty_dir) if (thirdparty_dir != thirdparty_dir_from_env and thirdparty_dir_from_env != os.path.join( YB_SRC_ROOT, 'thirdparty')): raise RuntimeError( "Mismatch between non-default valueo of env YB_THIRDPARTY_DIR: '{}' and build desc " "thirdparty_dir: '{}'".format(thirdparty_dir_from_env, thirdparty_dir)) # Set the var for in-memory access to it across the other python files. set_thirdparty_dir(thirdparty_dir) # This points to the release manifest within the release_manager, and we are modifying that # directly. release_util = ReleaseUtil(YB_SRC_ROOT, build_type, build_target, args.force, args.commit, build_root) system = platform.system().lower() library_packager_args = dict( build_dir=build_root, seed_executable_patterns=release_util.get_seed_executable_patterns(), dest_dir=yb_distribution_dir, verbose_mode=args.verbose) if system == "linux": library_packager = LibraryPackager(**library_packager_args) elif system == "darwin": library_packager = MacLibraryPackager(**library_packager_args) else: raise RuntimeError("System {} not supported".format(system)) library_packager.package_binaries() release_util.update_manifest(yb_distribution_dir) logging.info("Generating release distribution") if os.path.exists(build_target) and os.listdir(build_target): raise RuntimeError( "Directory '{}' exists and is non-empty".format(build_target)) release_util.create_distribution(build_target) # --------------------------------------------------------------------------------------------- # Invoke YugaWare packaging # --------------------------------------------------------------------------------------------- if args.yw: managed_dir = os.path.join(YB_SRC_ROOT, "managed") yw_dir = os.path.join(build_target, "ui") if not os.path.exists(yw_dir): os.makedirs(yw_dir) package_yw_cmd = [ os.path.join(managed_dir, "yb_release"), "--destination", yw_dir, "--unarchived" ] logging.info("Creating YugaWare package with command: {}".format( package_yw_cmd)) try: subprocess.check_output(package_yw_cmd, cwd=managed_dir) except subprocess.CalledProcessError as e: logging.error( "Failed to build YugaWare package:\n{}\nOutput:\n{}".format( traceback.format_exc(), e.output)) raise except OSError as e: logging.error("Failed to build YugaWare package: {}".format(e)) raise # --------------------------------------------------------------------------------------------- # Create a tar.gz (we almost always do this, can be skipped in debug mode) # --------------------------------------------------------------------------------------------- if args.build_archive: release_file = os.path.realpath(release_util.generate_release()) if args.destination: if not os.path.exists(args.destination): raise RuntimeError("Destination {} not a directory.".format( args.destination)) shutil.copy(release_file, args.destination) logging.info("Generated a package at '{}'".format(release_file)) if args.save_release_path_to_file: with open(args.save_release_path_to_file, 'w') as release_path_file: release_path_file.write(release_file) logging.info("Saved package path to '{}'".format( args.save_release_path_to_file))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--build', help='Build type (debug/release)', dest='build_type') parser.add_argument('--build_root', help='The root directory where the code is being built. If the build type ' 'is specified, it needs to be consistent with the build root.') parser.add_argument('--build_args', default='', help='Additional arguments to pass to the build script') parser.add_argument('--build_archive', action='store_true', help='Whether or not we should build a package. This defaults to ' 'false if --build_target is specified, true otherwise.') parser.add_argument('--destination', help='Copy release to Destination folder.') parser.add_argument('--force', help='Skip prompts', action='store_true') parser.add_argument('--edition', help='Which edition the code is built as.', default=None, choices=RELEASE_EDITION_ALLOWED_VALUES) parser.add_argument('--commit', help='Custom specify a git commit.') parser.add_argument('--skip_build', help='Skip building the code', action='store_true') parser.add_argument('--build_target', help='Target directory to put the YugaByte distribution into. This can ' 'be used for debugging this script without having to build the ' 'tarball. If specified, this directory must either not exist or be ' 'empty.') parser.add_argument('--keep_tmp_dir', action='store_true', help='Keep the temporary directory (for debugging).') parser.add_argument('--save_release_path_to_file', help='Save the newly built release path to a file with this name. ' 'This allows to post-process / upload the newly generated release ' 'in an enclosing script.') add_common_arguments(parser) args = parser.parse_args() init_env(args.verbose) if not args.build_target and not args.build_archive: logging.info("Implying --build_archive (build package) because --build_target " "(a custom directory to put YB distribution files into) is not specified.") args.build_archive = True if not args.build_archive and args.save_release_path_to_file: raise RuntimeError('--save_release_path_to_file does not make sense without ' '--build_archive') build_root = args.build_root if build_root and not args.edition: build_root_basename = os.path.basename(build_root) if '-community-' in build_root_basename or build_root_basename.endswith('community'): logging.info("Setting edition to Community based on build root") args.edition = RELEASE_EDITION_COMMUNITY elif '-enterprise-' in build_root_basename or build_root_basename.endswith('enterprise'): logging.info("Setting edition to Enterprise based on build root") args.edition = RELEASE_EDITION_ENTERPRISE # We must have either not had a build_root specified, or not been able to deduce from the root. if not args.edition: ent_path = "{}/ent".format(YB_SRC_ROOT) if os.path.isdir(ent_path): logging.info( "No --edition was specified, but found '{}', releasing Enterprise edition.".format( ent_path)) args.edition = RELEASE_EDITION_ENTERPRISE else: logging.info( "No --edition was specified and '{}' is not a valid dir, releasing Community " "edition.".format(ent_path)) args.edition = RELEASE_EDITION_COMMUNITY build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community" build_type = args.build_type tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4()))) try: os.mkdir(tmp_dir) except OSError as e: logging.error("Could not create directory at '{}'".format(tmp_dir)) raise e if not args.keep_tmp_dir: atexit.register(lambda: shutil.rmtree(tmp_dir)) yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution') os.chdir(YB_SRC_ROOT) if not args.force: check_for_local_changes() # This is not a "target" in terms of Make / CMake, but a target directory. build_target = args.build_target if build_target is None: # Use a temporary directory. build_target = os.path.join(tmp_dir, 'tmp_yb_distribution') if build_root: build_type_from_build_root = get_build_type_from_build_root(build_root) if build_type: if build_type != build_type_from_build_root: raise RuntimeError( ("Specified build type ('{}') is inconsistent with the specified build " "root ('{}')").format(build_type, build_root)) else: build_type = build_type_from_build_root if not build_type: build_type = 'release' logging.info("Building YugaByte DB {} Edition: '{}' build".format( build_edition.capitalize(), build_type)) build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml') build_cmd_list = [ "./yb_build.sh", "--write-build-descriptor", build_desc_path, "--edition", build_edition, build_type ] if build_root: # This will force yb_build.sh to use this build directory, and detect build type, # compiler type, edition, etc. based on that. build_cmd_list += ["--build-root", build_root] build_cmd_list += [ # This will build the exact set of targets that are needed for the release. "packaged_targets", # We do not need java code built for a release package "--skip-java" ] if args.skip_build: build_cmd_list += ["--skip-build"] if args.build_args: # TODO: run with shell=True and append build_args as is. build_cmd_list += args.build_args.strip().split() build_cmd_line = " ".join(build_cmd_list).strip() logging.info("Build command line: {}".format(build_cmd_line)) if not args.skip_build: # TODO: figure out the dependency issues in our CMake build instead. # TODO: move this into yb_build.sh itself. for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']: preliminary_step_cmd_list = [ arg for arg in build_cmd_list if arg != 'packaged_targets' ] + ['--target', preliminary_target, '--skip-java'] logging.info( "Running a preliminary step to build target %s: %s", preliminary_target, " ".join(preliminary_step_cmd_list)) subprocess.check_call(preliminary_step_cmd_list) subprocess.check_call(build_cmd_list) if not os.path.exists(build_desc_path): raise IOError("The build script failed to generate build descriptor file at '{}'".format( build_desc_path)) with open(build_desc_path) as build_desc_file: build_desc = yaml.load(build_desc_file) logging.info("Build descriptor: {}".format(build_desc)) build_root_from_build_desc = build_desc['build_root'] if not build_root: build_root = build_root_from_build_desc build_root = os.path.realpath(build_root) if build_root != os.path.realpath(build_root_from_build_desc): raise RuntimeError( "Build root from the build descriptor file (see above) is inconsistent with that " "specified on the command line ('{}')".format(build_root)) thirdparty_dir = build_desc["thirdparty_dir"] thirdparty_dir_from_env = os.environ.get("YB_THIRDPARTY_DIR", thirdparty_dir) if (thirdparty_dir != thirdparty_dir_from_env and thirdparty_dir_from_env != os.path.join(YB_SRC_ROOT, 'thirdparty')): raise RuntimeError( "Mismatch between non-default valueo of env YB_THIRDPARTY_DIR: '{}' and build desc " "thirdparty_dir: '{}'".format(thirdparty_dir_from_env, thirdparty_dir)) # Set the var for in-memory access to it across the other python files. set_thirdparty_dir(thirdparty_dir) # This points to the release manifest within the release_manager, and we are modifying that # directly. release_util = ReleaseUtil( YB_SRC_ROOT, build_type, args.edition, build_target, args.force, args.commit, build_root) system = platform.system().lower() library_packager_args = dict( build_dir=build_root, seed_executable_patterns=release_util.get_seed_executable_patterns(), dest_dir=yb_distribution_dir, verbose_mode=args.verbose ) if system == "linux": library_packager = LibraryPackager(**library_packager_args) elif system == "darwin": library_packager = MacLibraryPackager(**library_packager_args) else: raise RuntimeError("System {} not supported".format(system)) library_packager.package_binaries() release_util.update_manifest(yb_distribution_dir) logging.info("Generating release distribution") if os.path.exists(build_target) and os.listdir(build_target): raise RuntimeError("Directory '{}' exists and is non-empty".format(build_target)) release_util.create_distribution(build_target) if args.build_archive: release_file = os.path.realpath(release_util.generate_release()) if args.destination: if not os.path.exists(args.destination): raise RuntimeError("Destination {} not a directory.".format(args.destination)) shutil.copy(release_file, args.destination) logging.info("Generated a package at '{}'".format(release_file)) if args.save_release_path_to_file: with open(args.save_release_path_to_file, 'w') as release_path_file: release_path_file.write(release_file) logging.info("Saved package path to '{}'".format( args.save_release_path_to_file))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--build', help='Build type (debug/release)', dest='build_type') parser.add_argument('--build_root', help='The root directory where the code is being built. If the build type ' 'is specified, it needs to be consistent with the build root.') parser.add_argument('--build_args', default='', help='Additional arguments to pass to the build script') parser.add_argument('--build_archive', action='store_true', help='Whether or not we should build a package. This defaults to ' 'false if --build_target is specified, true otherwise.') parser.add_argument('--destination', help='Copy release to Destination folder.') parser.add_argument('--force', help='Skip prompts', action='store_true') parser.add_argument('--edition', help='Which edition the code is built as.', default=None, choices=RELEASE_EDITION_ALLOWED_VALUES) parser.add_argument('--commit', help='Custom specify a git commit.') parser.add_argument('--skip_build', help='Skip building the code', action='store_true') parser.add_argument('--build_target', help='Target directory to put the YugaByte distribution into. This can ' 'be used for debugging this script without having to build the ' 'tarball. If specified, this directory must either not exist or be ' 'empty.') parser.add_argument('--keep_tmp_dir', action='store_true', help='Keep the temporary directory (for debugging).') parser.add_argument('--save_release_path_to_file', help='Save the newly built release path to a file with this name. ' 'This allows to post-process / upload the newly generated release ' 'in an enclosing script.') parser.add_argument('--java_only', action="store_true", help="Only build the java part of the code. Does not generate an archive!") add_common_arguments(parser) args = parser.parse_args() init_env(args.verbose) if not args.build_target and not args.build_archive and not args.java_only: logging.info("Implying --build_archive (build package) because --build_target " "(a custom directory to put YB distribution files into) is not specified.") args.build_archive = True if not args.build_archive and args.save_release_path_to_file: raise RuntimeError('--save_release_path_to_file does not make sense without ' '--build_archive') build_root = args.build_root if build_root and not args.edition: build_root_basename = os.path.basename(build_root) if '-community-' in build_root_basename or build_root_basename.endswith('community'): logging.info("Setting edition to Community based on build root") args.edition = RELEASE_EDITION_COMMUNITY if not args.edition: # Here we are not detecting edition based on the existence of the enterprise source # directory. args.edition = RELEASE_EDITION_ENTERPRISE build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community" build_type = args.build_type tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4()))) try: os.mkdir(tmp_dir) except OSError as e: logging.error("Could not create directory at '{}'".format(tmp_dir)) raise e if not args.keep_tmp_dir: atexit.register(lambda: shutil.rmtree(tmp_dir)) yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution') os.chdir(YB_SRC_ROOT) if not args.force: check_for_local_changes() # This is not a "target" in terms of Make / CMake, but a target directory. build_target = args.build_target if build_target is None: # Use a temporary directory. build_target = os.path.join(tmp_dir, 'tmp_yb_distribution') if build_root: build_type_from_build_root = get_build_type_from_build_root(build_root) if build_type: if build_type != build_type_from_build_root: raise RuntimeError( ("Specified build type ('{}') is inconsistent with the specified build " "root ('{}')").format(build_type, build_root)) else: build_type = build_type_from_build_root if not build_type: build_type = 'release' logging.info("Building YugaByte DB {} Edition: '{}' build{}".format( build_edition.capitalize(), build_type, " Java only" if args.java_only else "")) build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml') build_cmd_list = [ "./yb_build.sh", "--with-assembly", "--write-build-descriptor", build_desc_path, "--edition", build_edition, build_type ] if build_root: # This will force yb_build.sh to use this build directory, and detect build type, # compiler type, edition, etc. based on that. build_cmd_list += ["--build-root", build_root] if args.java_only: build_cmd_list += ["--java-only"] else: build_cmd_list += [ # This will build the exact set of targets that are needed for the release. "packaged_targets" ] if args.skip_build: build_cmd_list += ["--skip-build"] if args.build_args: # TODO: run with shell=True and append build_args as is. build_cmd_list += args.build_args.strip().split() build_cmd_line = " ".join(build_cmd_list).strip() logging.info("Build command line: {}".format(build_cmd_line)) if not args.java_only and not args.skip_build: # TODO: figure out the dependency issues in our CMake build instead. # TODO: move this into yb_build.sh itself. for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']: preliminary_step_cmd_list = [ arg for arg in build_cmd_list if arg != 'packaged_targets' ] + ['--target', preliminary_target, '--skip-java'] logging.info( "Running a preliminary step to build target %s: %s", preliminary_target, " ".join(preliminary_step_cmd_list)) subprocess.check_call(preliminary_step_cmd_list) subprocess.check_call(build_cmd_list) if not os.path.exists(build_desc_path): raise IOError("The build script failed to generate build descriptor file at '{}'".format( build_desc_path)) with open(build_desc_path) as build_desc_file: build_desc = yaml.load(build_desc_file) logging.info("Build descriptor: {}".format(build_desc)) build_root_from_build_desc = build_desc['build_root'] if not build_root: build_root = build_root_from_build_desc build_root = os.path.realpath(build_root) if build_root != os.path.realpath(build_root_from_build_desc): raise RuntimeError( "Build root from the build descriptor file (see above) is inconsistent with that " "specified on the command line ('{}')".format(build_root)) thirdparty_dir = build_desc["thirdparty_dir"] if thirdparty_dir != os.environ.get("YB_THIRDPARTY_DIR", thirdparty_dir): raise RuntimeError( "Mismatch between env YB_THIRDPARTY_DIR: '{}' and build desc thirdparty_dir: '{}'" .format(os.environ["YB_THIRDPARTY_DIR"], thirdparty_dir)) # Set the var for in-memory access to it across the other python files. set_thirdparty_dir(thirdparty_dir) # This points to the release manifest within the release_manager, and we are modifying that # directly. release_util = ReleaseUtil(YB_SRC_ROOT, build_type, args.edition, build_target, args.force, args.commit) release_util.rewrite_manifest(build_root) if not args.java_only: system = platform.system().lower() if system == "linux": library_packager = LibraryPackager( build_dir=build_root, seed_executable_patterns=release_util.get_binary_path(), dest_dir=yb_distribution_dir, verbose_mode=args.verbose) elif system == "darwin": library_packager = MacLibraryPackager( build_dir=build_root, seed_executable_patterns=release_util.get_binary_path(), dest_dir=yb_distribution_dir, verbose_mode=args.verbose) else: raise RuntimeError("System {} not supported".format(system)) library_packager.package_binaries() release_util.update_manifest(yb_distribution_dir) logging.info("Generating release distribution") if os.path.exists(build_target) and os.listdir(build_target): if args.java_only: logging.info("Directory '{}' exists and is not empty, but we are using --java_only") else: raise RuntimeError("Directory '{}' exists and is non-empty".format(build_target)) release_util.create_distribution(build_target, "java" if args.java_only else None) if args.build_archive: release_file = os.path.realpath(release_util.generate_release()) if args.destination: if not os.path.exists(args.destination): raise RuntimeError("Destination {} not a directory.".format(args.destination)) shutil.copy(release_file, args.destination) logging.info("Generated a package at '{}'".format(release_file)) if args.save_release_path_to_file: with open(args.save_release_path_to_file, 'w') as release_path_file: release_path_file.write(release_file) logging.info("Saved package path to '{}'".format( args.save_release_path_to_file))
def main(): parser = argparse.ArgumentParser( description='Run FOSSA analysis (open source license compliance).') parser.add_argument('--verbose', action='store_true', help='Enable verbose output') parser.add_argument( 'fossa_cli_args', nargs='*', help='These arguments are passed directly to fossa-cli') args = parser.parse_args() init_env(args.verbose) # TODO: We may also want to try using the v2 option --unpack-archives # Though that may be going to deeper level than we want. fossa_cmd_line = ['fossa', 'analyze'] fossa_cmd_line.extend(args.fossa_cli_args) should_upload = not any( arg in args.fossa_cli_args for arg in ('--show-output', '--output', '-o')) if should_upload and not os.getenv('FOSSA_API_KEY'): # --output is used for local analysis only, without uploading the results. In all other # cases we would like . raise RuntimeError('FOSSA_API_KEY must be specified in order to upload analysis results.') logging.info( f"FOSSA CLI command line: {shlex_join(fossa_cmd_line)}") fossa_version_str = subprocess.check_output(['fossa', '--version']).decode('utf-8') fossa_version_match = FOSSA_VERSION_RE.match(fossa_version_str) if not fossa_version_match: raise RuntimeError(f"Cannot parse fossa-cli version: {fossa_version_str}") fossa_version = fossa_version_match.group(1) if version.parse(fossa_version) < version.parse(MIN_FOSSA_CLI_VERSION): raise RuntimeError( f"fossa version too old: {fossa_version} " f"(expected {MIN_FOSSA_CLI_VERSION} or later)") download_cache_path = get_download_cache_dir() logging.info(f"Using the download cache directory {download_cache_path}") download_config = DownloadConfig( verbose=args.verbose, cache_dir_path=download_cache_path ) downloader = Downloader(download_config) fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa-local.yml') fossa_yml_data = load_yaml_file(fossa_yml_path) modules = fossa_yml_data['analyze']['modules'] # fossa v2.6.1 does not pick up project name from config file version 2 format. # TODO: update to config file version 3 fossa_cmd_line.extend(["--project", fossa_yml_data['cli']['project']]) thirdparty_dir = get_thirdparty_dir() fossa_modules_path = os.path.join(thirdparty_dir, 'fossa_modules.yml') seen_urls = set() start_time_sec = time.time() if os.path.exists(fossa_modules_path): thirdparty_fossa_modules_data = load_yaml_file(fossa_modules_path) for thirdparty_module_data in thirdparty_fossa_modules_data: fossa_module_data = thirdparty_module_data['fossa_module'] module_name = fossa_module_data['name'] if not should_include_fossa_module(module_name): continue fossa_module_yb_metadata = thirdparty_module_data['yb_metadata'] expected_sha256 = fossa_module_yb_metadata['sha256sum'] url = fossa_module_yb_metadata['url'] if url in seen_urls: # Due to a bug in some versions of yugabyte-db-thirdparty scripts, as of 04/20/2021 # we may include the same dependency twice in the fossa_modules.yml file. We just # skip the duplicates here. continue seen_urls.add(url) logging.info(f"Adding module from {url}") downloaded_path = downloader.download_url( url, download_parent_dir_path=None, # Download to cache directly. verify_checksum=True, expected_sha256=expected_sha256 ) fossa_module_data['target'] = downloaded_path modules.append(fossa_module_data) # TODO: Once we move to v2 fossa, we may want to use fossa-dep.yml file instead of # re-writing the main file. effective_fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa.yml') write_yaml_file(fossa_yml_data, effective_fossa_yml_path) logging.info(f"Wrote the expanded FOSSA file to {effective_fossa_yml_path}") else: logging.warning( f"File {fossa_modules_path} does not exist. Some C/C++ dependencies will be missing " f"from FOSSA analysis.") effective_fossa_yml_path = fossa_yml_path elapsed_time_sec = time.time() - start_time_sec logging.info("Generated the effective FOSSA configuration file in %.1f sec", elapsed_time_sec) logging.info(f"Running command: {shlex_join(fossa_cmd_line)})") subprocess.check_call(fossa_cmd_line)
def main(): parser = argparse.ArgumentParser(prog=sys.argv[0]) parser.add_argument( '--host', type=str, default=None, help=( 'Host to build on. Can also be specified using the {} environment ' + 'variable.').format(remote.REMOTE_BUILD_HOST_ENV_VAR)) home = os.path.expanduser('~') cwd = os.getcwd() default_path = '~/{0}'.format( cwd[len(home) + 1:] if cwd.startswith(home + '/') else 'code/yugabyte') # Note: don't specify default arguments here, because they may come from the "profile". parser.add_argument('--remote-path', type=str, help='path used for build') parser.add_argument('--branch', type=str, default=None, help='base branch for build') parser.add_argument('--upstream', type=str, default=None, help='base upstream for remote host to fetch') parser.add_argument('--build-type', type=str, default=None, help='build type') parser.add_argument('--skip-build', action='store_true', help='skip build, only sync files') parser.add_argument('--wait-for-ssh', action='store_true', help='Wait for the remote server to be ssh-able') parser.add_argument('--profile', help='Use a "profile" specified in the {} file'.format( remote.CONFIG_FILE_PATH)) parser.add_argument('--verbose', action='store_true', help='Verbose output') parser.add_argument('build_args', nargs=argparse.REMAINDER, help='arguments for yb_build.sh') if len(sys.argv) >= 2 and sys.argv[1] in ['ybd', 'yb_build.sh']: # Allow the first argument to be 'ybd' so we can copy and paste a ybd command line directly # after remote_build.py. sys.argv[1:2] = ['--'] args = parser.parse_args() init_env(verbose=args.verbose) remote.load_profile(args, args.profile) # --------------------------------------------------------------------------------------------- # Default arguments go here. args.host = remote.apply_default_host_value(args.host) if args.branch is None: args.branch = remote.DEFAULT_BASE_BRANCH if args.remote_path is None: args.remote_path = default_path if args.upstream is None: args.upstream = remote.DEFAULT_UPSTREAM # End of default arguments. # --------------------------------------------------------------------------------------------- os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) print("Host: {0}, build type: {1}, remote path: {2}".format( args.host, args.build_type or 'N/A', args.remote_path)) print("Arguments to remote build: {}".format(args.build_args)) escaped_remote_path = \ remote.sync_changes(args.host, args.branch, args.remote_path, args.wait_for_ssh, args.upstream) if args.skip_build: sys.exit(0) remote_args = [] if args.build_type: remote_args.append(args.build_type) if len(args.build_args) != 0 and args.build_args[0] == '--': remote_args += args.build_args[1:] else: remote_args += args.build_args if '--host-for-tests' not in remote_args and 'YB_HOST_FOR_RUNNING_TESTS' in os.environ: remote_args = add_extra_ybd_args( remote_args, ['--host-for-tests', os.environ['YB_HOST_FOR_RUNNING_TESTS']]) remote.exec_command(args.host, escaped_remote_path, 'yb_build.sh', remote_args, do_quote_args=True)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--build', help='Build type (debug/release)', dest='build_type') parser.add_argument( '--build_root', help= 'The root directory where the code is being built. If the build type ' 'is specified, it needs to be consistent with the build root.') parser.add_argument( '--build_args', default='', help='Additional arguments to pass to the build script') parser.add_argument( '--build_archive', action='store_true', help='Whether or not we should build a package. This defaults to ' 'false if --build_target is specified, true otherwise.') parser.add_argument('--destination', help='Copy release to Destination folder.') parser.add_argument('--force', help='Skip prompts', action='store_true') parser.add_argument('--edition', help='Which edition the code is built as.', default=None, choices=RELEASE_EDITION_ALLOWED_VALUES) parser.add_argument('--commit', help='Custom specify a git commit.') parser.add_argument('--skip_build', help='Skip building the code', action='store_true') parser.add_argument( '--build_target', help='Target directory to put the YugaByte distribution into. This can ' 'be used for debugging this script without having to build the ' 'tarball. If specified, this directory must either not exist or be ' 'empty.') parser.add_argument('--keep_tmp_dir', action='store_true', help='Keep the temporary directory (for debugging).') parser.add_argument( '--save_release_path_to_file', help='Save the newly built release path to a file with this name. ' 'This allows to post-process / upload the newly generated release ' 'in an enclosing script.') add_common_arguments(parser) args = parser.parse_args() init_env(args.verbose) if not args.build_target and not args.build_archive: logging.info( "Implying --build_archive (build package) because --build_target " "(a custom directory to put YB distribution files into) is not specified." ) args.build_archive = True if not args.build_archive and args.save_release_path_to_file: raise RuntimeError( '--save_release_path_to_file does not make sense without ' '--build_archive') build_root = args.build_root if build_root and not args.edition: build_root_basename = os.path.basename(build_root) if '-community-' in build_root_basename or build_root_basename.endswith( 'community'): logging.info("Setting edition to Community based on build root") args.edition = RELEASE_EDITION_COMMUNITY if not args.edition: # Here we are not detecting edition based on the existence of the enterprise source # directory. args.edition = RELEASE_EDITION_ENTERPRISE build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community" build_type = args.build_type tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4()))) try: os.mkdir(tmp_dir) except OSError as e: logging.error("Could not create directory at '{}'".format(tmp_dir)) raise e if not args.keep_tmp_dir: atexit.register(lambda: shutil.rmtree(tmp_dir)) yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution') os.chdir(YB_SRC_ROOT) if not args.force: check_for_local_changes() # This is not a "target" in terms of Make / CMake, but a target directory. build_target = args.build_target if build_target is None: # Use a temporary directory. build_target = os.path.join(tmp_dir, 'tmp_yb_distribution') if build_root: build_type_from_build_root = get_build_type_from_build_root(build_root) if build_type: if build_type != build_type_from_build_root: raise RuntimeError(( "Specified build type ('{}') is inconsistent with the specified build " "root ('{}')").format(build_type, build_root)) else: build_type = build_type_from_build_root if not build_type: build_type = 'release' logging.info("Building YugaByte DB {} Edition: '{}' build".format( build_edition.capitalize(), build_type)) build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml') build_cmd_list = [ "./yb_build.sh", "--with-assembly", "--write-build-descriptor", build_desc_path, "--edition", build_edition, # This will build the exact set of targets that are needed for the release. "packaged_targets", build_type ] if build_root: # This will force yb_build.sh to use this build directory, and detect build type, # compiler type, edition, etc. based on that. build_cmd_list += ["--build-root", build_root] if args.skip_build: build_cmd_list += ["--skip-build"] if args.build_args: build_cmd_list += args.build_args.strip().split() build_cmd_line = " ".join(build_cmd_list).strip() logging.info("Build command line: {}".format(build_cmd_line)) subprocess.check_call(build_cmd_list) if not os.path.exists(build_desc_path): raise IOError( "The build script failed to generate build descriptor file at '{}'" .format(build_desc_path)) with open(build_desc_path) as build_desc_file: build_desc = yaml.load(build_desc_file) logging.info("Build descriptor: {}".format(build_desc)) build_root_from_build_desc = build_desc['build_root'] if not build_root: build_root = build_root_from_build_desc build_root = os.path.realpath(build_root) if build_root != os.path.realpath(build_root_from_build_desc): raise RuntimeError( "Build root from the build descriptor file (see above) is inconsistent with that " "specified on the command line ('{}')".format(build_root)) # This points to the release manifest within the release_manager, and we are modifying that # directly. release_util = ReleaseUtil(YB_SRC_ROOT, build_type, args.edition, build_target, args.force, args.commit) release_util.rewrite_manifest(build_root) system = platform.system().lower() if system == "linux": library_packager = LibraryPackager( build_dir=build_root, seed_executable_patterns=release_util.get_binary_path(), dest_dir=yb_distribution_dir, verbose_mode=args.verbose) elif system == "darwin": library_packager = MacLibraryPackager( build_dir=build_root, seed_executable_patterns=release_util.get_binary_path(), dest_dir=yb_distribution_dir, verbose_mode=args.verbose) else: raise RuntimeError("System {} not supported".format(system)) library_packager.package_binaries() release_util.update_manifest(yb_distribution_dir) logging.info("Generating release distribution") if os.path.exists(build_target) and os.listdir(build_target): raise RuntimeError( "Directory '{}' exists and is non-empty".format(build_target)) release_util.create_distribution(build_target) if args.build_archive: release_file = os.path.realpath(release_util.generate_release()) if args.destination: if not os.path.exists(args.destination): raise RuntimeError("Destination {} not a directory.".format( args.destination)) shutil.copy(release_file, args.destination) logging.info("Generated a package at '{}'".format(release_file)) if args.save_release_path_to_file: with open(args.save_release_path_to_file, 'w') as release_path_file: release_path_file.write(release_file) logging.info("Saved package path to '{}'".format( args.save_release_path_to_file))
def parse_args(self) -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--input-log-path', help='Input file log path. Must not be gzipped.', required=True) parser.add_argument('--verbose', help='Produce verbose output', action='store_true') parser.add_argument('--yb-src-root', help='YugabyteDB source tree root') parser.add_argument('--build-root', help='YugabyteDB build directory root') parser.add_argument( '--test-tmpdir', help='The temporary directory used to run the test') # This group of mutually exclusive arguments determines what output file to use. group = parser.add_mutually_exclusive_group(required=True) group.add_argument( '--replace-original', help= 'Replace the original file in case of success. In case of failure, append ' 'the error messages to the file.', action='store_true') group.add_argument('--output-log-path', help='Output file log path') group.add_argument( '--output-extension', help= 'The extension for the output file. Input file extension, if present, is removed ' 'and this extension is appended to obtain the output file name.') args = parser.parse_args() init_env(verbose=args.verbose) if args.output_extension: input_path, input_ext = os.path.splitext(args.input_log_path) new_ext = args.output_extension if not new_ext.startswith('.'): new_ext = '.' + new_ext args.output_log_path = input_path + new_ext logging.info("Automatically determined output log file path: %s", args.output_log_path) if args.replace_original: args.output_log_path = args.input_log_path else: if not args.output_log_path: raise ValueError( 'Output log path not specified. Please use --output-log-path, ' '--output-extension, or --replace-original.') if args.input_log_path == args.output_log_path: raise ValueError( "--replace-original not specified but input and output paths are the same: %s" % args.input_log_path) conf = LogRewriterConf(input_log_path=args.input_log_path, verbose=args.verbose, yb_src_root=args.yb_src_root, build_root=args.build_root, test_tmpdir=args.test_tmpdir, replace_original=args.replace_original, output_log_path=args.output_log_path) self.rewriter = LogRewriter(conf)