def main() -> None:
args = parse_args()
init_env(verbose=args.verbose)
if args.update:
updater = MetadataUpdater(
github_token_file_path=args.github_token_file,
tag_filter_regex_str=args.tag_filter_regex)
updater.update_archive_metadata_file()
return
metadata = load_metadata()
manual_metadata = load_manual_metadata()
if args.get_sha1:
print(metadata[SHA_FOR_LOCAL_CHECKOUT_KEY])
return
metadata['archives'].extend(manual_metadata['archives'])
if args.save_download_url_to_file:
if not args.compiler_type:
raise ValueError("Compiler type not specified")
url = get_download_url(metadata=metadata,
compiler_type=args.compiler_type,
os_type=args.os_type,
architecture=args.architecture)
if url is None:
raise RuntimeError("Could not determine download URL")
logging.info(f"Download URL for the third-party dependencies: {url}")
output_file_dir = os.path.dirname(
os.path.abspath(args.save_download_url_to_file))
os.makedirs(output_file_dir, exist_ok=True)
with open(args.save_download_url_to_file, 'w') as output_file:
output_file.write(url)
def main() -> None:
args = parse_args()
init_env(verbose=args.verbose)
if args.update:
updater = MetadataUpdater(
github_token_file_path=args.github_token_file,
tag_filter_regex_str=args.tag_filter_regex)
updater.update_archive_metadata_file()
return
metadata = load_metadata()
manual_metadata = load_manual_metadata()
if args.get_sha1:
print(metadata[SHA_FOR_LOCAL_CHECKOUT_KEY])
return
metadata_items = [
MetadataItem(item_json_data)
for item_json_data in metadata['archives'] + (manual_metadata['archives'] or [])
]
if args.list_compilers:
compiler_list = get_compilers(
metadata_items=metadata_items,
os_type=args.os_type,
architecture=args.architecture)
for compiler in compiler_list:
print(compiler)
return
if args.save_thirdparty_url_to_file or args.save_llvm_url_to_file:
if not args.compiler_type:
raise ValueError("Compiler type not specified")
thirdparty_release: Optional[MetadataItem] = get_third_party_release(
available_archives=metadata_items,
compiler_type=args.compiler_type,
os_type=args.os_type,
architecture=args.architecture,
is_linuxbrew=args.is_linuxbrew,
lto=args.lto)
if thirdparty_release is None:
raise RuntimeError("Could not determine third-party archive download URL")
thirdparty_url = thirdparty_release.url()
logging.info(f"Download URL for the third-party dependencies: {thirdparty_url}")
if args.save_thirdparty_url_to_file:
make_parent_dir(args.save_thirdparty_url_to_file)
write_file(thirdparty_url, args.save_thirdparty_url_to_file)
if (args.save_llvm_url_to_file and
thirdparty_release.compiler_type.startswith('clang') and
thirdparty_release.is_linuxbrew):
llvm_url = get_llvm_url(thirdparty_release.compiler_type)
if llvm_url is not None:
logging.info(f"Download URL for the LLVM toolchain: {llvm_url}")
make_parent_dir(args.save_llvm_url_to_file)
write_file(llvm_url, args.save_llvm_url_to_file)
else:
logging.info("Could not determine LLVM URL for compiler type %s" %
thirdparty_release.compiler_type)
else:
logging.info("Not a Linuxbrew URL, not saving LLVM URL to file")
def main() -> None:
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument('build_args', nargs=argparse.REMAINDER,
help='arguments for yb_build.sh')
remote.add_common_args(parser)
remote.handle_yb_build_cmd_line()
args = parser.parse_args()
init_env(verbose=args.verbose)
remote.load_profile(args, args.profile)
remote.apply_default_arg_values(args)
os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
remote.log_args(args)
escaped_remote_path = remote.sync_changes_with_args(args)
if args.skip_build:
sys.exit(0)
remote_args = []
if args.build_type:
remote_args.append(args.build_type)
if len(args.build_args) != 0 and args.build_args[0] == '--':
remote_args += args.build_args[1:]
else:
remote_args += args.build_args
if '--host-for-tests' not in remote_args and 'YB_HOST_FOR_RUNNING_TESTS' in os.environ:
remote_args = add_extra_yb_build_args(
remote_args,
['--host-for-tests', os.environ['YB_HOST_FOR_RUNNING_TESTS']])
remote.exec_command(
host=args.host,
escaped_remote_path=escaped_remote_path,
script_name='yb_build.sh',
script_args=remote_args,
should_quote_args=True,
extra_ssh_args=remote.process_extra_ssh_args(args.extra_ssh_args))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build',
help='Build type (debug/release)',
dest='build_type')
parser.add_argument(
'--build_root',
help=
'The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument(
'--build_args',
default='',
help='Additional arguments to pass to the build script')
parser.add_argument(
'--build_archive',
action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination',
help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build',
help='Skip building the code',
action='store_true')
parser.add_argument(
'--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir',
action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument(
'--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
parser.add_argument('--yw',
action='store_true',
help='Package YugaWare too.')
parser.add_argument(
'--no_reinitdb',
action='store_true',
help='Do not re-create the initial sys catalog snapshot. Useful when '
'debugging the release process.')
add_common_arguments(parser)
args = parser.parse_args()
# ---------------------------------------------------------------------------------------------
# Processing the arguments
# ---------------------------------------------------------------------------------------------
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info(
"Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified."
)
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError(
'--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build",
"yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError((
"Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugabyteDB {} build".format(build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh", "--write-build-descriptor", build_desc_path,
build_type
]
if args.force:
build_cmd_list.append("--force")
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, etc. based on that.
build_cmd_list += ["--build-root", build_root]
build_cmd_list += [
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
# We do not need java code built for a release package
"--skip-java"
]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
# TODO: run with shell=True and append build_args as is.
build_cmd_list += args.build_args.strip().split()
# ---------------------------------------------------------------------------------------------
# Perform the build
# ---------------------------------------------------------------------------------------------
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
if not args.skip_build:
# TODO: figure out the dependency issues in our CMake build instead.
# TODO: move this into yb_build.sh itself.
for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
preliminary_step_cmd_list = [
arg for arg in build_cmd_list if arg != 'packaged_targets'
] + ['--target', preliminary_target, '--skip-java']
logging.info("Running a preliminary step to build target %s: %s",
preliminary_target, preliminary_step_cmd_list)
subprocess.check_call(preliminary_step_cmd_list)
# We still need to call this even when --skip-build is specified to generate the "build
# descriptor" YAML file.
final_build_cmd_list = build_cmd_list + ([] if args.no_reinitdb else
['reinitdb'])
logging.info("Running final build step: %s", final_build_cmd_list)
subprocess.check_call(final_build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError(
"The build script failed to generate build descriptor file at '{}'"
.format(build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
# We are guaranteed to have a build_root by now.
library_packager_module.set_build_root(build_root)
thirdparty_dir = build_desc["thirdparty_dir"]
thirdparty_dir_from_env = os.environ.get("YB_THIRDPARTY_DIR",
thirdparty_dir)
if (thirdparty_dir != thirdparty_dir_from_env
and thirdparty_dir_from_env != os.path.join(
YB_SRC_ROOT, 'thirdparty')):
raise RuntimeError(
"Mismatch between non-default valueo of env YB_THIRDPARTY_DIR: '{}' and build desc "
"thirdparty_dir: '{}'".format(thirdparty_dir_from_env,
thirdparty_dir))
# Set the var for in-memory access to it across the other python files.
set_thirdparty_dir(thirdparty_dir)
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(YB_SRC_ROOT, build_type, build_target,
args.force, args.commit, build_root)
system = platform.system().lower()
library_packager_args = dict(
build_dir=build_root,
seed_executable_patterns=release_util.get_seed_executable_patterns(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
if system == "linux":
library_packager = LibraryPackager(**library_packager_args)
elif system == "darwin":
library_packager = MacLibraryPackager(**library_packager_args)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError(
"Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
# ---------------------------------------------------------------------------------------------
# Invoke YugaWare packaging
# ---------------------------------------------------------------------------------------------
if args.yw:
managed_dir = os.path.join(YB_SRC_ROOT, "managed")
yw_dir = os.path.join(build_target, "ui")
if not os.path.exists(yw_dir):
os.makedirs(yw_dir)
package_yw_cmd = [
os.path.join(managed_dir, "yb_release"), "--destination", yw_dir,
"--unarchived"
]
logging.info("Creating YugaWare package with command: {}".format(
package_yw_cmd))
try:
subprocess.check_output(package_yw_cmd, cwd=managed_dir)
except subprocess.CalledProcessError as e:
logging.error(
"Failed to build YugaWare package:\n{}\nOutput:\n{}".format(
traceback.format_exc(), e.output))
raise
except OSError as e:
logging.error("Failed to build YugaWare package: {}".format(e))
raise
# ---------------------------------------------------------------------------------------------
# Create a tar.gz (we almost always do this, can be skipped in debug mode)
# ---------------------------------------------------------------------------------------------
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(
args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file,
'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build', help='Build type (debug/release)', dest='build_type')
parser.add_argument('--build_root',
help='The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument('--build_args', default='',
help='Additional arguments to pass to the build script')
parser.add_argument('--build_archive', action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination', help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition', help='Which edition the code is built as.',
default=None,
choices=RELEASE_EDITION_ALLOWED_VALUES)
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build', help='Skip building the code', action='store_true')
parser.add_argument('--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir', action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument('--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
add_common_arguments(parser)
args = parser.parse_args()
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info("Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified.")
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError('--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
if build_root and not args.edition:
build_root_basename = os.path.basename(build_root)
if '-community-' in build_root_basename or build_root_basename.endswith('community'):
logging.info("Setting edition to Community based on build root")
args.edition = RELEASE_EDITION_COMMUNITY
elif '-enterprise-' in build_root_basename or build_root_basename.endswith('enterprise'):
logging.info("Setting edition to Enterprise based on build root")
args.edition = RELEASE_EDITION_ENTERPRISE
# We must have either not had a build_root specified, or not been able to deduce from the root.
if not args.edition:
ent_path = "{}/ent".format(YB_SRC_ROOT)
if os.path.isdir(ent_path):
logging.info(
"No --edition was specified, but found '{}', releasing Enterprise edition.".format(
ent_path))
args.edition = RELEASE_EDITION_ENTERPRISE
else:
logging.info(
"No --edition was specified and '{}' is not a valid dir, releasing Community "
"edition.".format(ent_path))
args.edition = RELEASE_EDITION_COMMUNITY
build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community"
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError(
("Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugaByte DB {} Edition: '{}' build".format(
build_edition.capitalize(), build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh",
"--write-build-descriptor", build_desc_path,
"--edition", build_edition,
build_type
]
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, edition, etc. based on that.
build_cmd_list += ["--build-root", build_root]
build_cmd_list += [
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
# We do not need java code built for a release package
"--skip-java"
]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
# TODO: run with shell=True and append build_args as is.
build_cmd_list += args.build_args.strip().split()
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
if not args.skip_build:
# TODO: figure out the dependency issues in our CMake build instead.
# TODO: move this into yb_build.sh itself.
for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
preliminary_step_cmd_list = [
arg for arg in build_cmd_list if arg != 'packaged_targets'
] + ['--target', preliminary_target, '--skip-java']
logging.info(
"Running a preliminary step to build target %s: %s",
preliminary_target,
" ".join(preliminary_step_cmd_list))
subprocess.check_call(preliminary_step_cmd_list)
subprocess.check_call(build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError("The build script failed to generate build descriptor file at '{}'".format(
build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
thirdparty_dir = build_desc["thirdparty_dir"]
thirdparty_dir_from_env = os.environ.get("YB_THIRDPARTY_DIR", thirdparty_dir)
if (thirdparty_dir != thirdparty_dir_from_env and
thirdparty_dir_from_env != os.path.join(YB_SRC_ROOT, 'thirdparty')):
raise RuntimeError(
"Mismatch between non-default valueo of env YB_THIRDPARTY_DIR: '{}' and build desc "
"thirdparty_dir: '{}'".format(thirdparty_dir_from_env, thirdparty_dir))
# Set the var for in-memory access to it across the other python files.
set_thirdparty_dir(thirdparty_dir)
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(
YB_SRC_ROOT, build_type, args.edition, build_target, args.force,
args.commit, build_root)
system = platform.system().lower()
library_packager_args = dict(
build_dir=build_root,
seed_executable_patterns=release_util.get_seed_executable_patterns(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose
)
if system == "linux":
library_packager = LibraryPackager(**library_packager_args)
elif system == "darwin":
library_packager = MacLibraryPackager(**library_packager_args)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError("Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file, 'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build', help='Build type (debug/release)', dest='build_type')
parser.add_argument('--build_root',
help='The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument('--build_args', default='',
help='Additional arguments to pass to the build script')
parser.add_argument('--build_archive', action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination', help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition', help='Which edition the code is built as.',
default=None,
choices=RELEASE_EDITION_ALLOWED_VALUES)
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build', help='Skip building the code', action='store_true')
parser.add_argument('--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir', action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument('--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
parser.add_argument('--java_only', action="store_true",
help="Only build the java part of the code. Does not generate an archive!")
add_common_arguments(parser)
args = parser.parse_args()
init_env(args.verbose)
if not args.build_target and not args.build_archive and not args.java_only:
logging.info("Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified.")
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError('--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
if build_root and not args.edition:
build_root_basename = os.path.basename(build_root)
if '-community-' in build_root_basename or build_root_basename.endswith('community'):
logging.info("Setting edition to Community based on build root")
args.edition = RELEASE_EDITION_COMMUNITY
if not args.edition:
# Here we are not detecting edition based on the existence of the enterprise source
# directory.
args.edition = RELEASE_EDITION_ENTERPRISE
build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community"
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError(
("Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugaByte DB {} Edition: '{}' build{}".format(
build_edition.capitalize(), build_type, " Java only" if args.java_only else ""))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh",
"--with-assembly",
"--write-build-descriptor", build_desc_path,
"--edition", build_edition,
build_type
]
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, edition, etc. based on that.
build_cmd_list += ["--build-root", build_root]
if args.java_only:
build_cmd_list += ["--java-only"]
else:
build_cmd_list += [
# This will build the exact set of targets that are needed for the release.
"packaged_targets"
]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
# TODO: run with shell=True and append build_args as is.
build_cmd_list += args.build_args.strip().split()
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
if not args.java_only and not args.skip_build:
# TODO: figure out the dependency issues in our CMake build instead.
# TODO: move this into yb_build.sh itself.
for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
preliminary_step_cmd_list = [
arg for arg in build_cmd_list if arg != 'packaged_targets'
] + ['--target', preliminary_target, '--skip-java']
logging.info(
"Running a preliminary step to build target %s: %s",
preliminary_target,
" ".join(preliminary_step_cmd_list))
subprocess.check_call(preliminary_step_cmd_list)
subprocess.check_call(build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError("The build script failed to generate build descriptor file at '{}'".format(
build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
thirdparty_dir = build_desc["thirdparty_dir"]
if thirdparty_dir != os.environ.get("YB_THIRDPARTY_DIR", thirdparty_dir):
raise RuntimeError(
"Mismatch between env YB_THIRDPARTY_DIR: '{}' and build desc thirdparty_dir: '{}'"
.format(os.environ["YB_THIRDPARTY_DIR"], thirdparty_dir))
# Set the var for in-memory access to it across the other python files.
set_thirdparty_dir(thirdparty_dir)
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(YB_SRC_ROOT, build_type, args.edition, build_target, args.force,
args.commit)
release_util.rewrite_manifest(build_root)
if not args.java_only:
system = platform.system().lower()
if system == "linux":
library_packager = LibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
elif system == "darwin":
library_packager = MacLibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
if args.java_only:
logging.info("Directory '{}' exists and is not empty, but we are using --java_only")
else:
raise RuntimeError("Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target, "java" if args.java_only else None)
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file, 'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def main():
parser = argparse.ArgumentParser(
description='Run FOSSA analysis (open source license compliance).')
parser.add_argument('--verbose', action='store_true', help='Enable verbose output')
parser.add_argument(
'fossa_cli_args',
nargs='*',
help='These arguments are passed directly to fossa-cli')
args = parser.parse_args()
init_env(args.verbose)
# TODO: We may also want to try using the v2 option --unpack-archives
# Though that may be going to deeper level than we want.
fossa_cmd_line = ['fossa', 'analyze']
fossa_cmd_line.extend(args.fossa_cli_args)
should_upload = not any(
arg in args.fossa_cli_args for arg in ('--show-output', '--output', '-o'))
if should_upload and not os.getenv('FOSSA_API_KEY'):
# --output is used for local analysis only, without uploading the results. In all other
# cases we would like .
raise RuntimeError('FOSSA_API_KEY must be specified in order to upload analysis results.')
logging.info(
f"FOSSA CLI command line: {shlex_join(fossa_cmd_line)}")
fossa_version_str = subprocess.check_output(['fossa', '--version']).decode('utf-8')
fossa_version_match = FOSSA_VERSION_RE.match(fossa_version_str)
if not fossa_version_match:
raise RuntimeError(f"Cannot parse fossa-cli version: {fossa_version_str}")
fossa_version = fossa_version_match.group(1)
if version.parse(fossa_version) < version.parse(MIN_FOSSA_CLI_VERSION):
raise RuntimeError(
f"fossa version too old: {fossa_version} "
f"(expected {MIN_FOSSA_CLI_VERSION} or later)")
download_cache_path = get_download_cache_dir()
logging.info(f"Using the download cache directory {download_cache_path}")
download_config = DownloadConfig(
verbose=args.verbose,
cache_dir_path=download_cache_path
)
downloader = Downloader(download_config)
fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa-local.yml')
fossa_yml_data = load_yaml_file(fossa_yml_path)
modules = fossa_yml_data['analyze']['modules']
# fossa v2.6.1 does not pick up project name from config file version 2 format.
# TODO: update to config file version 3
fossa_cmd_line.extend(["--project", fossa_yml_data['cli']['project']])
thirdparty_dir = get_thirdparty_dir()
fossa_modules_path = os.path.join(thirdparty_dir, 'fossa_modules.yml')
seen_urls = set()
start_time_sec = time.time()
if os.path.exists(fossa_modules_path):
thirdparty_fossa_modules_data = load_yaml_file(fossa_modules_path)
for thirdparty_module_data in thirdparty_fossa_modules_data:
fossa_module_data = thirdparty_module_data['fossa_module']
module_name = fossa_module_data['name']
if not should_include_fossa_module(module_name):
continue
fossa_module_yb_metadata = thirdparty_module_data['yb_metadata']
expected_sha256 = fossa_module_yb_metadata['sha256sum']
url = fossa_module_yb_metadata['url']
if url in seen_urls:
# Due to a bug in some versions of yugabyte-db-thirdparty scripts, as of 04/20/2021
# we may include the same dependency twice in the fossa_modules.yml file. We just
# skip the duplicates here.
continue
seen_urls.add(url)
logging.info(f"Adding module from {url}")
downloaded_path = downloader.download_url(
url,
download_parent_dir_path=None, # Download to cache directly.
verify_checksum=True,
expected_sha256=expected_sha256
)
fossa_module_data['target'] = downloaded_path
modules.append(fossa_module_data)
# TODO: Once we move to v2 fossa, we may want to use fossa-dep.yml file instead of
# re-writing the main file.
effective_fossa_yml_path = os.path.join(YB_SRC_ROOT, '.fossa.yml')
write_yaml_file(fossa_yml_data, effective_fossa_yml_path)
logging.info(f"Wrote the expanded FOSSA file to {effective_fossa_yml_path}")
else:
logging.warning(
f"File {fossa_modules_path} does not exist. Some C/C++ dependencies will be missing "
f"from FOSSA analysis.")
effective_fossa_yml_path = fossa_yml_path
elapsed_time_sec = time.time() - start_time_sec
logging.info("Generated the effective FOSSA configuration file in %.1f sec", elapsed_time_sec)
logging.info(f"Running command: {shlex_join(fossa_cmd_line)})")
subprocess.check_call(fossa_cmd_line)
def main():
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument(
'--host',
type=str,
default=None,
help=(
'Host to build on. Can also be specified using the {} environment '
+ 'variable.').format(remote.REMOTE_BUILD_HOST_ENV_VAR))
home = os.path.expanduser('~')
cwd = os.getcwd()
default_path = '~/{0}'.format(
cwd[len(home) + 1:] if cwd.startswith(home + '/') else 'code/yugabyte')
# Note: don't specify default arguments here, because they may come from the "profile".
parser.add_argument('--remote-path', type=str, help='path used for build')
parser.add_argument('--branch',
type=str,
default=None,
help='base branch for build')
parser.add_argument('--upstream',
type=str,
default=None,
help='base upstream for remote host to fetch')
parser.add_argument('--build-type',
type=str,
default=None,
help='build type')
parser.add_argument('--skip-build',
action='store_true',
help='skip build, only sync files')
parser.add_argument('--wait-for-ssh',
action='store_true',
help='Wait for the remote server to be ssh-able')
parser.add_argument('--profile',
help='Use a "profile" specified in the {} file'.format(
remote.CONFIG_FILE_PATH))
parser.add_argument('--verbose',
action='store_true',
help='Verbose output')
parser.add_argument('build_args',
nargs=argparse.REMAINDER,
help='arguments for yb_build.sh')
if len(sys.argv) >= 2 and sys.argv[1] in ['ybd', 'yb_build.sh']:
# Allow the first argument to be 'ybd' so we can copy and paste a ybd command line directly
# after remote_build.py.
sys.argv[1:2] = ['--']
args = parser.parse_args()
init_env(verbose=args.verbose)
remote.load_profile(args, args.profile)
# ---------------------------------------------------------------------------------------------
# Default arguments go here.
args.host = remote.apply_default_host_value(args.host)
if args.branch is None:
args.branch = remote.DEFAULT_BASE_BRANCH
if args.remote_path is None:
args.remote_path = default_path
if args.upstream is None:
args.upstream = remote.DEFAULT_UPSTREAM
# End of default arguments.
# ---------------------------------------------------------------------------------------------
os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
print("Host: {0}, build type: {1}, remote path: {2}".format(
args.host, args.build_type or 'N/A', args.remote_path))
print("Arguments to remote build: {}".format(args.build_args))
escaped_remote_path = \
remote.sync_changes(args.host, args.branch, args.remote_path, args.wait_for_ssh,
args.upstream)
if args.skip_build:
sys.exit(0)
remote_args = []
if args.build_type:
remote_args.append(args.build_type)
if len(args.build_args) != 0 and args.build_args[0] == '--':
remote_args += args.build_args[1:]
else:
remote_args += args.build_args
if '--host-for-tests' not in remote_args and 'YB_HOST_FOR_RUNNING_TESTS' in os.environ:
remote_args = add_extra_ybd_args(
remote_args,
['--host-for-tests', os.environ['YB_HOST_FOR_RUNNING_TESTS']])
remote.exec_command(args.host,
escaped_remote_path,
'yb_build.sh',
remote_args,
do_quote_args=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build',
help='Build type (debug/release)',
dest='build_type')
parser.add_argument(
'--build_root',
help=
'The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument(
'--build_args',
default='',
help='Additional arguments to pass to the build script')
parser.add_argument(
'--build_archive',
action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination',
help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition',
help='Which edition the code is built as.',
default=None,
choices=RELEASE_EDITION_ALLOWED_VALUES)
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build',
help='Skip building the code',
action='store_true')
parser.add_argument(
'--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir',
action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument(
'--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
add_common_arguments(parser)
args = parser.parse_args()
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info(
"Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified."
)
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError(
'--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
if build_root and not args.edition:
build_root_basename = os.path.basename(build_root)
if '-community-' in build_root_basename or build_root_basename.endswith(
'community'):
logging.info("Setting edition to Community based on build root")
args.edition = RELEASE_EDITION_COMMUNITY
if not args.edition:
# Here we are not detecting edition based on the existence of the enterprise source
# directory.
args.edition = RELEASE_EDITION_ENTERPRISE
build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community"
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build",
"yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError((
"Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugaByte DB {} Edition: '{}' build".format(
build_edition.capitalize(), build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh",
"--with-assembly",
"--write-build-descriptor",
build_desc_path,
"--edition",
build_edition,
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
build_type
]
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, edition, etc. based on that.
build_cmd_list += ["--build-root", build_root]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
build_cmd_list += args.build_args.strip().split()
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
subprocess.check_call(build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError(
"The build script failed to generate build descriptor file at '{}'"
.format(build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(YB_SRC_ROOT, build_type, args.edition,
build_target, args.force, args.commit)
release_util.rewrite_manifest(build_root)
system = platform.system().lower()
if system == "linux":
library_packager = LibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
elif system == "darwin":
library_packager = MacLibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError(
"Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(
args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file,
'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def parse_args(self) -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--input-log-path',
help='Input file log path. Must not be gzipped.',
required=True)
parser.add_argument('--verbose',
help='Produce verbose output',
action='store_true')
parser.add_argument('--yb-src-root',
help='YugabyteDB source tree root')
parser.add_argument('--build-root',
help='YugabyteDB build directory root')
parser.add_argument(
'--test-tmpdir',
help='The temporary directory used to run the test')
# This group of mutually exclusive arguments determines what output file to use.
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'--replace-original',
help=
'Replace the original file in case of success. In case of failure, append '
'the error messages to the file.',
action='store_true')
group.add_argument('--output-log-path', help='Output file log path')
group.add_argument(
'--output-extension',
help=
'The extension for the output file. Input file extension, if present, is removed '
'and this extension is appended to obtain the output file name.')
args = parser.parse_args()
init_env(verbose=args.verbose)
if args.output_extension:
input_path, input_ext = os.path.splitext(args.input_log_path)
new_ext = args.output_extension
if not new_ext.startswith('.'):
new_ext = '.' + new_ext
args.output_log_path = input_path + new_ext
logging.info("Automatically determined output log file path: %s",
args.output_log_path)
if args.replace_original:
args.output_log_path = args.input_log_path
else:
if not args.output_log_path:
raise ValueError(
'Output log path not specified. Please use --output-log-path, '
'--output-extension, or --replace-original.')
if args.input_log_path == args.output_log_path:
raise ValueError(
"--replace-original not specified but input and output paths are the same: %s"
% args.input_log_path)
conf = LogRewriterConf(input_log_path=args.input_log_path,
verbose=args.verbose,
yb_src_root=args.yb_src_root,
build_root=args.build_root,
test_tmpdir=args.test_tmpdir,
replace_original=args.replace_original,
output_log_path=args.output_log_path)
self.rewriter = LogRewriter(conf)
