def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build', help='Build type (debug/release)', dest='build_type')
parser.add_argument('--build_root',
help='The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument('--build_args', default='',
help='Additional arguments to pass to the build script')
parser.add_argument('--build_archive', action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination', help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition', help='Which edition the code is built as.',
default=None,
choices=RELEASE_EDITION_ALLOWED_VALUES)
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build', help='Skip building the code', action='store_true')
parser.add_argument('--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir', action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument('--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
add_common_arguments(parser)
args = parser.parse_args()
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info("Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified.")
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError('--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
if build_root and not args.edition:
build_root_basename = os.path.basename(build_root)
if '-community-' in build_root_basename or build_root_basename.endswith('community'):
logging.info("Setting edition to Community based on build root")
args.edition = RELEASE_EDITION_COMMUNITY
elif '-enterprise-' in build_root_basename or build_root_basename.endswith('enterprise'):
logging.info("Setting edition to Enterprise based on build root")
args.edition = RELEASE_EDITION_ENTERPRISE
# We must have either not had a build_root specified, or not been able to deduce from the root.
if not args.edition:
ent_path = "{}/ent".format(YB_SRC_ROOT)
if os.path.isdir(ent_path):
logging.info(
"No --edition was specified, but found '{}', releasing Enterprise edition.".format(
ent_path))
args.edition = RELEASE_EDITION_ENTERPRISE
else:
logging.info(
"No --edition was specified and '{}' is not a valid dir, releasing Community "
"edition.".format(ent_path))
args.edition = RELEASE_EDITION_COMMUNITY
build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community"
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError(
("Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugaByte DB {} Edition: '{}' build".format(
build_edition.capitalize(), build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh",
"--write-build-descriptor", build_desc_path,
"--edition", build_edition,
build_type
]
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, edition, etc. based on that.
build_cmd_list += ["--build-root", build_root]
build_cmd_list += [
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
# We do not need java code built for a release package
"--skip-java"
]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
# TODO: run with shell=True and append build_args as is.
build_cmd_list += args.build_args.strip().split()
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
if not args.skip_build:
# TODO: figure out the dependency issues in our CMake build instead.
# TODO: move this into yb_build.sh itself.
for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
preliminary_step_cmd_list = [
arg for arg in build_cmd_list if arg != 'packaged_targets'
] + ['--target', preliminary_target, '--skip-java']
logging.info(
"Running a preliminary step to build target %s: %s",
preliminary_target,
" ".join(preliminary_step_cmd_list))
subprocess.check_call(preliminary_step_cmd_list)
subprocess.check_call(build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError("The build script failed to generate build descriptor file at '{}'".format(
build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
thirdparty_dir = build_desc["thirdparty_dir"]
thirdparty_dir_from_env = os.environ.get("YB_THIRDPARTY_DIR", thirdparty_dir)
if (thirdparty_dir != thirdparty_dir_from_env and
thirdparty_dir_from_env != os.path.join(YB_SRC_ROOT, 'thirdparty')):
raise RuntimeError(
"Mismatch between non-default valueo of env YB_THIRDPARTY_DIR: '{}' and build desc "
"thirdparty_dir: '{}'".format(thirdparty_dir_from_env, thirdparty_dir))
# Set the var for in-memory access to it across the other python files.
set_thirdparty_dir(thirdparty_dir)
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(
YB_SRC_ROOT, build_type, args.edition, build_target, args.force,
args.commit, build_root)
system = platform.system().lower()
library_packager_args = dict(
build_dir=build_root,
seed_executable_patterns=release_util.get_seed_executable_patterns(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose
)
if system == "linux":
library_packager = LibraryPackager(**library_packager_args)
elif system == "darwin":
library_packager = MacLibraryPackager(**library_packager_args)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError("Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file, 'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build',
help='Build type (debug/release)',
dest='build_type')
parser.add_argument(
'--build_root',
help=
'The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument(
'--build_args',
default='',
help='Additional arguments to pass to the build script')
parser.add_argument(
'--build_archive',
action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination',
help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build',
help='Skip building the code',
action='store_true')
parser.add_argument(
'--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir',
action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument(
'--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
parser.add_argument('--yw',
action='store_true',
help='Package YugaWare too.')
parser.add_argument(
'--no_reinitdb',
action='store_true',
help='Do not re-create the initial sys catalog snapshot. Useful when '
'debugging the release process.')
add_common_arguments(parser)
args = parser.parse_args()
# ---------------------------------------------------------------------------------------------
# Processing the arguments
# ---------------------------------------------------------------------------------------------
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info(
"Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified."
)
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError(
'--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build",
"yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError((
"Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugabyteDB {} build".format(build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh", "--write-build-descriptor", build_desc_path,
build_type
]
if args.force:
build_cmd_list.append("--force")
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, etc. based on that.
build_cmd_list += ["--build-root", build_root]
build_cmd_list += [
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
# We do not need java code built for a release package
"--skip-java"
]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
# TODO: run with shell=True and append build_args as is.
build_cmd_list += args.build_args.strip().split()
# ---------------------------------------------------------------------------------------------
# Perform the build
# ---------------------------------------------------------------------------------------------
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
if not args.skip_build:
# TODO: figure out the dependency issues in our CMake build instead.
# TODO: move this into yb_build.sh itself.
for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
preliminary_step_cmd_list = [
arg for arg in build_cmd_list if arg != 'packaged_targets'
] + ['--target', preliminary_target, '--skip-java']
logging.info("Running a preliminary step to build target %s: %s",
preliminary_target, preliminary_step_cmd_list)
subprocess.check_call(preliminary_step_cmd_list)
# We still need to call this even when --skip-build is specified to generate the "build
# descriptor" YAML file.
final_build_cmd_list = build_cmd_list + ([] if args.no_reinitdb else
['reinitdb'])
logging.info("Running final build step: %s", final_build_cmd_list)
subprocess.check_call(final_build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError(
"The build script failed to generate build descriptor file at '{}'"
.format(build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
# We are guaranteed to have a build_root by now.
library_packager_module.set_build_root(build_root)
thirdparty_dir = build_desc["thirdparty_dir"]
thirdparty_dir_from_env = os.environ.get("YB_THIRDPARTY_DIR",
thirdparty_dir)
if (thirdparty_dir != thirdparty_dir_from_env
and thirdparty_dir_from_env != os.path.join(
YB_SRC_ROOT, 'thirdparty')):
raise RuntimeError(
"Mismatch between non-default valueo of env YB_THIRDPARTY_DIR: '{}' and build desc "
"thirdparty_dir: '{}'".format(thirdparty_dir_from_env,
thirdparty_dir))
# Set the var for in-memory access to it across the other python files.
set_thirdparty_dir(thirdparty_dir)
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(YB_SRC_ROOT, build_type, build_target,
args.force, args.commit, build_root)
system = platform.system().lower()
library_packager_args = dict(
build_dir=build_root,
seed_executable_patterns=release_util.get_seed_executable_patterns(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
if system == "linux":
library_packager = LibraryPackager(**library_packager_args)
elif system == "darwin":
library_packager = MacLibraryPackager(**library_packager_args)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError(
"Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
# ---------------------------------------------------------------------------------------------
# Invoke YugaWare packaging
# ---------------------------------------------------------------------------------------------
if args.yw:
managed_dir = os.path.join(YB_SRC_ROOT, "managed")
yw_dir = os.path.join(build_target, "ui")
if not os.path.exists(yw_dir):
os.makedirs(yw_dir)
package_yw_cmd = [
os.path.join(managed_dir, "yb_release"), "--destination", yw_dir,
"--unarchived"
]
logging.info("Creating YugaWare package with command: {}".format(
package_yw_cmd))
try:
subprocess.check_output(package_yw_cmd, cwd=managed_dir)
except subprocess.CalledProcessError as e:
logging.error(
"Failed to build YugaWare package:\n{}\nOutput:\n{}".format(
traceback.format_exc(), e.output))
raise
except OSError as e:
logging.error("Failed to build YugaWare package: {}".format(e))
raise
# ---------------------------------------------------------------------------------------------
# Create a tar.gz (we almost always do this, can be skipped in debug mode)
# ---------------------------------------------------------------------------------------------
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(
args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file,
'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build',
help='Build type (debug/release)',
dest='build_type')
parser.add_argument(
'--build_root',
help=
'The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument(
'--build_args',
default='',
help='Additional arguments to pass to the build script')
parser.add_argument(
'--build_archive',
action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination',
help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition',
help='Which edition the code is built as.',
default=None,
choices=RELEASE_EDITION_ALLOWED_VALUES)
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build',
help='Skip building the code',
action='store_true')
parser.add_argument(
'--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir',
action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument(
'--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
add_common_arguments(parser)
args = parser.parse_args()
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info(
"Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified."
)
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError(
'--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
if build_root and not args.edition:
build_root_basename = os.path.basename(build_root)
if '-community-' in build_root_basename or build_root_basename.endswith(
'community'):
logging.info("Setting edition to Community based on build root")
args.edition = RELEASE_EDITION_COMMUNITY
if not args.edition:
# Here we are not detecting edition based on the existence of the enterprise source
# directory.
args.edition = RELEASE_EDITION_ENTERPRISE
build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community"
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build",
"yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError((
"Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugaByte DB {} Edition: '{}' build".format(
build_edition.capitalize(), build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh",
"--with-assembly",
"--write-build-descriptor",
build_desc_path,
"--edition",
build_edition,
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
build_type
]
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, edition, etc. based on that.
build_cmd_list += ["--build-root", build_root]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
build_cmd_list += args.build_args.strip().split()
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
subprocess.check_call(build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError(
"The build script failed to generate build descriptor file at '{}'"
.format(build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(YB_SRC_ROOT, build_type, args.edition,
build_target, args.force, args.commit)
release_util.rewrite_manifest(build_root)
system = platform.system().lower()
if system == "linux":
library_packager = LibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
elif system == "darwin":
library_packager = MacLibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError(
"Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(
args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file,
'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))