def parse_args(self):
parser = argparse.ArgumentParser(
description=
'A tool for building the PostgreSQL code subtree in YugaByte DB codebase'
)
parser.add_argument(
'--build_root',
default=os.environ.get('BUILD_ROOT'),
help='YugaByte build root directory. The PostgreSQL build/install '
'directories will be created under here.')
parser.add_argument('--run_tests',
action='store_true',
help='Run PostgreSQL tests after building it.')
parser.add_argument(
'--clean',
action='store_true',
help='Clean PostgreSQL build and installation directories.')
parser.add_argument('--cflags', help='C compiler flags')
parser.add_argument('--cxxflags', help='C++ compiler flags')
parser.add_argument('--ldflags', help='Linker flags for all binaries')
parser.add_argument('--ldflags_ex',
help='Linker flags for executables')
parser.add_argument('--compiler_type',
help='Compiler type, e.g. gcc or clang')
self.args = parser.parse_args()
if not self.args.build_root:
raise RuntimeError("Neither BUILD_ROOT or --build-root specified")
self.build_root = os.path.abspath(self.args.build_root)
self.build_root_realpath = os.path.realpath(self.build_root)
self.postgres_install_dir_include_realpath = \
os.path.join(self.build_root_realpath, 'postgres', 'include')
self.build_type = get_build_type_from_build_root(self.build_root)
self.pg_build_root = os.path.join(self.build_root, 'postgres_build')
self.build_stamp_path = os.path.join(self.pg_build_root, 'build_stamp')
self.pg_prefix = os.path.join(self.build_root, 'postgres')
self.build_type = get_build_type_from_build_root(self.build_root)
self.postgres_src_dir = os.path.join(YB_SRC_ROOT, 'src', 'postgres')
self.compiler_type = self.args.compiler_type or os.getenv(
'YB_COMPILER_TYPE')
if not self.compiler_type:
raise RuntimeError(
"Compiler type not specified using either --compiler_type or YB_COMPILER_TYPE"
)
self.export_compile_commands = os.environ.get(
'YB_EXPORT_COMPILE_COMMANDS') == '1'
# This allows to skip the time-consuming compile commands file generation if it already
# exists during debugging of this script.
self.export_compile_commands_lazily = \
os.environ.get('YB_EXPORT_COMPILE_COMMANDS_LAZILY') == '1'
def parse_args(self):
parser = argparse.ArgumentParser(
description='A tool for building the PostgreSQL code subtree in YugaByte DB codebase')
parser.add_argument('--build_root',
default=os.environ.get('BUILD_ROOT'),
help='YugaByte build root directory. The PostgreSQL build/install '
'directories will be created under here.')
parser.add_argument('--run_tests',
action='store_true',
help='Run PostgreSQL tests after building it.')
parser.add_argument('--clean',
action='store_true',
help='Clean PostgreSQL build and installation directories.')
parser.add_argument('--cflags', help='C compiler flags')
parser.add_argument('--cxxflags', help='C++ compiler flags')
parser.add_argument('--ldflags', help='Linker flags for all binaries')
parser.add_argument('--ldflags_ex', help='Linker flags for executables')
parser.add_argument('--compiler_type', help='Compiler type, e.g. gcc or clang')
parser.add_argument('--openssl_include_dir', help='OpenSSL include dir')
parser.add_argument('--openssl_lib_dir', help='OpenSSL lib dir')
parser.add_argument(
'--step', choices=['configure', 'make'],
help='Run a specific step of the build process (configure or make)')
self.args = parser.parse_args()
if not self.args.build_root:
raise RuntimeError("Neither BUILD_ROOT or --build-root specified")
self.build_root = os.path.abspath(self.args.build_root)
self.build_root_realpath = os.path.realpath(self.build_root)
self.postgres_install_dir_include_realpath = \
os.path.join(self.build_root_realpath, 'postgres', 'include')
self.build_type = get_build_type_from_build_root(self.build_root)
self.pg_build_root = os.path.join(self.build_root, 'postgres_build')
self.build_stamp_path = os.path.join(self.pg_build_root, 'build_stamp')
self.pg_prefix = os.path.join(self.build_root, 'postgres')
self.build_type = get_build_type_from_build_root(self.build_root)
self.postgres_src_dir = os.path.join(YB_SRC_ROOT, 'src', 'postgres')
self.compiler_type = self.args.compiler_type or os.getenv('YB_COMPILER_TYPE')
self.openssl_include_dir = self.args.openssl_include_dir
self.openssl_lib_dir = self.args.openssl_lib_dir
if not self.compiler_type:
raise RuntimeError(
"Compiler type not specified using either --compiler_type or YB_COMPILER_TYPE")
self.export_compile_commands = os.environ.get('YB_EXPORT_COMPILE_COMMANDS') == '1'
self.should_configure = self.args.step is None or self.args.step == 'configure'
self.should_make = self.args.step is None or self.args.step == 'make'
def __init__(self, args):
self.args = args
self.verbose = args.verbose
self.build_root = os.path.realpath(args.build_root)
self.is_ninja = is_ninja_build_root(self.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.yb_src_root = get_yb_src_root_from_build_root(self.build_root,
must_succeed=True)
self.src_dir_path = os.path.join(self.yb_src_root, 'src')
self.ent_src_dir_path = os.path.join(self.yb_src_root, 'ent', 'src')
self.rel_path_base_dirs = set(
[self.build_root,
os.path.join(self.src_dir_path, 'yb')])
self.incomplete_build = args.incomplete_build
self.file_regex = args.file_regex
if not self.file_regex and args.file_name_glob:
self.file_regex = fnmatch.translate('*/' + args.file_name_glob)
for dir_path in [self.src_dir_path, self.ent_src_dir_path]:
if not os.path.isdir(dir_path):
raise RuntimeError(
"Directory does not exist, or is not a directory: %s" %
dir_path)
def __init__(self, args):
self.args = args
self.verbose = args.verbose
self.build_root = os.path.realpath(args.build_root)
# We can't use Ninja build files for dependency tracking. We will need to also create a
# build directory with Make-compatible files. In practice, even this turns out not to be
# sufficient, as we still need depend.make files that only get generated when we perform
# the Make-based build, so in practice we just switch to using Make for Phabricator builds.
# However, keeping this logic as we might find a way to use Ninja builds for dependency
# tracking in the future.
self.build_root_make = convert_to_non_ninja_build_root(self.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.yb_src_root = get_yb_src_root_from_build_root(self.build_root, must_succeed=True)
self.src_dir_path = os.path.join(self.yb_src_root, 'src')
self.ent_src_dir_path = os.path.join(self.yb_src_root, 'ent', 'src')
self.rel_path_base_dirs = set([self.build_root, os.path.join(self.src_dir_path, 'yb')])
self.incomplete_build = args.incomplete_build
self.file_regex = args.file_regex
if not self.file_regex and args.file_name_glob:
self.file_regex = fnmatch.translate('*/' + args.file_name_glob)
for dir_path in [self.src_dir_path, self.ent_src_dir_path]:
if not os.path.isdir(dir_path):
raise RuntimeError("Directory does not exist, or is not a directory: %s" % dir_path)
def validate_and_process_args(self):
if not self.args.build_root:
raise RuntimeError("Neither BUILD_ROOT or --build-root specified")
self.build_root = os.path.abspath(self.args.build_root)
self.build_root_realpath = os.path.realpath(self.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.pg_build_root = os.path.join(self.build_root, 'postgres_build')
self.build_stamp_path = os.path.join(self.pg_build_root, 'build_stamp')
self.pg_prefix = os.path.join(self.build_root, 'postgres')
self.postgres_src_dir = os.path.join(YB_SRC_ROOT, 'src', 'postgres')
self.pg_config_root = os.path.join(self.build_root, 'postgres', 'bin',
'pg_config')
self.compiler_type = self.args.compiler_type
self.openssl_include_dir = self.args.openssl_include_dir
self.openssl_lib_dir = self.args.openssl_lib_dir
if not self.compiler_type:
raise RuntimeError(
"Compiler type not specified using either --compiler_type or YB_COMPILER_TYPE"
)
self.export_compile_commands = os.environ.get(
'YB_EXPORT_COMPILE_COMMANDS') == '1'
self.should_configure = self.args.step is None or self.args.step == 'configure'
self.should_make = self.args.step is None or self.args.step == 'make'
self.thirdparty_dir = self.args.thirdparty_dir
self.original_path = os.getenv('PATH').split(':')
def __init__(self, args):
self.args = args
self.verbose = args.verbose
self.build_root = os.path.realpath(args.build_root)
# We can't use Ninja build files for dependency tracking. We will need to also create a
# build directory with Make-compatible files. In practice, even this turns out not to be
# sufficient, as we still need depend.make files that only get generated when we perform
# the Make-based build, so in practice we just switch to using Make for Phabricator builds.
# However, keeping this logic as we might find a way to use Ninja builds for dependency
# tracking in the future.
self.build_root_make = convert_to_non_ninja_build_root(self.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.yb_src_root = get_yb_src_root_from_build_root(self.build_root,
must_succeed=True)
self.src_dir_path = os.path.join(self.yb_src_root, 'src')
self.ent_src_dir_path = os.path.join(self.yb_src_root, 'ent', 'src')
self.rel_path_base_dirs = set(
[self.build_root,
os.path.join(self.src_dir_path, 'yb')])
self.incomplete_build = args.incomplete_build
self.file_regex = args.file_regex
if not self.file_regex and args.file_name_glob:
self.file_regex = fnmatch.translate('*/' + args.file_name_glob)
for dir_path in [self.src_dir_path, self.ent_src_dir_path]:
if not os.path.isdir(dir_path):
raise RuntimeError(
"Directory does not exist, or is not a directory: %s" %
dir_path)
def __init__(
self,
verbose: bool,
build_root: str,
incomplete_build: bool,
file_regex: Optional[str],
file_name_glob: Optional[str],
build_args: Optional[str]) -> None:
self.verbose = verbose
self.build_root = os.path.realpath(build_root)
self.is_ninja = is_ninja_build_root(self.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.yb_src_root = ensure_yb_src_root_from_build_root(self.build_root)
self.src_dir_path = os.path.join(self.yb_src_root, 'src')
self.ent_src_dir_path = os.path.join(self.yb_src_root, 'ent', 'src')
self.rel_path_base_dirs = {self.build_root, os.path.join(self.src_dir_path, 'yb')}
self.incomplete_build = incomplete_build
self.file_regex = None
if file_regex:
self.file_regex = file_regex
elif file_name_glob:
self.file_regex = fnmatch.translate('*/' + file_name_glob)
self.src_dir_paths = [self.src_dir_path, self.ent_src_dir_path]
for dir_path in self.src_dir_paths:
if not os.path.isdir(dir_path):
raise RuntimeError("Directory does not exist, or is not a directory: %s" % dir_path)
self.build_args = build_args
def parse_args(self):
parser = argparse.ArgumentParser(
description=
'A tool for building the PostgreSQL code subtree in YugaByte DB codebase'
)
parser.add_argument(
'--build_root',
default=os.environ.get('BUILD_ROOT'),
help='YugaByte build root directory. The PostgreSQL build/install '
'directories will be created under here.')
parser.add_argument('--run_tests',
action='store_true',
help='Run PostgreSQL tests after building it.')
parser.add_argument(
'--clean',
action='store_true',
help='Clean PostgreSQL build and installation directories.')
parser.add_argument('--cflags', help='C compiler flags')
parser.add_argument('--cxxflags', help='C++ compiler flags')
parser.add_argument('--ldflags', help='Linker flags for all binaries')
parser.add_argument('--ldflags_ex',
help='Linker flags for executables')
parser.add_argument('--compiler_type',
help='Compiler type, e.g. gcc or clang')
self.args = parser.parse_args()
if not self.args.build_root:
raise RuntimeError("Neither BUILD_ROOT or --build-root specified")
self.build_root = os.path.abspath(self.args.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.pg_build_root = os.path.join(self.build_root, 'postgres_build')
self.pg_prefix = os.path.join(self.build_root, 'postgres')
self.build_type = get_build_type_from_build_root(self.build_root)
self.postgres_src_dir = os.path.join(YB_SRC_ROOT, 'src', 'postgres')
self.compiler_type = self.args.compiler_type or os.getenv(
'YB_COMPILER_TYPE')
if not self.compiler_type:
raise RuntimeError(
"Compiler type not specified using either --compiler_type or YB_COMPILER_TYPE"
)
def set_global_conf_from_args(args: argparse.Namespace) -> GlobalTestConfig:
build_root = os.path.realpath(args.build_root)
# This module is expected to be under python/yb.
yb_src_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
# Ensure that build_root is consistent with yb_src_root above.
yb_src_root_from_build_root = os.path.dirname(os.path.dirname(build_root))
build_type = get_build_type_from_build_root(build_root)
archive_for_workers = None
if args.send_archive_to_workers:
archive_for_workers = os.path.abspath(
os.path.join(build_root,
'archive_for_tests_on_spark.tar.gz.spark-no-extract'))
assert yb_src_root == yb_src_root_from_build_root, \
("An inconstency between YB_SRC_ROOT derived from module location ({}) vs. the one derived "
"from BUILD_ROOT ({})").format(yb_src_root, yb_src_root_from_build_root)
rel_build_root = os.path.relpath(os.path.abspath(build_root),
os.path.abspath(yb_src_root))
if len(rel_build_root.split('/')) != 2:
raise ValueError(
"Unexpected number of components in the relative path of build root to "
"source root: %s. build_root=%s, yb_src_root=%s" %
(rel_build_root, build_root, yb_src_root))
compiler_type = get_compiler_type_from_build_root(build_root)
compiler_type_from_env = os.environ.get('YB_COMPILER_TYPE')
if compiler_type_from_env is not None and compiler_type_from_env != compiler_type:
raise ValueError(
"Build root '%s' implies compiler type '%s' but YB_COMPILER_TYPE is '%s'"
% (build_root, compiler_type, compiler_type_from_env))
from yb import common_util
os.environ['YB_COMPILER_TYPE'] = compiler_type
global global_conf
global_conf = GlobalTestConfig(
build_root=build_root,
build_type=build_type,
yb_src_root=yb_src_root,
archive_for_workers=archive_for_workers,
rel_build_root=rel_build_root,
compiler_type=compiler_type,
# The archive might not even exist yet.
archive_sha256sum=None)
return global_conf
def parse_args(self):
parser = argparse.ArgumentParser(
description='A tool for building the PostgreSQL code subtree in YugaByte DB codebase')
parser.add_argument('--build_root',
default=os.environ.get('BUILD_ROOT'),
help='YugaByte build root directory. The PostgreSQL build/install '
'directories will be created under here.')
parser.add_argument('--run_tests',
action='store_true',
help='Run PostgreSQL tests after building it.')
parser.add_argument('--clean',
action='store_true',
help='Clean PostgreSQL build and installation directories.')
parser.add_argument('--cflags', help='C compiler flags')
parser.add_argument('--cxxflags', help='C++ compiler flags')
parser.add_argument('--ldflags', help='Linker flags for all binaries')
parser.add_argument('--ldflags_ex', help='Linker flags for executables')
parser.add_argument('--compiler_type', help='Compiler type, e.g. gcc or clang')
self.args = parser.parse_args()
if not self.args.build_root:
raise RuntimeError("Neither BUILD_ROOT or --build-root specified")
self.build_root = os.path.abspath(self.args.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.pg_build_root = os.path.join(self.build_root, 'postgres_build')
self.pg_prefix = os.path.join(self.build_root, 'postgres')
self.build_type = get_build_type_from_build_root(self.build_root)
self.postgres_src_dir = os.path.join(YB_SRC_ROOT, 'src', 'postgres')
self.compiler_type = self.args.compiler_type or os.getenv('YB_COMPILER_TYPE')
if not self.compiler_type:
raise RuntimeError(
"Compiler type not specified using either --compiler_type or YB_COMPILER_TYPE")
def __init__(self, args):
self.args = args
self.verbose = args.verbose
self.build_root = os.path.abspath(args.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.yb_src_root = os.path.dirname(os.path.dirname(self.build_root))
self.src_dir_path = os.path.join(self.yb_src_root, 'src')
self.ent_src_dir_path = os.path.join(self.yb_src_root, 'ent', 'src')
self.rel_path_base_dirs = set([self.build_root, os.path.join(self.src_dir_path, 'yb')])
self.incomplete_build = args.incomplete_build
self.file_regex = args.file_regex
if not self.file_regex and args.file_name_glob:
self.file_regex = fnmatch.translate('*/' + args.file_name_glob)
assert os.path.exists(self.src_dir_path)
assert os.path.exists(self.ent_src_dir_path)
def set_global_conf_from_args(args):
build_root = os.path.realpath(args.build_root)
# This module is expected to be under python/yb.
yb_src_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
# Ensure that build_root is consistent with yb_src_root above.
yb_src_root_from_build_root = os.path.dirname(os.path.dirname(build_root))
build_type = get_build_type_from_build_root(build_root)
assert yb_src_root == yb_src_root_from_build_root, \
("An inconstency between YB_SRC_ROOT derived from module location ({}) vs. the one derived "
"from BUILD_ROOT ({})").format(yb_src_root, yb_src_root_from_build_root)
global global_conf
global_conf = GlobalTestConfig(build_root=build_root,
build_type=build_type,
yb_src_root=yb_src_root)
return global_conf
def validate_and_process_args(self) -> None:
if not self.args.build_root:
raise RuntimeError("Neither BUILD_ROOT or --build-root specified")
self.build_root = os.path.abspath(self.args.build_root)
self.build_root_realpath = os.path.realpath(self.build_root)
self.build_type = get_build_type_from_build_root(self.build_root)
self.pg_build_root = os.path.join(self.build_root, 'postgres_build')
self.build_stamp_path = os.path.join(self.pg_build_root, 'build_stamp')
self.pg_prefix = os.path.join(self.build_root, 'postgres')
self.postgres_src_dir = os.path.join(YB_SRC_ROOT, 'src', 'postgres')
self.pg_config_path = os.path.join(self.build_root, 'postgres', 'bin',
'pg_config')
self.compiler_type = self.args.compiler_type
self.openssl_include_dir = self.args.openssl_include_dir
self.openssl_lib_dir = self.args.openssl_lib_dir
if not self.compiler_type:
raise RuntimeError(
"Compiler type not specified using either --compiler_type or YB_COMPILER_TYPE"
)
self.export_compile_commands = os.environ.get(
'YB_EXPORT_COMPILE_COMMANDS') == '1'
self.skip_pg_compile_commands = os.environ.get(
'YB_SKIP_PG_COMPILE_COMMANDS') == '1'
self.should_configure = self.args.step is None or self.args.step == 'configure'
self.should_make = self.args.step is None or self.args.step == 'make'
self.thirdparty_dir = self.args.thirdparty_dir
path_env_var_value: Optional[str] = os.getenv('PATH')
if path_env_var_value is None:
self.original_path = []
logging.warning("PATH is not set")
else:
self.original_path = path_env_var_value.split(':')
if not self.original_path:
logging.warning("PATH is empty")
self.compiler_family = self.args.compiler_family
self.compiler_version = self.args.compiler_version
def set_global_conf_from_args(args):
build_root = os.path.realpath(args.build_root)
# This module is expected to be under python/yb.
yb_src_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
# Ensure that build_root is consistent with yb_src_root above.
yb_src_root_from_build_root = os.path.dirname(os.path.dirname(build_root))
build_type = get_build_type_from_build_root(build_root)
assert yb_src_root == yb_src_root_from_build_root, \
("An inconstency between YB_SRC_ROOT derived from module location ({}) vs. the one derived "
"from BUILD_ROOT ({})").format(yb_src_root, yb_src_root_from_build_root)
global global_conf
global_conf = GlobalTestConfig(
build_root=build_root,
build_type=build_type,
yb_src_root=yb_src_root,
mvn_local_repo=os.environ.get(
'YB_MVN_LOCAL_REPO',
os.path.join(os.path.expanduser('~'), '.m2', 'repository')))
return global_conf
def set_global_conf_from_args(args):
build_root = os.path.realpath(args.build_root)
# This module is expected to be under python/yb.
yb_src_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
# Ensure that build_root is consistent with yb_src_root above.
yb_src_root_from_build_root = os.path.dirname(os.path.dirname(build_root))
build_type = get_build_type_from_build_root(build_root)
assert yb_src_root == yb_src_root_from_build_root, \
("An inconstency between YB_SRC_ROOT derived from module location ({}) vs. the one derived "
"from BUILD_ROOT ({})").format(yb_src_root, yb_src_root_from_build_root)
global global_conf
global_conf = GlobalTestConfig(
build_root=build_root,
build_type=build_type,
yb_src_root=yb_src_root,
mvn_local_repo=os.environ.get(
'YB_MVN_LOCAL_REPO',
os.path.join(os.path.expanduser('~'), '.m2', 'repository')))
return global_conf
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build',
help='Build type (debug/release)',
dest='build_type')
parser.add_argument(
'--build_root',
help=
'The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument(
'--build_args',
default='',
help='Additional arguments to pass to the build script')
parser.add_argument(
'--build_archive',
action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination',
help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build',
help='Skip building the code',
action='store_true')
parser.add_argument(
'--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir',
action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument(
'--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
parser.add_argument('--yw',
action='store_true',
help='Package YugaWare too.')
parser.add_argument(
'--no_reinitdb',
action='store_true',
help='Do not re-create the initial sys catalog snapshot. Useful when '
'debugging the release process.')
add_common_arguments(parser)
args = parser.parse_args()
# ---------------------------------------------------------------------------------------------
# Processing the arguments
# ---------------------------------------------------------------------------------------------
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info(
"Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified."
)
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError(
'--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build",
"yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError((
"Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugabyteDB {} build".format(build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh", "--write-build-descriptor", build_desc_path,
build_type
]
if args.force:
build_cmd_list.append("--force")
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, etc. based on that.
build_cmd_list += ["--build-root", build_root]
build_cmd_list += [
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
# We do not need java code built for a release package
"--skip-java"
]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
# TODO: run with shell=True and append build_args as is.
build_cmd_list += args.build_args.strip().split()
# ---------------------------------------------------------------------------------------------
# Perform the build
# ---------------------------------------------------------------------------------------------
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
if not args.skip_build:
# TODO: figure out the dependency issues in our CMake build instead.
# TODO: move this into yb_build.sh itself.
for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
preliminary_step_cmd_list = [
arg for arg in build_cmd_list if arg != 'packaged_targets'
] + ['--target', preliminary_target, '--skip-java']
logging.info("Running a preliminary step to build target %s: %s",
preliminary_target, preliminary_step_cmd_list)
subprocess.check_call(preliminary_step_cmd_list)
# We still need to call this even when --skip-build is specified to generate the "build
# descriptor" YAML file.
final_build_cmd_list = build_cmd_list + ([] if args.no_reinitdb else
['reinitdb'])
logging.info("Running final build step: %s", final_build_cmd_list)
subprocess.check_call(final_build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError(
"The build script failed to generate build descriptor file at '{}'"
.format(build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
# We are guaranteed to have a build_root by now.
library_packager_module.set_build_root(build_root)
thirdparty_dir = build_desc["thirdparty_dir"]
thirdparty_dir_from_env = os.environ.get("YB_THIRDPARTY_DIR",
thirdparty_dir)
if (thirdparty_dir != thirdparty_dir_from_env
and thirdparty_dir_from_env != os.path.join(
YB_SRC_ROOT, 'thirdparty')):
raise RuntimeError(
"Mismatch between non-default valueo of env YB_THIRDPARTY_DIR: '{}' and build desc "
"thirdparty_dir: '{}'".format(thirdparty_dir_from_env,
thirdparty_dir))
# Set the var for in-memory access to it across the other python files.
set_thirdparty_dir(thirdparty_dir)
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(YB_SRC_ROOT, build_type, build_target,
args.force, args.commit, build_root)
system = platform.system().lower()
library_packager_args = dict(
build_dir=build_root,
seed_executable_patterns=release_util.get_seed_executable_patterns(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
if system == "linux":
library_packager = LibraryPackager(**library_packager_args)
elif system == "darwin":
library_packager = MacLibraryPackager(**library_packager_args)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError(
"Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
# ---------------------------------------------------------------------------------------------
# Invoke YugaWare packaging
# ---------------------------------------------------------------------------------------------
if args.yw:
managed_dir = os.path.join(YB_SRC_ROOT, "managed")
yw_dir = os.path.join(build_target, "ui")
if not os.path.exists(yw_dir):
os.makedirs(yw_dir)
package_yw_cmd = [
os.path.join(managed_dir, "yb_release"), "--destination", yw_dir,
"--unarchived"
]
logging.info("Creating YugaWare package with command: {}".format(
package_yw_cmd))
try:
subprocess.check_output(package_yw_cmd, cwd=managed_dir)
except subprocess.CalledProcessError as e:
logging.error(
"Failed to build YugaWare package:\n{}\nOutput:\n{}".format(
traceback.format_exc(), e.output))
raise
except OSError as e:
logging.error("Failed to build YugaWare package: {}".format(e))
raise
# ---------------------------------------------------------------------------------------------
# Create a tar.gz (we almost always do this, can be skipped in debug mode)
# ---------------------------------------------------------------------------------------------
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(
args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file,
'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build', help='Build type (debug/release)', dest='build_type')
parser.add_argument('--build_root',
help='The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument('--build_args', default='',
help='Additional arguments to pass to the build script')
parser.add_argument('--build_archive', action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination', help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition', help='Which edition the code is built as.',
default=None,
choices=RELEASE_EDITION_ALLOWED_VALUES)
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build', help='Skip building the code', action='store_true')
parser.add_argument('--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir', action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument('--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
add_common_arguments(parser)
args = parser.parse_args()
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info("Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified.")
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError('--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
if build_root and not args.edition:
build_root_basename = os.path.basename(build_root)
if '-community-' in build_root_basename or build_root_basename.endswith('community'):
logging.info("Setting edition to Community based on build root")
args.edition = RELEASE_EDITION_COMMUNITY
elif '-enterprise-' in build_root_basename or build_root_basename.endswith('enterprise'):
logging.info("Setting edition to Enterprise based on build root")
args.edition = RELEASE_EDITION_ENTERPRISE
# We must have either not had a build_root specified, or not been able to deduce from the root.
if not args.edition:
ent_path = "{}/ent".format(YB_SRC_ROOT)
if os.path.isdir(ent_path):
logging.info(
"No --edition was specified, but found '{}', releasing Enterprise edition.".format(
ent_path))
args.edition = RELEASE_EDITION_ENTERPRISE
else:
logging.info(
"No --edition was specified and '{}' is not a valid dir, releasing Community "
"edition.".format(ent_path))
args.edition = RELEASE_EDITION_COMMUNITY
build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community"
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError(
("Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugaByte DB {} Edition: '{}' build".format(
build_edition.capitalize(), build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh",
"--write-build-descriptor", build_desc_path,
"--edition", build_edition,
build_type
]
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, edition, etc. based on that.
build_cmd_list += ["--build-root", build_root]
build_cmd_list += [
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
# We do not need java code built for a release package
"--skip-java"
]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
# TODO: run with shell=True and append build_args as is.
build_cmd_list += args.build_args.strip().split()
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
if not args.skip_build:
# TODO: figure out the dependency issues in our CMake build instead.
# TODO: move this into yb_build.sh itself.
for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
preliminary_step_cmd_list = [
arg for arg in build_cmd_list if arg != 'packaged_targets'
] + ['--target', preliminary_target, '--skip-java']
logging.info(
"Running a preliminary step to build target %s: %s",
preliminary_target,
" ".join(preliminary_step_cmd_list))
subprocess.check_call(preliminary_step_cmd_list)
subprocess.check_call(build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError("The build script failed to generate build descriptor file at '{}'".format(
build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
thirdparty_dir = build_desc["thirdparty_dir"]
thirdparty_dir_from_env = os.environ.get("YB_THIRDPARTY_DIR", thirdparty_dir)
if (thirdparty_dir != thirdparty_dir_from_env and
thirdparty_dir_from_env != os.path.join(YB_SRC_ROOT, 'thirdparty')):
raise RuntimeError(
"Mismatch between non-default valueo of env YB_THIRDPARTY_DIR: '{}' and build desc "
"thirdparty_dir: '{}'".format(thirdparty_dir_from_env, thirdparty_dir))
# Set the var for in-memory access to it across the other python files.
set_thirdparty_dir(thirdparty_dir)
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(
YB_SRC_ROOT, build_type, args.edition, build_target, args.force,
args.commit, build_root)
system = platform.system().lower()
library_packager_args = dict(
build_dir=build_root,
seed_executable_patterns=release_util.get_seed_executable_patterns(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose
)
if system == "linux":
library_packager = LibraryPackager(**library_packager_args)
elif system == "darwin":
library_packager = MacLibraryPackager(**library_packager_args)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError("Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file, 'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build', help='Build type (debug/release)', dest='build_type')
parser.add_argument('--build_root',
help='The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument('--build_args', default='',
help='Additional arguments to pass to the build script')
parser.add_argument('--build_archive', action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination', help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition', help='Which edition the code is built as.',
default=None,
choices=RELEASE_EDITION_ALLOWED_VALUES)
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build', help='Skip building the code', action='store_true')
parser.add_argument('--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir', action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument('--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
parser.add_argument('--java_only', action="store_true",
help="Only build the java part of the code. Does not generate an archive!")
add_common_arguments(parser)
args = parser.parse_args()
init_env(args.verbose)
if not args.build_target and not args.build_archive and not args.java_only:
logging.info("Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified.")
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError('--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
if build_root and not args.edition:
build_root_basename = os.path.basename(build_root)
if '-community-' in build_root_basename or build_root_basename.endswith('community'):
logging.info("Setting edition to Community based on build root")
args.edition = RELEASE_EDITION_COMMUNITY
if not args.edition:
# Here we are not detecting edition based on the existence of the enterprise source
# directory.
args.edition = RELEASE_EDITION_ENTERPRISE
build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community"
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build", "yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError(
("Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugaByte DB {} Edition: '{}' build{}".format(
build_edition.capitalize(), build_type, " Java only" if args.java_only else ""))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh",
"--with-assembly",
"--write-build-descriptor", build_desc_path,
"--edition", build_edition,
build_type
]
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, edition, etc. based on that.
build_cmd_list += ["--build-root", build_root]
if args.java_only:
build_cmd_list += ["--java-only"]
else:
build_cmd_list += [
# This will build the exact set of targets that are needed for the release.
"packaged_targets"
]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
# TODO: run with shell=True and append build_args as is.
build_cmd_list += args.build_args.strip().split()
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
if not args.java_only and not args.skip_build:
# TODO: figure out the dependency issues in our CMake build instead.
# TODO: move this into yb_build.sh itself.
for preliminary_target in ['protoc-gen-insertions', 'bfql_codegen']:
preliminary_step_cmd_list = [
arg for arg in build_cmd_list if arg != 'packaged_targets'
] + ['--target', preliminary_target, '--skip-java']
logging.info(
"Running a preliminary step to build target %s: %s",
preliminary_target,
" ".join(preliminary_step_cmd_list))
subprocess.check_call(preliminary_step_cmd_list)
subprocess.check_call(build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError("The build script failed to generate build descriptor file at '{}'".format(
build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
thirdparty_dir = build_desc["thirdparty_dir"]
if thirdparty_dir != os.environ.get("YB_THIRDPARTY_DIR", thirdparty_dir):
raise RuntimeError(
"Mismatch between env YB_THIRDPARTY_DIR: '{}' and build desc thirdparty_dir: '{}'"
.format(os.environ["YB_THIRDPARTY_DIR"], thirdparty_dir))
# Set the var for in-memory access to it across the other python files.
set_thirdparty_dir(thirdparty_dir)
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(YB_SRC_ROOT, build_type, args.edition, build_target, args.force,
args.commit)
release_util.rewrite_manifest(build_root)
if not args.java_only:
system = platform.system().lower()
if system == "linux":
library_packager = LibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
elif system == "darwin":
library_packager = MacLibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
if args.java_only:
logging.info("Directory '{}' exists and is not empty, but we are using --java_only")
else:
raise RuntimeError("Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target, "java" if args.java_only else None)
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file, 'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build',
help='Build type (debug/release)',
dest='build_type')
parser.add_argument(
'--build_root',
help=
'The root directory where the code is being built. If the build type '
'is specified, it needs to be consistent with the build root.')
parser.add_argument(
'--build_args',
default='',
help='Additional arguments to pass to the build script')
parser.add_argument(
'--build_archive',
action='store_true',
help='Whether or not we should build a package. This defaults to '
'false if --build_target is specified, true otherwise.')
parser.add_argument('--destination',
help='Copy release to Destination folder.')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition',
help='Which edition the code is built as.',
default=None,
choices=RELEASE_EDITION_ALLOWED_VALUES)
parser.add_argument('--commit', help='Custom specify a git commit.')
parser.add_argument('--skip_build',
help='Skip building the code',
action='store_true')
parser.add_argument(
'--build_target',
help='Target directory to put the YugaByte distribution into. This can '
'be used for debugging this script without having to build the '
'tarball. If specified, this directory must either not exist or be '
'empty.')
parser.add_argument('--keep_tmp_dir',
action='store_true',
help='Keep the temporary directory (for debugging).')
parser.add_argument(
'--save_release_path_to_file',
help='Save the newly built release path to a file with this name. '
'This allows to post-process / upload the newly generated release '
'in an enclosing script.')
add_common_arguments(parser)
args = parser.parse_args()
init_env(args.verbose)
if not args.build_target and not args.build_archive:
logging.info(
"Implying --build_archive (build package) because --build_target "
"(a custom directory to put YB distribution files into) is not specified."
)
args.build_archive = True
if not args.build_archive and args.save_release_path_to_file:
raise RuntimeError(
'--save_release_path_to_file does not make sense without '
'--build_archive')
build_root = args.build_root
if build_root and not args.edition:
build_root_basename = os.path.basename(build_root)
if '-community-' in build_root_basename or build_root_basename.endswith(
'community'):
logging.info("Setting edition to Community based on build root")
args.edition = RELEASE_EDITION_COMMUNITY
if not args.edition:
# Here we are not detecting edition based on the existence of the enterprise source
# directory.
args.edition = RELEASE_EDITION_ENTERPRISE
build_edition = "enterprise" if args.edition == RELEASE_EDITION_ENTERPRISE else "community"
build_type = args.build_type
tmp_dir = os.path.join(YB_SRC_ROOT, "build",
"yb_release_tmp_{}".format(str(uuid.uuid4())))
try:
os.mkdir(tmp_dir)
except OSError as e:
logging.error("Could not create directory at '{}'".format(tmp_dir))
raise e
if not args.keep_tmp_dir:
atexit.register(lambda: shutil.rmtree(tmp_dir))
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(YB_SRC_ROOT)
if not args.force:
check_for_local_changes()
# This is not a "target" in terms of Make / CMake, but a target directory.
build_target = args.build_target
if build_target is None:
# Use a temporary directory.
build_target = os.path.join(tmp_dir, 'tmp_yb_distribution')
if build_root:
build_type_from_build_root = get_build_type_from_build_root(build_root)
if build_type:
if build_type != build_type_from_build_root:
raise RuntimeError((
"Specified build type ('{}') is inconsistent with the specified build "
"root ('{}')").format(build_type, build_root))
else:
build_type = build_type_from_build_root
if not build_type:
build_type = 'release'
logging.info("Building YugaByte DB {} Edition: '{}' build".format(
build_edition.capitalize(), build_type))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
build_cmd_list = [
"./yb_build.sh",
"--with-assembly",
"--write-build-descriptor",
build_desc_path,
"--edition",
build_edition,
# This will build the exact set of targets that are needed for the release.
"packaged_targets",
build_type
]
if build_root:
# This will force yb_build.sh to use this build directory, and detect build type,
# compiler type, edition, etc. based on that.
build_cmd_list += ["--build-root", build_root]
if args.skip_build:
build_cmd_list += ["--skip-build"]
if args.build_args:
build_cmd_list += args.build_args.strip().split()
build_cmd_line = " ".join(build_cmd_list).strip()
logging.info("Build command line: {}".format(build_cmd_line))
subprocess.check_call(build_cmd_list)
if not os.path.exists(build_desc_path):
raise IOError(
"The build script failed to generate build descriptor file at '{}'"
.format(build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
logging.info("Build descriptor: {}".format(build_desc))
build_root_from_build_desc = build_desc['build_root']
if not build_root:
build_root = build_root_from_build_desc
build_root = os.path.realpath(build_root)
if build_root != os.path.realpath(build_root_from_build_desc):
raise RuntimeError(
"Build root from the build descriptor file (see above) is inconsistent with that "
"specified on the command line ('{}')".format(build_root))
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_util = ReleaseUtil(YB_SRC_ROOT, build_type, args.edition,
build_target, args.force, args.commit)
release_util.rewrite_manifest(build_root)
system = platform.system().lower()
if system == "linux":
library_packager = LibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
elif system == "darwin":
library_packager = MacLibraryPackager(
build_dir=build_root,
seed_executable_patterns=release_util.get_binary_path(),
dest_dir=yb_distribution_dir,
verbose_mode=args.verbose)
else:
raise RuntimeError("System {} not supported".format(system))
library_packager.package_binaries()
release_util.update_manifest(yb_distribution_dir)
logging.info("Generating release distribution")
if os.path.exists(build_target) and os.listdir(build_target):
raise RuntimeError(
"Directory '{}' exists and is non-empty".format(build_target))
release_util.create_distribution(build_target)
if args.build_archive:
release_file = os.path.realpath(release_util.generate_release())
if args.destination:
if not os.path.exists(args.destination):
raise RuntimeError("Destination {} not a directory.".format(
args.destination))
shutil.copy(release_file, args.destination)
logging.info("Generated a package at '{}'".format(release_file))
if args.save_release_path_to_file:
with open(args.save_release_path_to_file,
'w') as release_path_file:
release_path_file.write(release_file)
logging.info("Saved package path to '{}'".format(
args.save_release_path_to_file))
