def get_release_file(self) -> str:
"""
This method does couple of checks before generating the release file name.
- Checks if there are local uncommitted changes.
- Checks if there are local commits which aren't merged upstream.
- Reads the base version from the version.txt file and appends to the filename.
Also fetches the platform the release file is being built and adds that to the file name
along with commit hash and built type.
Returns:
(string): Release file path.
"""
components: List[str] = [
self.base_version, self.commit, self.build_type
]
compiler_type = get_compiler_type_from_build_root(self.build_root)
# Make the clang12 release package the default, and append the compiler type for all other
# compiler types so we can still use them with the appropriate support from the downstream
# tooling.
if compiler_type != 'clang12':
components.append(compiler_type)
release_name = "-".join(components)
system = platform.system().lower()
if system == "linux":
system = distro.linux_distribution(
full_distribution_name=False)[0].lower()
release_file_name = "yugabyte-{}-{}-{}.tar.gz".format(
release_name, system,
platform.machine().lower())
return os.path.join(self.build_path, release_file_name)
def set_global_conf_from_args(args: argparse.Namespace) -> GlobalTestConfig:
build_root = os.path.realpath(args.build_root)
# This module is expected to be under python/yb.
yb_src_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
# Ensure that build_root is consistent with yb_src_root above.
yb_src_root_from_build_root = os.path.dirname(os.path.dirname(build_root))
build_type = get_build_type_from_build_root(build_root)
archive_for_workers = None
if args.send_archive_to_workers:
archive_for_workers = os.path.abspath(
os.path.join(build_root,
'archive_for_tests_on_spark.tar.gz.spark-no-extract'))
assert yb_src_root == yb_src_root_from_build_root, \
("An inconstency between YB_SRC_ROOT derived from module location ({}) vs. the one derived "
"from BUILD_ROOT ({})").format(yb_src_root, yb_src_root_from_build_root)
rel_build_root = os.path.relpath(os.path.abspath(build_root),
os.path.abspath(yb_src_root))
if len(rel_build_root.split('/')) != 2:
raise ValueError(
"Unexpected number of components in the relative path of build root to "
"source root: %s. build_root=%s, yb_src_root=%s" %
(rel_build_root, build_root, yb_src_root))
compiler_type = get_compiler_type_from_build_root(build_root)
compiler_type_from_env = os.environ.get('YB_COMPILER_TYPE')
if compiler_type_from_env is not None and compiler_type_from_env != compiler_type:
raise ValueError(
"Build root '%s' implies compiler type '%s' but YB_COMPILER_TYPE is '%s'"
% (build_root, compiler_type, compiler_type_from_env))
from yb import common_util
os.environ['YB_COMPILER_TYPE'] = compiler_type
global global_conf
global_conf = GlobalTestConfig(
build_root=build_root,
build_type=build_type,
yb_src_root=yb_src_root,
archive_for_workers=archive_for_workers,
rel_build_root=rel_build_root,
compiler_type=compiler_type,
# The archive might not even exist yet.
archive_sha256sum=None)
return global_conf
def get_release_file(self) -> str:
"""
This method does couple of checks before generating the release file name.
- Checks if there are local uncommitted changes.
- Checks if there are local commits which aren't merged upstream.
- Reads the base version from the version.txt file and appends to the filename.
Also fetches the platform the release file is being built and adds that to the file name
along with commit hash and built type.
Returns:
(string): Release file path.
"""
components: List[str] = [
self.base_version, self.commit, self.build_type
]
compiler_type = get_compiler_type_from_build_root(self.build_root)
# Make the clang12 release package the default, and append the compiler type for all other
# compiler types so we can still use them with the appropriate support from the downstream
# tooling.
if compiler_type != 'clang12':
components.append(compiler_type)
release_name = "-".join(components)
system = platform.system().lower()
if system == "linux":
# We recently moved from centos7 to almalinux8 as the build host for our universal
# x86_64 linux build. This changes the name of the release tarball we create.
# Unfortunately, we have a lot of hard coded references to the centos package names
# in our downsstream release code. So here we munge the name to 'centos' to keep things
# working while we fix downstream code.
# TODO(jharveymsith): Remove the almalinux to centos mapping once downstream is fixed.
if distro.id() == "centos" and distro.major_version() == "7" \
or distro.id() == "almalinux" and platform.machine().lower() == "x86_64":
system = "centos"
elif distro.id == "ubuntu":
system = distro.id() + distro.version()
else:
system = distro.id() + distro.major_version()
release_file_name = "{}-{}-{}-{}.tar.gz".format(
self.package_name, release_name, system,
platform.machine().lower())
return os.path.join(self.build_path, release_file_name)
def create_archive_for_workers() -> None:
assert global_conf is not None
dest_path = global_conf.archive_for_workers
if dest_path is None:
return
tmp_dest_path = '%s.tmp.%d' % (dest_path, random.randint(0, 2**64 - 1))
start_time_sec = time.time()
try:
build_root = os.path.abspath(global_conf.build_root)
compiler_type = get_compiler_type_from_build_root(build_root)
yb_src_root = os.path.abspath(global_conf.yb_src_root)
build_root_parent = os.path.join(yb_src_root, 'build')
rel_build_root = global_conf.rel_build_root
if os.path.exists(dest_path):
logging.info("Removing existing archive file %s", dest_path)
os.remove(dest_path)
paths_in_src_dir = ARCHIVED_PATHS_IN_SRC_DIR + find_rel_java_paths_to_archive(
yb_src_root)
added_local_repo = False
mvn_local_repo = os.environ.get('YB_MVN_LOCAL_REPO')
if mvn_local_repo:
mvn_local_repo = os.path.abspath(mvn_local_repo)
if mvn_local_repo.startswith(build_root_parent + '/'):
# Here, the path we're adding has to be relative to YB_SRC_ROOT.
paths_in_src_dir.append(
os.path.relpath(mvn_local_repo, yb_src_root))
logging.info("Will add YB_MVN_LOCAL_REPO to archive: %s",
mvn_local_repo)
validate_mvn_local_repo(mvn_local_repo)
added_local_repo = True
if not added_local_repo:
raise ValueError(
"YB_MVN_LOCAL_REPO (%s) must be within $YB_SRC_ROOT/build (%s)"
% (mvn_local_repo, build_root_parent))
files_that_must_exist_in_build_dir = ['thirdparty_path.txt']
# This will not include version-specific compiler types like clang11 or gcc9.
# We will eventually get rid of Linuxbrew and simplify this.
if sys.platform == 'linux' and compiler_type in ['gcc', 'clang']:
files_that_must_exist_in_build_dir.append('linuxbrew_path.txt')
for rel_file_path in files_that_must_exist_in_build_dir:
full_path = os.path.join(build_root, rel_file_path)
if not os.path.exists(full_path):
raise IOError("Path does not exist: %s" % full_path)
# TODO: save the list of files added to the archive to a separate file for debuggability.
# TODO: use zip instead of tar/gz.
tar_args = ['tar', 'cz', '-f', tmp_dest_path] + [
path_rel_to_src_dir for path_rel_to_src_dir in paths_in_src_dir
if os.path.exists(os.path.join(yb_src_root, path_rel_to_src_dir))
] + [
os.path.join(rel_build_root, path_rel_to_build_root)
for path_rel_to_build_root in ARCHIVED_PATHS_IN_BUILD_DIR
if os.path.exists(os.path.join(build_root, path_rel_to_build_root))
] + [
os.path.relpath(test_program_path, yb_src_root)
for test_program_path in glob.glob(
os.path.join(build_root, 'tests-*'))
if os.path.exists(test_program_path)
]
logging.info("Running the tar command: %s", tar_args)
subprocess.check_call(tar_args, cwd=global_conf.yb_src_root)
if not os.path.exists(tmp_dest_path):
raise IOError("Archive '%s' did not get created after command %s" %
(tmp_dest_path, tar_args))
os.rename(tmp_dest_path, dest_path)
logging.info("Size of the archive: %.1f MiB",
os.path.getsize(dest_path) / (1024.0 * 1024))
finally:
elapsed_time_sec = time.time() - start_time_sec
logging.info("Elapsed archive creation time: %.1f seconds",
elapsed_time_sec)
if os.path.exists(tmp_dest_path):
logging.warning("Removing unfinished temporary archive file %s",
tmp_dest_path)
os.remove(tmp_dest_path)