def get_short_os_name() -> str:
platform_system = platform.system()
os_name = ''
os_version = ''
if platform_system == 'Linux':
os_release_path = '/etc/os-release'
if os.path.exists(os_release_path):
os_release_str = read_file(os_release_path)
os_release_lines = [line.strip() for line in os_release_str.split('\n') if line.strip()]
os_release_tuples = [line.split('=') for line in os_release_lines]
kvs = {t[0]: t[1].strip('"') for t in os_release_tuples if len(t) == 2}
name = kvs.get('NAME')
version_id = kvs.get('VERSION_ID')
if name and version_id:
os_name = name.split()[0].lower()
if os_name == 'ubuntu':
# For Ubuntu we will keep the full version, such as 18.04.
os_version = version_id
else:
os_version = version_id.split('.')[0]
else:
raise ValueError(
f"Could not determine OS name and version from the contents of "
f"{os_release_path}. Parsed data: {kvs}. Raw contents:\n{os_release_str}")
else:
raise ValueError(
f"Cannot identify OS release. File {os_release_path} is not present.")
elif platform_system == 'Darwin':
os_name = 'macos'
else:
raise ValueError(f"Unsupported platform: {platform_system}")
return os_name + os_version
def _read_path_file(self, path_file_name: str) -> Optional[str]:
path_file_path = os.path.join(self.build_root, path_file_name)
if not os.path.exists(path_file_path):
return None
dir_path = read_file(path_file_path).strip()
if not os.path.exists(dir_path):
raise IOError(
"Path contained in file %s does not exist or is not a directory: '%s'"
% (path_file_path, dir_path))
return dir_path
def get_github_token(token_file_path: Optional[str]) -> Optional[str]:
github_token: Optional[str]
if token_file_path:
github_token = read_file(token_file_path).strip()
else:
github_token = os.getenv('GITHUB_TOKEN')
if github_token is None:
return github_token
if len(github_token) != 40:
raise ValueError(f"Invalid GitHub token length: {len(github_token)}, expected 40.")
return github_token
def get_yb_pgbackend_link_cmd(build_root: str) -> Tuple[str, List[str]]:
"""
Returns a tuple containing the Postgres backend build directory and a list representation
of the linker command line for the yb_pgbackend library.
"""
pg_backend_build_dir = os.path.join(build_root, 'postgres_build', 'src',
'backend')
if not os.path.exists(pg_backend_build_dir):
raise IOError("Directory does not exist: %s" % pg_backend_build_dir)
prefix_str = 'link_cmd_libyb_pgbackend.so.'
matched_files = [
os.fspath(p)
for p in Path(pg_backend_build_dir).glob('%s*' % prefix_str)
]
if len(matched_files) != 1:
raise ValueError(
"Looking for the build command for the yb_pgbackend library, failed to find exactly "
"one file starting with %s in %s. Got %s" %
(prefix_str, pg_backend_build_dir, matched_files))
return pg_backend_build_dir, read_file(matched_files[0]).strip().split()
def update_archive_metadata_file(self) -> None:
yb_version = read_file(os.path.join(YB_SRC_ROOT, 'version.txt')).strip()
archive_metadata_path = get_archive_metadata_file_path()
logging.info(f"Updating third-party archive metadata file in {archive_metadata_path}")
github_client = Github(get_github_token(self.github_token_file_path))
repo = github_client.get_repo('yugabyte/yugabyte-db-thirdparty')
releases_by_commit: Dict[str, ReleaseGroup] = {}
num_skipped_old_tag_format = 0
num_skipped_wrong_branch = 0
num_releases_found = 0
releases = []
get_releases_start_time_sec = time.time()
try:
for release in repo.get_releases():
releases.append(release)
except GithubException as exc:
if 'Only the first 1000 results are available.' in str(exc):
logging.info("Ignoring exception: %s", exc)
else:
raise exc
logging.info("Time spent to iterate all releases: %.1f sec",
time.time() - get_releases_start_time_sec)
for release in releases:
sha: str = release.target_commitish
assert(isinstance(sha, str))
if SHA_HASH.match(sha) is None:
sha = repo.get_commit(sha).sha
tag_name = release.tag_name
if len(tag_name.split('-')) <= 2:
logging.debug(f"Skipping release tag: {tag_name} (old format, too few components)")
num_skipped_old_tag_format += 1
continue
if self.tag_filter_pattern and not self.tag_filter_pattern.match(tag_name):
logging.info(f'Skipping tag {tag_name}, does not match the filter')
continue
try:
yb_dep_release = GitHubThirdPartyRelease(release, target_commitish=sha)
except SkipThirdPartyReleaseException as ex:
logging.warning("Skipping release: %s", ex)
continue
if not yb_dep_release.is_consistent_with_yb_version(yb_version):
logging.debug(
f"Skipping release tag: {tag_name} (does not match version {yb_version}")
num_skipped_wrong_branch += 1
continue
if sha not in releases_by_commit:
releases_by_commit[sha] = ReleaseGroup(sha)
num_releases_found += 1
logging.debug(f"Found release: {yb_dep_release}")
releases_by_commit[sha].add_release(yb_dep_release)
if num_skipped_old_tag_format > 0:
logging.info(f"Skipped {num_skipped_old_tag_format} releases due to old tag format")
if num_skipped_wrong_branch > 0:
logging.info(f"Skipped {num_skipped_wrong_branch} releases due to branch mismatch")
logging.info(
f"Found {num_releases_found} releases for {len(releases_by_commit)} different commits")
latest_group_by_max = max(
releases_by_commit.values(), key=ReleaseGroup.get_max_creation_timestamp)
latest_group_by_min = max(
releases_by_commit.values(), key=ReleaseGroup.get_min_creation_timestamp)
if latest_group_by_max is not latest_group_by_min:
raise ValueError(
"Overlapping releases for different commits. No good way to identify latest "
"release: e.g. {latest_group_by_max.sha} and {latest_group_by_min.sha}.")
latest_group: ReleaseGroup = latest_group_by_max
latest_release_sha = latest_group.sha
logging.info(
f"Latest released yugabyte-db-thirdparty commit: {latest_release_sha}. "
f"Released at: {latest_group.get_max_creation_timestamp()}.")
groups_to_use: List[ReleaseGroup] = [latest_group]
if self.also_use_commits:
for extra_commit in self.also_use_commits:
logging.info(f"Additional manually specified commit to use: {extra_commit}")
if extra_commit == latest_release_sha:
logging.info(
f"(already matches the latest commit {latest_release_sha}, skipping.)")
continue
if extra_commit not in releases_by_commit:
raise ValueError(
f"No releases found for user-specified commit {extra_commit}. "
"Please check if there is an error.")
groups_to_use.append(releases_by_commit[extra_commit])
new_metadata: Dict[str, Any] = {
SHA_FOR_LOCAL_CHECKOUT_KEY: latest_release_sha,
'archives': []
}
releases_to_use: List[GitHubThirdPartyRelease] = [
rel for release_group in groups_to_use
for rel in release_group.releases
if rel.tag not in BROKEN_TAGS
]
releases_by_key_without_tag: DefaultDict[Tuple[str, ...], List[GitHubThirdPartyRelease]] = \
defaultdict(list)
num_valid_releases = 0
num_invalid_releases = 0
for yb_thirdparty_release in releases_to_use:
if yb_thirdparty_release.validate_url():
num_valid_releases += 1
releases_by_key_without_tag[
yb_thirdparty_release.get_sort_key(include_tag=False)
].append(yb_thirdparty_release)
else:
num_invalid_releases += 1
logging.info(
f"Valid releases found: {num_valid_releases}, invalid releases: {num_invalid_releases}")
filtered_releases_to_use = []
for key_without_tag, releases_for_key in releases_by_key_without_tag.items():
if len(releases_for_key) > 1:
picked_release = max(releases_for_key, key=lambda r: r.tag)
logging.info(
"Multiple releases found for the same key (excluding the tag). "
"Using the latest one: %s\n"
"Key: %s.\nReleases:\n %s" % (
picked_release,
key_without_tag,
'\n '.join([str(r) for r in releases_for_key])))
filtered_releases_to_use.append(picked_release)
else:
filtered_releases_to_use.append(releases_for_key[0])
filtered_releases_to_use.sort(key=GitHubThirdPartyRelease.get_sort_key)
for yb_thirdparty_release in filtered_releases_to_use:
new_metadata['archives'].append(yb_thirdparty_release.as_dict())
write_yaml_file(new_metadata, archive_metadata_path)
logging.info(
f"Wrote information for {len(filtered_releases_to_use)} pre-built "
f"yugabyte-db-thirdparty archives to {archive_metadata_path}.")
def update_archive_metadata_file(self) -> None:
yb_version = read_file(os.path.join(YB_SRC_ROOT,
'version.txt')).strip()
archive_metadata_path = get_archive_metadata_file_path()
logging.info(
f"Updating third-party archive metadata file in {archive_metadata_path}"
)
github_client = Github(get_github_token(self.github_token_file_path))
repo = github_client.get_repo('yugabyte/yugabyte-db-thirdparty')
releases_by_commit: Dict[str, ReleaseGroup] = {}
num_skipped_old_tag_format = 0
num_skipped_wrong_branch = 0
num_releases_found = 0
for release in repo.get_releases():
sha: str = release.target_commitish
assert (isinstance(sha, str))
tag_name = release.tag_name
if len(tag_name.split('-')) <= 2:
logging.debug(
f"Skipping release tag: {tag_name} (old format, too few components)"
)
num_skipped_old_tag_format += 1
continue
if self.tag_filter_pattern and not self.tag_filter_pattern.match(
tag_name):
logging.info(
f'Skipping tag {tag_name}, does not match the filter')
continue
yb_dep_release = GitHubThirdPartyRelease(release)
if not yb_dep_release.is_consistent_with_yb_version(yb_version):
logging.debug(
f"Skipping release tag: {tag_name} (does not match version {yb_version}"
)
num_skipped_wrong_branch += 1
continue
if sha not in releases_by_commit:
releases_by_commit[sha] = ReleaseGroup(sha)
num_releases_found += 1
logging.info(f"Found release: {yb_dep_release}")
releases_by_commit[sha].add_release(yb_dep_release)
if num_skipped_old_tag_format > 0:
logging.info(
f"Skipped {num_skipped_old_tag_format} releases due to old tag format"
)
if num_skipped_wrong_branch > 0:
logging.info(
f"Skipped {num_skipped_wrong_branch} releases due to branch mismatch"
)
logging.info(
f"Found {num_releases_found} releases for {len(releases_by_commit)} different commits"
)
latest_group_by_max = max(releases_by_commit.values(),
key=ReleaseGroup.get_max_creation_timestamp)
latest_group_by_min = max(releases_by_commit.values(),
key=ReleaseGroup.get_min_creation_timestamp)
if latest_group_by_max is not latest_group_by_min:
raise ValueError(
"Overlapping releases for different commits. No good way to identify latest "
"release: e.g. {latest_group_by_max.sha} and {latest_group_by_min.sha}."
)
latest_group = latest_group_by_max
sha = latest_group.sha
logging.info(
f"Latest released yugabyte-db-thirdparty commit: f{sha}. "
f"Released at: {latest_group.get_max_creation_timestamp()}.")
new_metadata: Dict[str, Any] = {
SHA_FOR_LOCAL_CHECKOUT_KEY: sha,
'archives': []
}
releases_for_one_commit = [
rel for rel in latest_group.releases if rel.tag not in BROKEN_TAGS
]
releases_by_key_without_tag: DefaultDict[Tuple[str, ...], List[GitHubThirdPartyRelease]] = \
defaultdict(list)
num_valid_releases = 0
num_invalid_releases = 0
for yb_thirdparty_release in releases_for_one_commit:
if yb_thirdparty_release.validate_url():
num_valid_releases += 1
releases_by_key_without_tag[yb_thirdparty_release.get_sort_key(
include_tag=False)].append(yb_thirdparty_release)
else:
num_invalid_releases += 1
logging.info(
f"Valid releases found: {num_valid_releases}, invalid releases: {num_invalid_releases}"
)
filtered_releases_for_one_commit = []
for key_without_tag, releases_for_key in releases_by_key_without_tag.items(
):
if len(releases_for_key) > 1:
picked_release = max(releases_for_key, key=lambda r: r.tag)
logging.info(
"Multiple releases found for the same key (excluding the tag). "
"Using the latest one: %s\n"
"Key: %s.\nReleases:\n %s" %
(picked_release, key_without_tag, '\n '.join(
[str(r) for r in releases_for_key])))
filtered_releases_for_one_commit.append(picked_release)
else:
filtered_releases_for_one_commit.append(releases_for_key[0])
filtered_releases_for_one_commit.sort(
key=GitHubThirdPartyRelease.get_sort_key)
for yb_thirdparty_release in filtered_releases_for_one_commit:
new_metadata['archives'].append(yb_thirdparty_release.as_dict())
write_yaml_file(new_metadata, archive_metadata_path)
logging.info(
f"Wrote information for {len(filtered_releases_for_one_commit)} pre-built "
f"yugabyte-db-thirdparty archives to {archive_metadata_path}.")
