def add_metadata_cli(runtime, kind, impetus, advisory):
"""Add metadata to an advisory. This is usually called by
create immediately after creation. It is only useful to you if
you are going back and adding metadata to older advisories.
Note: Requires you provide a --group
Example to add standard metadata to a 3.10 images release
\b
$ elliott --group=openshift-3.10 add-metadata --impetus standard --kind image
"""
runtime.initialize()
release = release_from_branch(runtime.group_config.branch)
try:
advisory = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
result = elliottlib.errata.add_comment(
advisory.errata_id, {'release': release, 'kind': kind, 'impetus': impetus})
if result.status_code == 201:
green_prefix("Added metadata successfully")
click.echo()
elif result.status_code == 403:
exit_unauthorized()
else:
red_print("Something weird may have happened")
raise ElliottFatalError(
"Unexpected response from ET API: {code}".format(code=result.status_code))
def _filter_out_inviable_builds(kind, results, errata):
unshipped_builds = []
errata_version_cache = {
} # avoid reloading the same errata for multiple builds
for b in results:
# check if build is attached to any existing advisory for this version
in_same_version = False
for eid in [e['id'] for e in b.all_errata]:
if eid not in errata_version_cache:
metadata_comments_json = errata.get_metadata_comments_json(eid)
if not metadata_comments_json:
# Does not contain ART metadata; consider it unversioned
red_print(
"Errata {} Does not contain ART metadata\n".format(
eid))
errata_version_cache[eid] = ''
continue
# it's possible for an advisory to have multiple metadata comments,
# though not very useful (there's a command for adding them,
# but not much point in doing it). just looking at the first one is fine.
errata_version_cache[eid] = metadata_comments_json[0][
'release']
if errata_version_cache[eid] == get_release_version(
b.product_version):
in_same_version = True
break
if not in_same_version:
unshipped_builds.append(b)
return unshipped_builds
def _attached_to_open_erratum_with_correct_product_version(
results, product_version, errata):
unshipped_builds = []
# will probably end up loading the same errata and
# its comments many times, which is pretty slow
# so we cached the result.
errata_version_cache = {}
for b in results:
same_version_exist = False
# We only want builds not attached to an existing open advisory
if b.attached_to_open_erratum:
for e in b.open_errata_id:
if not errata_version_cache.get(e):
metadata_comments_json = errata.get_metadata_comments_json(
e)
if not metadata_comments_json:
# Does not contain ART metadata, skip it
red_print(
"Errata {} Does not contain ART metadata\n".format(
e))
continue
# it's possible for an advisory to have multiple metadata comments,
# though not very useful (there's a command for adding them,
# but not much point in doing it). just looking at the first one is fine.
errata_version_cache[e] = metadata_comments_json[0][
'release']
if errata_version_cache[e] == get_release_version(
product_version):
same_version_exist = True
break
if not same_version_exist or not b.attached_to_open_erratum:
unshipped_builds.append(b)
return unshipped_builds
def validate(self, advisories):
bugs = self._get_attached_filtered_bugs(advisories)
blocking_bugs_for = self._get_blocking_bugs_for(bugs)
self._verify_blocking_bugs(blocking_bugs_for)
if self.problems:
red_print("Some bug problems were listed above. Please investigate.")
exit(1)
green_print("All bugs were verified.")
def main():
try:
cli(obj={})
except ElliottFatalError as ex:
# Allow capturing actual tool errors and print them
# nicely instead of a gross stack-trace.
# All internal errors that should simply cause the app
# to exit with an error code should use ElliottFatalError
red_print(getattr(ex, 'message', repr(ex)))
sys.exit(1)
def _any_references_are_missing(references, available):
# check that referenced are all attached
missing = False
for image_pullspec, metadata in references.items():
digest = image_pullspec.split("@")[1] # just the shasum
if digest not in available:
missing = True
ref = image_pullspec.rsplit('/', 1)[1] # cut off the registry/namespace, just need the name:shasum
red_print(f"{metadata} has a reference to {ref} not present in the advisories nor shipped images.")
return missing
def validate(self, bugs, verify_bug_status):
blocking_bugs_for = self._get_blocking_bugs_for(bugs)
self._verify_blocking_bugs(blocking_bugs_for)
if verify_bug_status:
self._verify_bug_status(bugs)
if self.problems:
red_print(
"Some bug problems were listed above. Please investigate.")
exit(1)
green_print("All bugs were verified.")
def main():
try:
if 'REQUESTS_CA_BUNDLE' not in os.environ:
os.environ['REQUESTS_CA_BUNDLE'] = '/etc/pki/tls/certs/ca-bundle.crt'
cli(obj={})
except ElliottFatalError as ex:
# Allow capturing actual tool errors and print them
# nicely instead of a gross stack-trace.
# All internal errors that should simply cause the app
# to exit with an error code should use ElliottFatalError
red_print(getattr(ex, 'message', repr(ex)))
sys.exit(1)
def _fetch_builds_by_kind_rpm(tag_pv_map, brew_event, brew_session):
green_prefix('Generating list of rpms: ')
click.echo('Hold on a moment, fetching Brew builds')
rpm_tuple = []
for tag in tag_pv_map:
if tag.endswith('-candidate'):
base_tag = tag[:-10]
else:
red_print("key of brew_tag_product_version_mapping in erratatool.yml must be candidate\n")
continue
candidates = elliottlib.brew.find_unshipped_build_candidates(base_tag, event=brew_event, kind='rpm', session=brew_session)
rpm_tuple.extend(_gen_nvrp_tuples(candidates, tag_pv_map, tag))
return rpm_tuple
def verify_attached_bugs_cli(runtime, advisories):
"""
Verify the bugs in the advisories (specified as arguments or in group.yml) for a release.
Requires a runtime to ensure that all bugs in the advisories match the runtime version.
Also ensures that bugs in the next release which block bugs in these advisories have
been verified, as those represent backports we do not want to regress in upgrades.
If any verification fails, a text explanation is given and the return code is 1.
Otherwise, prints the number of bugs in the advisories and exits with success.
"""
runtime.initialize()
advisories = advisories or [a for a in runtime.group_config.get('advisories', {}).values()]
if not advisories:
red_print("No advisories specified on command line or in group.yml")
exit(1)
BugValidator(runtime).validate(advisories)
async def verify_attached_flaws(self, advisory_bugs: Dict[int, List[Bug]]):
futures = []
for advisory_id, attached_bugs in advisory_bugs.items():
attached_trackers = [
b for b in attached_bugs if b.is_tracker_bug()
]
attached_flaws = [b for b in attached_bugs if b.is_flaw_bug()]
futures.append(
self._verify_attached_flaws_for(advisory_id, attached_trackers,
attached_flaws))
await asyncio.gather(*futures)
if self.problems:
red_print(
"Some bug problems were listed above. Please investigate.")
exit(1)
green_print("All CVE flaw bugs were verified.")
def _missing_references(runtime, references, available):
# check that referenced are all attached
missing = set()
for image_pullspec, metadata in references.items():
digest = image_pullspec.split("@")[1] # just the shasum
if digest not in available:
ref = image_pullspec.rsplit('/', 1)[
1] # cut off the registry/namespace, just need the name:shasum
try:
ref = _nvr_for_operand_pullspec(runtime, ref)
except RuntimeError:
pass # just leave it as-is if something goes wrong with looking it up
missing.add(ref)
red_print(
f"{metadata} has a reference to {ref} not present in the advisories nor shipped images."
)
return missing
def get_rpm_nvrs(build_id, version, arch, private=''):
stream_name = f"{arch}{'-priv' if private else ''}"
try:
commitmeta = get_build_meta(build_id,
version,
arch,
private,
meta_type="commitmeta")
rpm_list = commitmeta.get("rpmostree.rpmdb.pkglist")
if not rpm_list:
raise Exception(f"no pkglist in {commitmeta}")
except Exception as ex:
problem = f"{stream_name}: {ex}"
util.red_print(f"error finding RHCOS {problem}")
return None
rpms = [(r[0], r[2], r[3]) for r in rpm_list]
return rpms
def verify_attached_operators_cli(runtime, exclude_shipped, advisories):
"""
Verify attached operator manifest references are shipping or already shipped.
Takes a list of advisories that may contain operator metadata/bundle builds
or image builds that are shipping alongside. Then determines whether the
operator manifests refer only to images that have shipped in the past or
are shipping in these advisories. An error is raised if there are no
manifest builds attached, or if any references are missing.
NOTE: this will fail before 4.3 because they referred to images not manifest lists.
"""
runtime.initialize()
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
image_builds = _get_attached_image_builds(brew_session, advisories)
referenced_specs = _extract_operator_manifest_image_references(
image_builds)
if not referenced_specs:
# you are probably using this because you expect attached operator bundles or metadata
adv_str = ", ".join(str(a) for a in advisories)
raise ElliottFatalError(
f"No bundle or appregistry builds found in advisories ({adv_str})."
)
if not exclude_shipped:
image_builds.extend(_get_shipped_images(runtime, brew_session))
# check if references are satisfied by any image we are shipping or have shipped
available_shasums = _extract_available_image_shasums(image_builds)
missing = _missing_references(runtime, referenced_specs, available_shasums)
if missing:
missing_str = "\n ".join(missing)
red_print(f"""
Some references were missing:
{missing_str}
Ensure all manifest references are shipped or shipping.
""")
raise ElliottFatalError("Some bundle references were missing.")
green_print("All operator manifest references were found.")
def get_latest_fast_ocp(version, arch):
"""
Queries Cincinnati and returns latest release version for the given X.Y version
from the fast channel
"""
arch = 'amd64' if arch == 'x86_64' else arch
channel = f'fast-{version}'
url = f'{constants.CINCINNATI_BASE_URL}?arch={arch}&channel={channel}'
req = urllib.request.Request(url)
req.add_header('Accept', 'application/json')
content = exectools.urlopen_assert(req).read()
graph = json.loads(content)
versions = [node['version'] for node in graph['nodes']]
if not versions:
util.red_print(f"No releases found in {channel}")
return
descending_versions = sort_semver(versions)
return descending_versions[0]
def validate(
self,
non_flaw_bugs: Iterable[Bug],
verify_bug_status: bool,
):
non_flaw_bugs = self.filter_bugs_by_release(non_flaw_bugs,
complain=True)
blocking_bugs_for = self._get_blocking_bugs_for(non_flaw_bugs)
self._verify_blocking_bugs(blocking_bugs_for)
if verify_bug_status:
self._verify_bug_status(non_flaw_bugs)
if self.problems:
if self.output != 'slack':
red_print(
"Some bug problems were listed above. Please investigate.")
exit(1)
green_print(
"All bugs were verified. This check doesn't cover CVE flaw bugs.")
async def verify_attached_bugs_cli(runtime: Runtime, verify_bug_status: bool,
advisories: Tuple[int,
...], verify_flaws: bool):
"""
Verify the bugs in the advisories (specified as arguments or in group.yml) for a release.
Requires a runtime to ensure that all bugs in the advisories match the runtime version.
Also ensures that bugs in the next release which block bugs in these advisories have
been verified, as those represent backports we do not want to regress in upgrades.
If any verification fails, a text explanation is given and the return code is 1.
Otherwise, prints the number of bugs in the advisories and exits with success.
"""
runtime.initialize()
advisories = advisories or [
a for a in runtime.group_config.get('advisories', {}).values()
]
if not advisories:
red_print("No advisories specified on command line or in group.yml")
exit(1)
if runtime.use_jira:
await verify_attached_bugs(runtime, verify_bug_status, advisories,
verify_flaws, True)
await verify_attached_bugs(runtime, verify_bug_status, advisories,
verify_flaws, False)
def _unstructured_output(bad_runs, rpmdiff_client):
for run in bad_runs:
attr = run["attributes"]
run_id = attr["external_id"]
run_url = "{}/run/{}/".format(constants.RPMDIFF_WEB_URL, run_id)
test_results = rpmdiff_client.get_test_results(run_id)
run_obj = rpmdiff_client.get_run(run_id)
print("----------------")
msg = "{0} {1}".format(run["relationships"]["brew_build"]["nvr"],
attr["status"])
if attr["status"] == "NEEDS_INSPECTION":
util.yellow_print(msg)
else:
util.red_print(msg)
for result in test_results:
score = result["score"]
if score >= 0 and score < 3: # good test result
continue
result_id = result["result_id"]
test = result["test"]
details = result["details"]
test_id = test["test_id"]
package_name = run_obj["package_name"]
result_url = run_url + str(test_id) + "/"
result_msg = "* TEST {0} {2} {1} {3}".format(
result_id, constants.RPMDIFF_SCORE_NAMES[score],
test["description"], result_url)
if score == 3: # NEEDS_INSPECTION
util.yellow_print(result_msg)
else:
util.red_print(result_msg)
# get last waiver message
waivers = rpmdiff_client.list_waivers(package_name,
test_id,
limit=1)
if waivers:
util.green_print(" Last waiver: @" +
waivers[0]["owner"]["username"] + ": " +
waivers[0]["description"])
else:
util.yellow_print(" No last waiver found.")
for detail in details:
detail_msg = " * {1} {0}".format(
constants.RPMDIFF_SCORE_NAMES[detail["score"]],
detail["subpackage"])
if detail["score"] == 3:
util.yellow_print(detail_msg)
else:
util.red_print(detail_msg)
content = re.sub('^',
' ',
detail["content"],
flags=re.MULTILINE)
print(content)
print()
async def verify_cvp_cli(runtime: Runtime, all_images, nvrs, optional_checks, all_optional_checks, fix, message):
""" Verify CVP test results
Example 1: Verify CVP test results for all latest 4.4 image builds, also warn those with failed content_set_check
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check
Example 2: Apply patches to ocp-build-data to fix the redundant content sets error:
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check --fix
Note:
1. If `--message` is not given, `--fix` will leave changed ocp-build-data files uncommitted.
2. Make sure your ocp-build-data directory is clean before running `--fix`.
"""
if bool(all_images) + bool(nvrs) != 1:
raise click.BadParameter('You must use one of --all or --build.')
if all_optional_checks and optional_checks:
raise click.BadParameter('Use only one of --all-optional-checks or --include-optional-check.')
runtime.initialize(mode='images')
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub or constants.BREW_HUB)
builds = []
if all_images:
image_metas = runtime.image_metas()
builds = await get_latest_image_builds(image_metas)
elif nvrs:
runtime.logger.info(f"Finding {len(builds)} builds from Brew...")
builds = brew.get_build_objects(nvrs, brew_session)
runtime.logger.info(f"Found {len(builds)} image builds.")
resultsdb_api = ResultsDBAPI()
nvrs = [b["nvr"] for b in builds]
runtime.logger.info(f"Getting CVP test results for {len(builds)} image builds...")
latest_cvp_results = await get_latest_cvp_results(runtime, resultsdb_api, nvrs)
# print a summary for all CVP results
good_results = [] # good means PASSED or INFO
bad_results = [] # bad means NEEDS_INSPECTION or FAILED
incomplete_nvrs = []
for nvr, result in zip(nvrs, latest_cvp_results):
if not result:
incomplete_nvrs.append(nvr)
continue
outcome = result.get("outcome") # only PASSED, FAILED, INFO, NEEDS_INSPECTION are now valid outcome values (https://resultsdb20.docs.apiary.io/#introduction/changes-since-1.0)
if outcome in {"PASSED", "INFO"}:
good_results.append(result)
elif outcome in {"NEEDS_INSPECTION", "FAILED"}:
bad_results.append(result)
green_prefix("good: {}".format(len(good_results)))
click.echo(", ", nl=False)
red_prefix("bad: {}".format(len(bad_results)))
click.echo(", ", nl=False)
yellow_print("incomplete: {}".format(len(incomplete_nvrs)))
if bad_results:
red_print("The following builds didn't pass CVP tests:")
for r in bad_results:
nvr = r["data"]["item"][0]
red_print(f"{nvr} {r['outcome']}: {r['ref_url']}")
if incomplete_nvrs:
yellow_print("We couldn't find CVP test results for the following builds:")
for nvr in incomplete_nvrs:
yellow_print(nvr)
if not optional_checks and not all_optional_checks:
return # no need to print failed optional CVP checks
# Find failed optional CVP checks in case some of the tiem *will* become required.
optional_checks = set(optional_checks)
complete_results = good_results + bad_results
runtime.logger.info(f"Getting optional checks for {len(complete_results)} CVP tests...")
optional_check_results = await get_optional_checks(runtime, complete_results)
component_distgit_keys = {} # a dict of brew component names to distgit keys
content_set_to_repo_names = {} # a map of content set names to group.yml repo names
for image in runtime.image_metas():
component_distgit_keys[image.get_component_name()] = image.distgit_key
for repo_name, repo_info in runtime.group_config.get("repos", {}).items():
for arch, cs_name in repo_info.get('content_set', {}).items():
if arch == "optional":
continue # not a real arch name
content_set_to_repo_names[cs_name] = repo_name
nvr_to_builds = {build["nvr"]: build for build in builds}
ocp_build_data_updated = False
failed_with_not_covered_rpms = set()
failed_with_redundant_repos = set()
only_failed_in_non_x86_with_not_covered_rpms = set()
only_failed_in_non_x86_with_redundant_repos = set()
for cvp_result, checks in zip(complete_results, optional_check_results):
# example optional checks: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/hive-container-v4.6.0-202008010302.p0/da01e36c-8c69-4a19-be7d-ba4593a7b085/sanity-tests-optional-results.json
bad_checks = [check for check in checks["checks"] if check["status"] != "PASS" and (all_optional_checks or check["name"] in optional_checks)]
if not bad_checks:
continue
nvr = cvp_result["data"]["item"][0]
build = nvr_to_builds[nvr]
yellow_print("----------")
yellow_print(f"Build {nvr} (https://brewweb.engineering.redhat.com/brew/buildinfo?buildID={nvr_to_builds[nvr]['id']}) has {len(bad_checks)} problematic CVP optional checks:")
for check in bad_checks:
yellow_print(f"* {check['name']} {check['status']}")
try:
amd64_result = list(filter(lambda item: item.get("arch") == "amd64", check["logs"][-1]))
except AttributeError:
red_print("CVP result malformed.")
if len(amd64_result) != 1:
red_print("WHAT?! This build doesn't include an amd64 image? This shouldn't happen. Check Brew and CVP logs with the CVP team!")
continue
amd64_result = amd64_result[0]
image_component_name = nvr.rsplit('-', 2)[0]
distgit_key = component_distgit_keys.get(image_component_name)
amd64_redundant_cs = amd64_result.get("redundant_cs", [])
amd64_redundant_repos = {content_set_to_repo_names[cs] for cs in amd64_redundant_cs}
def _strip_arch_suffix(rpm):
# rh-nodejs10-3.2-3.el7.x86_64 -> rh-nodejs10-3.2-3.el7
rpm_split = rpm.rsplit(".", 1)
return rpm_split[0]
amd64_not_covered_rpms = {_strip_arch_suffix(rpm) for rpm in amd64_result.get("not_covered_rpms", [])}
if check["name"] == "content_set_check":
details = check["logs"][-1] # example: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/logging-fluentd-container-v4.6.0-202008261251.p0/dd9f2024-5440-4f33-b508-472ccf258439/sanity-tests-optional-results.json
if not details:
red_print("content_set_check failed without any explanation. Report to CVP team!")
continue
if len(details) > 1: # if this build is multi-arch, check if all per-arch results are consistent
for result in details:
if result["arch"] == "amd64":
continue
redundant_repos = {content_set_to_repo_names[cs] for cs in result.get("redundant_cs", [])}
if redundant_repos != amd64_redundant_repos:
only_failed_in_non_x86_with_redundant_repos.add(nvr)
red_print(f"""content_set_check for {nvr} arch {result["arch"]} has different redundant_cs result from the one for amd64:
{result["arch"]} has redundant_cs {result.get("redundant_cs")},
but amd64 has redundant_cs {amd64_redundant_cs}.
Not sure what happened. Please see Brew and CVP logs and/or check with the CVP team.""")
not_covered_rpms = {_strip_arch_suffix(rpm) for rpm in result.get("not_covered_rpms", [])}
if not_covered_rpms != amd64_not_covered_rpms:
only_failed_in_non_x86_with_not_covered_rpms.add(nvr)
red_print(f"""content_set_check for {nvr} arch {result["arch"]} has different not_covered_rpms result from the one for amd64:
{result["arch"]} has extra not_covered_rpms {not_covered_rpms - amd64_not_covered_rpms},
and missing not_covered_rpms {amd64_not_covered_rpms - not_covered_rpms}.
Not sure what happened. Check Brew and CVP logs with the CVP team!""")
if amd64_not_covered_rpms: # This build has not_covered_rpms
failed_with_not_covered_rpms.add(nvr)
yellow_print(f"Image {distgit_key} has not_covered_rpms: {amd64_not_covered_rpms}")
brew_repos = await find_repos_for_rpms(amd64_not_covered_rpms, build)
yellow_print(f"Those repos shown in Brew logs might be a good hint: {brew_repos}")
runtime.logger.info("Looking for parent image's content_sets...")
parent = get_parent_build_ids([build])[0]
if parent:
parent_build = brew.get_build_objects([parent])[0]
parent_cs = await get_content_sets_for_build(parent_build)
parent_enabled_repos = {content_set_to_repo_names[cs] for cs in parent_cs.get("x86_64", [])}
enabled_repos = set(runtime.image_map[distgit_key].config.get("enabled_repos", []))
missing_repos = parent_enabled_repos - enabled_repos
yellow_print(f"""The following repos are defined in parent {parent_build["nvr"]} {component_distgit_keys.get(parent_build["name"], "?")}.yml but not in
{component_distgit_keys[build["name"]]}.yml: {missing_repos}""")
if fix and missing_repos:
runtime.logger.info("Trying to merge parent image's content_sets...")
fix_missing_content_set(runtime, distgit_key, missing_repos)
ocp_build_data_updated = True
runtime.logger.info(f"{distgit_key}.yml patched")
if amd64_redundant_repos: # This build has redundant_cs
failed_with_redundant_repos.add(nvr)
yellow_print(f"Image {distgit_key} has redundant repos: {amd64_redundant_repos}")
if not fix:
yellow_print(f"Please add the following repos to non_shipping_repos in {distgit_key}.yml: {amd64_redundant_repos}")
else:
runtime.logger.info(f"Applying redundant content sets fix to {distgit_key}.yml...")
fix_redundant_content_set(runtime, distgit_key, amd64_redundant_repos)
ocp_build_data_updated = True
runtime.logger.info(f"{distgit_key}.yml patched")
print(f"See {cvp_result['ref_url']}sanity-tests-optional-results.json for more details.")
if failed_with_not_covered_rpms or failed_with_redundant_repos:
yellow_print(f"{len(failed_with_not_covered_rpms | failed_with_redundant_repos)} images failed content_sets.\n Where")
if failed_with_not_covered_rpms:
yellow_print(f"\t{len(failed_with_not_covered_rpms)} images failed content_sets check because of not_covered_rpms:")
for rpm in failed_with_not_covered_rpms:
line = f"\t\t{rpm}"
if rpm in only_failed_in_non_x86_with_not_covered_rpms:
line += " - non-x86 arches are different from x86 one"
yellow_print(line)
if failed_with_redundant_repos:
yellow_print(f"\t{len(failed_with_redundant_repos)} images failed content_sets check because of redundant_repos:")
for rpm in failed_with_redundant_repos:
line = f"\t\t{rpm}"
if rpm in only_failed_in_non_x86_with_redundant_repos:
line += " - non-x86 arches are different from x86 one"
yellow_print(line)
if message and ocp_build_data_updated:
runtime.gitdata.commit(message)
def create(ctx, advisories, out_dir, out_layout, components, force):
""" Create tarball sources for advisories.
To create tarball sources for Brew component (package) logging-fluentd-container that was shipped on advisories 45606, 45527, and 46049:
$ elliott tarball-sources create --component logging-fluentd-container --out-dir=out/ 45606 45527 46049
"""
if not force and os.path.isdir(out_dir) and os.listdir(out_dir):
util.red_print("Output directory {} is not empty.\n\
Use --force to add new tarball sources to an existing directory.".format(
os.path.abspath(out_dir)))
exit(1)
mkdirs(out_dir)
working_dir = os.path.join(ctx.obj.working_dir, "tarball-sources")
LOGGER.debug("Use working directory {}.".format(
os.path.abspath(working_dir)))
mkdirs(working_dir)
# `nvr_dirs` is a dict with brew build NVRs as keys, values are
# a set of directories for the generated tarballs,
# since a build can be attached to multiple advisories.
# For example:
# nvr_dirs = {
# "logging-fluentd-container-v3.11.141-2": {
# "RHOSE/RHEL-7-OSE-3.11/45606/release/"
# },
# "logging-fluentd-container-v4.1.14-201908291507": {
# "RHOSE/RHEL-7-OSE-4.1/45527/release/"
# },
# "logging-fluentd-container-v4.1.15-201909041605": {
# "RHOSE/RHEL-7-OSE-4.1/46049/release/"
# }
# }
nvr_dirs = {} # type: Dict[str, Set[str]]
# Getting build NVRs for specified Koji/Brew components from advisories
# NOTE This is SLOW. However doing this in parallel doesn't work
# due to a race condition existing in the implementation of `errata_tool.Erratum`'s parant class ErrataConnector.
for advisory in advisories:
click.echo("Finding builds from advisory {}...".format(advisory))
builds = tarball_sources.find_builds_from_advisory(
advisory, components)
if not builds:
util.yellow_print(
"No matched builds found from advisory {}. Wrong advisory number?"
.format(advisory))
continue
util.green_print("Found {} matched build(s) from advisory {}".format(
len(builds), advisory))
for nvr, product, product_version in builds:
util.green_print("\t{}\t{}\t{}".format(nvr, product,
product_version))
for nvr, product, product_version in builds:
if nvr not in nvr_dirs:
nvr_dirs[nvr] = set()
if out_layout == "flat":
nvr_dirs[nvr].add(out_dir)
else:
nvr_dirs[nvr].add(
os.path.join(out_dir, product_version, str(advisory),
"release"))
if not nvr_dirs:
util.red_print(
"Exiting because no matched builds from all specified advisories.")
exit(1)
# Check build infos from Koji/Brew
# in order to figure out the source Git repo and commit hash for each build.
click.echo("Fetching build infos for {} from Koji/Brew...".format(
", ".join(nvr_dirs.keys())))
brew_session = koji.ClientSession(constants.BREW_HUB)
brew_builds = brew.get_build_objects(nvr_dirs.keys(), brew_session)
# Ready to generate tarballs
tarball_sources_list = []
for build_info in brew_builds:
nvr = build_info["nvr"]
tarball_filename = nvr + ".tar.gz"
click.echo("Generating tarball source {} for {}...".format(
tarball_filename, nvr))
with tempfile.NamedTemporaryFile(suffix="-" + tarball_filename,
dir=working_dir) as temp_tarball:
temp_tarball_path = temp_tarball.name
LOGGER.debug(
"Temporary tarball file is {}".format(temp_tarball_path))
tarball_sources.generate_tarball_source(
temp_tarball, nvr + "/",
os.path.join(working_dir, "repos", build_info["name"]),
build_info["source"])
for dest_dir in nvr_dirs[nvr]:
mkdirs(dest_dir)
tarball_abspath = os.path.abspath(
os.path.join(dest_dir, tarball_filename))
if os.path.exists(tarball_abspath):
util.yellow_print(
"File {} will be overwritten.".format(tarball_abspath))
LOGGER.debug("Copying {} to {}...".format(
temp_tarball_path, tarball_abspath))
shutil.copyfile(
temp_tarball_path,
tarball_abspath) # `shutil.copyfile` uses default umask
tarball_sources_list.append(tarball_abspath)
util.green_print(
"Created tarball source {}.".format(tarball_abspath))
print_success_message(tarball_sources_list, out_dir)
def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, builds,
kind, from_diff, as_json, allow_attached, remove, clean,
no_cdn_repos, payload, non_payload):
'''Automatically or manually find or attach/remove viable rpm or image builds
to ADVISORY. Default behavior searches Brew for viable builds in the
given group. Provide builds manually by giving one or more --build
(-b) options. Manually provided builds are verified against the Errata
Tool API.
\b
* Attach the builds to ADVISORY by giving --attach
* Remove the builds to ADVISORY by giving --remove
* Specify the build type using --kind KIND
Example: Assuming --group=openshift-3.7, then a build is a VIABLE
BUILD IFF it meets ALL of the following criteria:
\b
* HAS the tag in brew: rhaos-3.7-rhel7-candidate
* DOES NOT have the tag in brew: rhaos-3.7-rhel7
* IS NOT attached to ANY existing RHBA, RHSA, or RHEA
That is to say, a viable build is tagged as a "candidate", has NOT
received the "shipped" tag yet, and is NOT attached to any PAST or
PRESENT advisory. Here are some examples:
SHOW the latest OSE 3.6 image builds that would be attached to a
3.6 advisory:
$ elliott --group openshift-3.6 find-builds -k image
ATTACH the latest OSE 3.6 rpm builds to advisory 123456:
\b
$ elliott --group openshift-3.6 find-builds -k rpm --attach 123456
VERIFY (no --attach) that the manually provided RPM NVR and build
ID are viable builds:
$ elliott --group openshift-3.6 find-builds -k rpm -b megafrobber-1.0.1-2.el7 -a 93170
\b
Remove specific RPM NVR and build ID from advisory:
$ elliott --group openshift-4.3 find-builds -k image -b oauth-server-container-v4.3.22-202005212137 -a 55017 --remove
'''
if from_diff and builds:
raise click.BadParameter(
'Use only one of --build or --from-diff/--between.')
if clean and (remove or from_diff or builds):
raise click.BadParameter(
'Option --clean cannot be used with --build or --from-diff/--between.'
)
if not builds and remove:
raise click.BadParameter(
'Option --remove only support removing specific build with -b.')
if from_diff and kind != "image":
raise click.BadParameter(
'Option --from-diff/--between should be used with --kind/-k image.'
)
if advisory and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --attach')
if payload and non_payload:
raise click.BadParameter('Use only one of --payload or --non-payload.')
runtime.initialize(mode='images' if kind == 'image' else 'rpms')
replace_vars = runtime.group_config.vars.primitive(
) if runtime.group_config.vars else {}
et_data = runtime.gitdata.load_data(key='erratatool',
replace_vars=replace_vars).data
tag_pv_map = et_data.get('brew_tag_product_version_mapping')
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
ensure_erratatool_auth(
) # before we waste time looking up builds we can't process
unshipped_nvrps = []
unshipped_builds = []
to_remove = []
# get the builds we want to add
brew_session = runtime.build_retrying_koji_client(caching=True)
if builds:
green_prefix('Fetching builds...')
unshipped_nvrps = _fetch_nvrps_by_nvr_or_id(
builds,
tag_pv_map,
ignore_product_version=remove,
brew_session=brew_session)
elif clean:
unshipped_builds = errata.get_brew_builds(advisory)
elif from_diff:
unshipped_nvrps = _fetch_builds_from_diff(from_diff[0], from_diff[1],
tag_pv_map)
else:
if kind == 'image':
unshipped_nvrps = _fetch_builds_by_kind_image(
runtime, tag_pv_map, brew_session, payload, non_payload)
elif kind == 'rpm':
unshipped_nvrps = _fetch_builds_by_kind_rpm(
runtime, tag_pv_map, brew_session)
pbar_header('Fetching builds from Errata: ',
'Hold on a moment, fetching buildinfos from Errata Tool...',
unshipped_builds if clean else unshipped_nvrps)
if not clean and not remove:
# if is --clean then batch fetch from Erratum no need to fetch them individually
# if is not for --clean fetch individually using nvrp tuples then get specific
# elliottlib.brew.Build Objects by get_brew_build()
# e.g. :
# ('atomic-openshift-descheduler-container', 'v4.3.23', '202005250821', 'RHEL-7-OSE-4.3').
# Build(atomic-openshift-descheduler-container-v4.3.23-202005250821).
unshipped_builds = parallel_results_with_progress(
unshipped_nvrps, lambda nvrp: elliottlib.errata.get_brew_build(
'{}-{}-{}'.format(nvrp[0], nvrp[1], nvrp[2]),
nvrp[3],
session=requests.Session()))
if not allow_attached:
unshipped_builds = _filter_out_inviable_builds(
kind, unshipped_builds, elliottlib.errata)
_json_dump(as_json, unshipped_builds, kind, tag_pv_map)
if not unshipped_builds:
green_print('No builds needed to be attached.')
return
if not advisory:
click.echo('The following {n} builds '.format(n=len(unshipped_builds)),
nl=False)
if not (remove or clean):
click.secho('may be attached', bold=True, nl=False)
click.echo(' to an advisory:')
else:
click.secho('may be removed from', bold=True, nl=False)
click.echo(' from an advisory:')
for b in sorted(unshipped_builds):
click.echo(' ' + b.nvr)
return
if not unshipped_builds and not (remove and unshipped_nvrps):
# Do not change advisory state unless strictly necessary
return
try:
erratum = elliottlib.errata.Advisory(errata_id=advisory)
erratum.ensure_state('NEW_FILES')
if remove:
to_remove = [
f"{nvrp[0]}-{nvrp[1]}-{nvrp[2]}" for nvrp in unshipped_nvrps
]
elif clean:
to_remove = [b.nvr for b in unshipped_builds]
if to_remove:
erratum.remove_builds(to_remove)
else: # attach
erratum.attach_builds(unshipped_builds, kind)
cdn_repos = et_data.get('cdn_repos')
if cdn_repos and not no_cdn_repos and kind == "image":
erratum.set_cdn_repos(cdn_repos)
except GSSError:
exit_unauthenticated()
except ErrataException as e:
red_print(f'Cannot change advisory {advisory}: {e}')
exit(1)
def tag_builds_cli(runtime: Runtime, advisories: Tuple[int],
default_advisory_type: str, product_version: str,
builds: Tuple[str], tag: str, dont_untag: bool,
dry_run: bool):
""" Tag builds into Brew tag and optionally untag unspecified builds.
Example 1: Tag RHEL7 RPMs that on ocp-build-data recorded advisory into rhaos-4.3-rhel-7-image-build
$ elliott --group=openshift-4.3 tag-builds --use-default-advisory rpm --product-version RHEL-7-OSE-4.3 --tag rhaos-4.3-rhel-7-image-build
Example 2: Tag RHEL8 RPMs that are on advisory 55016 into rhaos-4.3-rhel-8-image-build
$ elliott --group=openshift-4.3 tag-builds --advisory 55016 --product-version OSE-4.4-RHEL-8 --tag rhaos-4.3-rhel-8-image-build
Example 3: Tag specified builds into rhaos-4.3-rhel-8-image-build
$ elliott --group=openshift-4.3 tag-builds --build buildah-1.11.6-6.rhaos4.3.el8 --build openshift-4.3.23-202005230952.g1.b596217.el8 --tag rhaos-4.3-rhel-8-image-build
"""
if advisories and builds:
raise click.BadParameter('Use only one of --build or --advisory/-a.')
if advisories and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --advisory/-a.')
if default_advisory_type and builds:
raise click.BadParameter(
'Use only one of --build or --use-default-advisory.')
if product_version and not advisories and not default_advisory_type:
raise click.BadParameter(
'--product-version should only be used with --use-default-advisory or --advisory/-a.'
)
runtime.initialize()
logger = runtime.logger
if default_advisory_type:
advisories = (find_default_advisory(runtime, default_advisory_type), )
all_builds = set() # All Brew builds that should be in the tag
if advisories:
errata_session = requests.session()
for advisory in advisories:
logger.info(
f"Fetching attached Brew builds from advisory {advisory}...")
errata_builds = errata.get_builds(advisory, errata_session)
product_versions = list(errata_builds.keys())
logger.debug(
f"Advisory {advisory} has builds for {len(product_versions)} product versions: {product_versions}"
)
if product_version: # Only this product version should be concerned
product_versions = [product_version]
for pv in product_versions:
logger.debug(f"Extract Errata builds for product version {pv}")
nvrs = _extract_nvrs_from_errata_build_list(errata_builds, pv)
logger.info(
f"Found {len(nvrs)} builds from advisory {advisory} with product version {pv}"
)
logger.debug(
f"The following builds are found for product version {pv}:\n\t{list(nvrs)}"
)
all_builds |= set(nvrs)
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
if builds: # NVRs are directly specified with --build
build_objs = brew.get_build_objects(list(builds), brew_session)
all_builds = {build["nvr"] for build in build_objs}
click.echo(
f"The following {len(all_builds)} build(s) should be in tag {tag}:")
for nvr in all_builds:
green_print(f"\t{nvr}")
# get NVRs that have been tagged
tagged_build_objs = brew_session.listTagged(tag,
latest=False,
inherit=False)
tagged_builds = {build["nvr"] for build in tagged_build_objs}
# get NVRs that should be tagged
missing_builds = all_builds - tagged_builds
click.echo(f"{len(missing_builds)} build(s) need to be tagged into {tag}:")
for nvr in missing_builds:
green_print(f"\t{nvr}")
# get NVRs that should be untagged
extra_builds = tagged_builds - all_builds
click.echo(f"{len(extra_builds)} build(s) need to be untagged from {tag}:")
for nvr in extra_builds:
green_print(f"\t{nvr}")
if dry_run:
yellow_print("Dry run: Do nothing.")
return
brew_session.gssapi_login()
if not dont_untag:
# untag extra builds
extra_builds = list(extra_builds)
logger.info(f"Untagging {len(extra_builds)} build(s) from {tag}...")
multicall_tasks = brew.untag_builds(tag, extra_builds, brew_session)
failed_to_untag = []
for index, task in enumerate(multicall_tasks):
try:
task.result
click.echo(f"{nvr} has been successfully untagged from {tag}")
except Exception as ex:
nvr = extra_builds[index]
failed_to_untag.append(nvr)
logger.error(f"Failed to untag {nvr}: {ex}")
# tag missing builds
missing_builds = list(missing_builds)
task_id_nvr_map = {}
logger.info(f"Tagging {len(missing_builds)} build(s) into {tag}...")
multicall_tasks = brew.tag_builds(tag, missing_builds, brew_session)
failed_to_tag = []
for index, task in enumerate(multicall_tasks):
nvr = missing_builds[index]
try:
task_id = task.result
task_id_nvr_map[task_id] = nvr
except Exception as ex:
failed_to_tag.append(nvr)
logger.error(f"Failed to tag {nvr}: {ex}")
if task_id_nvr_map:
# wait for tag task to finish
logger.info("Waiting for tag tasks to finish")
brew.wait_tasks(task_id_nvr_map.keys(), brew_session, logger=logger)
# get tagging results
stopped_tasks = list(task_id_nvr_map.keys())
with brew_session.multicall(strict=False) as m:
multicall_tasks = []
for task_id in stopped_tasks:
multicall_tasks.append(
m.getTaskResult(task_id, raise_fault=False))
for index, t in enumerate(multicall_tasks):
task_id = stopped_tasks[index]
nvr = task_id_nvr_map[task_id]
tag_res = t.result
logger.debug(
f"Tagging task {task_id} {nvr} returned result {tag_res}")
click.echo(f"{nvr} has been successfully tagged into {tag}")
if tag_res and 'faultCode' in tag_res:
if "already tagged" not in tag_res["faultString"]:
failed_to_tag.append(nvr)
logger.error(
f'Failed to tag {nvr} into {tag}: {tag_res["faultString"]}'
)
if failed_to_untag:
red_print("The following builds were failed to untag:")
for nvr in failed_to_untag:
red_print(f"\t{nvr}")
elif not dont_untag:
green_print(
f"All unspecified builds have been successfully untagged from {tag}."
)
if failed_to_tag:
red_print("The following builds were failed to tag:")
for nvr in failed_to_tag:
red_print(f"\t{nvr}")
else:
green_print(f"All builds have been successfully tagged into {tag}.")
if failed_to_untag or failed_to_tag:
raise exceptions.ElliottFatalError(
"Not all builds were successfully tagged/untagged.")
def verify_cvp_cli(runtime: Runtime, all_images, nvrs, optional_checks,
all_optional_checks, fix, message):
""" Verify CVP test results
Example 1: Verify CVP test results for all latest 4.4 image builds, also warn those with failed content_set_check
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check
Example 2: Apply patches to ocp-build-data to fix the redundant content sets error:
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check --fix
Note:
1. If `--message` is not given, `--fix` will leave changed ocp-build-data files uncommitted.
2. Make sure your ocp-build-data directory is clean before running `--fix`.
"""
if bool(all_images) + bool(nvrs) != 1:
raise click.BadParameter('You must use one of --all or --build.')
if all_optional_checks and optional_checks:
raise click.BadParameter(
'Use only one of --all-optional-checks or --include-optional-check.'
)
runtime.initialize(mode='images')
tag_pv_map = runtime.gitdata.load_data(
key='erratatool',
replace_vars=runtime.group_config.vars.primitive()
if runtime.group_config.vars else
{}).data.get('brew_tag_product_version_mapping')
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
builds = []
if all_images:
runtime.logger.info("Getting latest image builds from Brew...")
builds = get_latest_image_builds(brew_session, tag_pv_map.keys(),
runtime.image_metas)
elif nvrs:
runtime.logger.info(f"Finding {len(builds)} builds from Brew...")
builds = brew.get_build_objects(nvrs, brew_session)
runtime.logger.info(f"Found {len(builds)} image builds.")
resultsdb_api = ResultsDBAPI()
nvrs = [b["nvr"] for b in builds]
runtime.logger.info(
f"Getting CVP test results for {len(builds)} image builds...")
latest_cvp_results = get_latest_cvp_results(runtime, resultsdb_api, nvrs)
# print a summary for all CVP results
good_results = [] # good means PASSED or INFO
bad_results = [] # bad means NEEDS_INSPECTION or FAILED
incomplete_nvrs = []
for nvr, result in zip(nvrs, latest_cvp_results):
if not result:
incomplete_nvrs.append(nvr)
continue
outcome = result.get(
"outcome"
) # only PASSED, FAILED, INFO, NEEDS_INSPECTION are now valid outcome values (https://resultsdb20.docs.apiary.io/#introduction/changes-since-1.0)
if outcome in {"PASSED", "INFO"}:
good_results.append(result)
elif outcome in {"NEEDS_INSPECTION", "FAILED"}:
bad_results.append(result)
green_prefix("good: {}".format(len(good_results)))
click.echo(", ", nl=False)
red_prefix("bad: {}".format(len(bad_results)))
click.echo(", ", nl=False)
yellow_print("incomplete: {}".format(len(incomplete_nvrs)))
if bad_results:
red_print("The following builds didn't pass CVP tests:")
for r in bad_results:
nvr = r["data"]["item"][0]
red_print(f"{nvr} {r['outcome']}: {r['ref_url']}")
if incomplete_nvrs:
yellow_print(
"We couldn't find CVP test results for the following builds:")
for nvr in incomplete_nvrs:
yellow_print(nvr)
if not optional_checks and not all_optional_checks:
return # no need to print failed optional CVP checks
# Find failed optional CVP checks in case some of the tiem *will* become required.
optional_checks = set(optional_checks)
complete_results = good_results + bad_results
runtime.logger.info(
f"Getting optional checks for {len(complete_results)} CVP tests...")
optional_check_results = get_optional_checks(runtime, complete_results)
component_distgit_keys = {
} # a dict of brew component names to distgit keys
content_set_repo_names = {
} # a map of x86_64 content set names to group.yml repo names
if fix: # Fixing redundant content sets requires those dicts
for image in runtime.image_metas():
component_distgit_keys[
image.get_component_name()] = image.distgit_key
for repo_name, repo_info in runtime.group_config.get("repos",
{}).items():
content_set_name = repo_info.get(
'content_set', {}).get('x86_64') or repo_info.get(
'content_set', {}).get('default')
if content_set_name:
content_set_repo_names[content_set_name] = repo_name
ocp_build_data_updated = False
for cvp_result, checks in zip(complete_results, optional_check_results):
# example optional checks: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/hive-container-v4.6.0-202008010302.p0/da01e36c-8c69-4a19-be7d-ba4593a7b085/sanity-tests-optional-results.json
bad_checks = [
check for check in checks["checks"]
if check["status"] != "PASS" and (
all_optional_checks or check["name"] in optional_checks)
]
if not bad_checks:
continue
nvr = cvp_result["data"]["item"][0]
yellow_print("----------")
yellow_print(
f"Build {nvr} has {len(bad_checks)} problematic CVP optional checks:"
)
for check in bad_checks:
yellow_print(f"* {check['name']} {check['status']}")
if fix and check["name"] == "content_set_check":
if "Some content sets are redundant." in check["logs"]:
# fix redundant content sets
name = nvr.rsplit('-', 2)[0]
distgit_keys = component_distgit_keys.get(name)
if not distgit_keys:
runtime.logger.warning(
f"Will not apply the redundant content sets fix to image {name}: We don't know its distgit key."
)
continue
amd64_content_sets = list(
filter(lambda item: item.get("arch") == "amd64",
check["logs"][-1])
) # seems only x86_64 (amd64) content sets are defined in ocp-build-data.
if not amd64_content_sets:
runtime.logger.warning(
f"Will not apply the redundant content sets fix to image {name}: It doesn't have redundant x86_64 (amd64) content sets"
)
continue
amd64_redundant_cs = amd64_content_sets[0]["redundant_cs"]
redundant_repos = [
content_set_repo_names[cs] for cs in amd64_redundant_cs
if cs in content_set_repo_names
]
if len(redundant_repos) != len(amd64_redundant_cs):
runtime.logger.error(
f"Not all content sets have a repo entry in group.yml: #content_sets is {len(amd64_redundant_cs)}, #repos is {len(redundant_repos)}"
)
runtime.logger.info(
f"Applying redundant content sets fix to {distgit_keys}..."
)
fix_redundant_content_set(runtime, distgit_keys,
redundant_repos)
ocp_build_data_updated = True
runtime.logger.info(
f"Fixed redundant content sets for {distgit_keys}")
yellow_print(
f"See {cvp_result['ref_url']}sanity-tests-optional-results.json for more details."
)
if message and ocp_build_data_updated:
runtime.gitdata.commit(message)
def _complain(self, problem):
red_print(problem)
self.problems.append(problem)
def show(ctx, advisory):
""" Show RPMDiff failures for an advisory.
"""
runtime = ctx.obj # type: Runtime
if not advisory:
runtime.initialize()
advisory = runtime.group_config.advisories.get("rpm", 0)
if not advisory:
raise ElliottFatalError(
"No RPM advisory number configured in ocp-build-data.")
else:
runtime.initialize(no_group=True)
logger = runtime.logger
logger.info(
"Fetching RPMDiff runs from Errata Tool for advisory {}...".format(
advisory))
rpmdiff_runs = list(errata.get_rpmdiff_runs(advisory))
logger.info("Found {} RPMDiff runs.".format(len(rpmdiff_runs)))
# "good" means PASSED, INFO, or WAIVED
good_runs = []
# "bad" means NEEDS_INSPECTION or FAILED
bad_runs = []
incomplete_runs = []
for rpmdiff_run in rpmdiff_runs:
attr = rpmdiff_run['attributes']
if attr["status"] in constants.ET_GOOD_EXTERNAL_TEST_STATUSES:
good_runs.append(rpmdiff_run)
elif attr["status"] in constants.ET_BAD_EXTERNAL_TEST_STATUSES:
bad_runs.append(rpmdiff_run)
else:
incomplete_runs.append(rpmdiff_run)
util.green_prefix("good: {}".format(len(good_runs)))
click.echo(", ", nl=False)
util.red_prefix("bad:{}".format(len(bad_runs)))
click.echo(", ", nl=False)
util.yellow_print("incomplete: {}".format(len(incomplete_runs)))
if not bad_runs:
return
logger.info(
"Fetching detailed information from RPMDiff for bad RPMDiff runs...")
rpmdiff_client = RPMDiffClient(constants.RPMDIFF_HUB_URL)
rpmdiff_client.authenticate()
for run in bad_runs:
attr = run["attributes"]
run_id = attr["external_id"]
run_url = "{}/run/{}/".format(constants.RPMDIFF_WEB_URL, run_id)
print("----------------")
msg = "{0} {1}".format(run["relationships"]["brew_build"]["nvr"],
attr["status"])
if attr["status"] == "NEEDS_INSPECTION":
util.yellow_print(msg)
else:
util.red_print(msg)
test_results = rpmdiff_client.get_test_results(run_id)
run_obj = rpmdiff_client.get_run(run_id)
for result in test_results:
score = result["score"]
if score >= 0 and score < 3: # good test result
continue
result_id = result["result_id"]
test = result["test"]
details = result["details"]
test_id = test["test_id"]
package_name = run_obj["package_name"]
result_url = run_url + str(test_id) + "/"
result_msg = "* TEST {0} {2} {1} {3}".format(
result_id, constants.RPMDIFF_SCORE_NAMES[score],
test["description"], result_url)
if score == 3: # NEEDS_INSPECTION
util.yellow_print(result_msg)
else:
util.red_print(result_msg)
# get last waiver message
waivers = rpmdiff_client.list_waivers(package_name,
test_id,
limit=1)
if waivers:
util.green_print(" Last waiver: @" +
waivers[0]["owner"]["username"] + ": " +
waivers[0]["description"])
else:
util.yellow_print(" No last waiver found.")
for detail in details:
detail_msg = " * {1} {0}".format(
constants.RPMDIFF_SCORE_NAMES[detail["score"]],
detail["subpackage"])
if detail["score"] == 3:
util.yellow_print(detail_msg)
else:
util.red_print(detail_msg)
content = re.sub('^',
' ',
detail["content"],
flags=re.MULTILINE)
print(content)
print()
async def verify_payload(ctx, payload, advisory):
"""Cross-check that the builds present in PAYLOAD match the builds
attached to ADVISORY. The payload is treated as the source of
truth. If something is absent or different in the advisory it is
treated as an error with the advisory.
\b
PAYLOAD - Full pullspec of the payload to verify
ADVISORY - Numerical ID of the advisory
Two checks are made:
\b
1. Missing in Advisory - No payload components are absent from the given advisory
2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory
Results are summarily printed at the end of the run. They are also
written out to summary_results.json.
Verify builds in the given payload match the builds attached to
advisory 41567
\b
$ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567
"""
try:
green_prefix("Fetching advisory builds: ")
click.echo("Advisory - {}".format(advisory))
builds = elliottlib.errata.get_builds(advisory)
except GSSError:
exit_unauthenticated()
except elliottlib.exceptions.ErrataToolError as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
all_advisory_nvrs = {}
# Results come back with top level keys which are brew tags
green_prefix("Looping over tags: ")
click.echo("{} tags to check".format(len(builds)))
for tag in builds.keys():
# Each top level has a key 'builds' which is a list of dicts
green_prefix("Looping over builds in tag: ")
click.echo("{} with {} builds".format(tag, len(builds[tag]['builds'])))
for build in builds[tag]['builds']:
# Each dict has a top level key which might be the actual
# 'nvr' but I don't have enough data to know for sure
# yet. Also I don't know when there might be more than one
# key in the build dict. We'll loop over it to be sure.
for name in build.keys():
n, v, r = name.rsplit('-', 2)
version_release = "{}-{}".format(v, r)
all_advisory_nvrs[n] = version_release
click.echo("Found {} builds".format(len(all_advisory_nvrs)))
all_payload_nvrs = {}
click.echo("Fetching release info")
release_export_cmd = 'oc adm release info {} -o json'.format(payload)
rc, stdout, stderr = exectools.cmd_gather(release_export_cmd)
if rc != 0:
# Probably no point in continuing.. can't contact brew?
print("Unable to run oc release info: out={} ; err={}".format(
stdout, stderr))
exit(1)
else:
click.echo("Got release info")
payload_json = json.loads(stdout)
green_prefix("Looping over payload images: ")
click.echo("{} images to check".format(
len(payload_json['references']['spec']['tags'])))
cmds = [['oc', 'image', 'info', '-o', 'json', tag['from']['name']]
for tag in payload_json['references']['spec']['tags']]
green_prefix("Querying image infos...")
cmd_results = await asyncio.gather(
*[exectools.cmd_gather_async(cmd) for cmd in cmds])
for image, cmd, cmd_result in zip(
payload_json['references']['spec']['tags'], cmds, cmd_results):
click.echo("----")
image_name = image['name']
rc, stdout, stderr = cmd_result
if rc != 0:
# Probably no point in continuing.. can't contact brew?
red_prefix("Unable to run oc image info: ")
red_print(f"cmd={cmd!r}, out={stdout} ; err={stderr}")
exit(1)
image_info = json.loads(stdout)
labels = image_info['config']['config']['Labels']
# The machine-os-content image doesn't follow the standard
# pattern. We need to skip that image when we find it, it is
# not attached to advisories.
if 'com.coreos.ostree-commit' in labels:
yellow_prefix("Skipping machine-os-content image: ")
click.echo("Not required for checks")
continue
component = labels['com.redhat.component']
n = image_name
click.echo("Payload name: {}".format(n))
click.echo("Brew name: {}".format(component))
if labels:
v = labels['version']
r = labels['release']
all_payload_nvrs[component] = "{}-{}".format(v, r)
else:
print("For image {} Labels doesn't exist, image_info: {}".format(
image_name, image_info))
missing_in_errata = {}
payload_doesnt_match_errata = {}
in_other_advisories = {}
output = {
'missing_in_advisory': missing_in_errata,
'payload_advisory_mismatch': payload_doesnt_match_errata,
"in_other_advisories": in_other_advisories,
}
green_prefix("Analyzing data: ")
click.echo("{} images to consider from payload".format(
len(all_payload_nvrs)))
for image, vr in all_payload_nvrs.items():
yellow_prefix("Cross-checking from payload: ")
click.echo("{}-{}".format(image, vr))
if image not in all_advisory_nvrs:
missing_in_errata[image] = "{}-{}".format(image, vr)
click.echo("{} in payload not found in advisory".format(
"{}-{}".format(image, vr)))
elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]:
click.echo(
"{} from payload has version {} which does not match {} from advisory"
.format(image, vr, all_advisory_nvrs[image]))
payload_doesnt_match_errata[image] = {
'payload': vr,
'errata': all_advisory_nvrs[image]
}
if missing_in_errata:
green_print(
f"Checking if {len(missing_in_errata)} missing images are shipped..."
)
nvrs = list(missing_in_errata.values())
tag_lists = elliottlib.brew.get_builds_tags(nvrs)
for nvr, tags in zip(nvrs, tag_lists):
name = nvr.rsplit("-", 2)[0]
if any(map(lambda tag: tag["name"].endswith('-released'), tags)):
green_print(f"Build {nvr} is shipped. Skipping...")
del missing_in_errata[name]
elif any(map(lambda tag: tag["name"].endswith('-pending'), tags)):
green_print(f"Build {nvr} is in another advisory.")
del missing_in_errata[name]
in_other_advisories[name] = nvr
green_print("Summary results:")
click.echo(json.dumps(output, indent=4))
with open('summary_results.json', 'w') as fp:
json.dump(output, fp, indent=4)
green_prefix("Wrote out summary results: ")
click.echo("summary_results.json")
async def verify_payload(ctx, payload, advisory):
"""Cross-check that the builds present in PAYLOAD match the builds
attached to ADVISORY. The payload is treated as the source of
truth. If something is absent or different in the advisory it is
treated as an error with the advisory.
\b
PAYLOAD - Full pullspec of the payload to verify
ADVISORY - Numerical ID of the advisory
Two checks are made:
\b
1. Missing in Advisory - No payload components are absent from the given advisory
2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory
Results are summarily printed at the end of the run. They are also
written out to summary_results.json.
Verify builds in the given payload match the builds attached to
advisory 41567
\b
$ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567
"""
all_advisory_nvrs = elliottlib.errata.get_advisory_nvrs(advisory)
click.echo("Found {} builds".format(len(all_advisory_nvrs)))
all_payload_nvrs = {}
click.echo("Fetching release info")
release_export_cmd = 'oc adm release info {} -o json'.format(payload)
rc, stdout, stderr = exectools.cmd_gather(release_export_cmd)
if rc != 0:
# Probably no point in continuing.. can't contact brew?
print("Unable to run oc release info: out={} ; err={}".format(
stdout, stderr))
exit(1)
else:
click.echo("Got release info")
payload_json = json.loads(stdout)
green_prefix("Looping over payload images: ")
click.echo("{} images to check".format(
len(payload_json['references']['spec']['tags'])))
cmds = [['oc', 'image', 'info', '-o', 'json', tag['from']['name']]
for tag in payload_json['references']['spec']['tags']]
green_prefix("Querying image infos...")
cmd_results = await asyncio.gather(
*[exectools.cmd_gather_async(cmd) for cmd in cmds])
for image, cmd, cmd_result in zip(
payload_json['references']['spec']['tags'], cmds, cmd_results):
click.echo("----")
image_name = image['name']
rc, stdout, stderr = cmd_result
if rc != 0:
# Probably no point in continuing.. can't contact brew?
red_prefix("Unable to run oc image info: ")
red_print(f"cmd={cmd!r}, out={stdout} ; err={stderr}")
exit(1)
image_info = json.loads(stdout)
labels = image_info['config']['config']['Labels']
# The machine-os-content image doesn't follow the standard
# pattern. We need to skip that image when we find it, it is
# not attached to advisories.
if 'com.coreos.ostree-commit' in labels:
yellow_prefix("Skipping machine-os-content image: ")
click.echo("Not required for checks")
continue
component = labels['com.redhat.component']
n = image_name
click.echo("Payload name: {}".format(n))
click.echo("Brew name: {}".format(component))
if labels:
v = labels['version']
r = labels['release']
all_payload_nvrs[component] = "{}-{}".format(v, r)
else:
print("For image {} Labels doesn't exist, image_info: {}".format(
image_name, image_info))
missing_in_errata = {}
payload_doesnt_match_errata = {}
in_pending_advisory = []
in_shipped_advisory = []
output = {
'missing_in_advisory': missing_in_errata,
'payload_advisory_mismatch': payload_doesnt_match_errata,
"in_pending_advisory": in_pending_advisory,
"in_shipped_advisory": in_shipped_advisory,
}
green_prefix("Analyzing data: ")
click.echo("{} images to consider from payload".format(
len(all_payload_nvrs)))
for image, vr in all_payload_nvrs.items():
yellow_prefix("Cross-checking from payload: ")
click.echo("{}-{}".format(image, vr))
if image not in all_advisory_nvrs:
missing_in_errata[image] = "{}-{}".format(image, vr)
click.echo("{} in payload not found in advisory".format(
"{}-{}".format(image, vr)))
elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]:
click.echo(
"{} from payload has version {} which does not match {} from advisory"
.format(image, vr, all_advisory_nvrs[image]))
payload_doesnt_match_errata[image] = {
'payload': vr,
'errata': all_advisory_nvrs[image]
}
if missing_in_errata: # check if missing images are already shipped or pending to ship
advisory_nvrs: Dict[int, List[str]] = {
} # a dict mapping advisory numbers to lists of NVRs
green_print(
f"Checking if {len(missing_in_errata)} missing images are shipped..."
)
for nvr in missing_in_errata.copy().values():
# get the list of advisories that this build has been attached to
build = elliottlib.errata.get_brew_build(nvr)
# filter out dropped advisories
advisories = [
ad for ad in build.all_errata
if ad["status"] != "DROPPED_NO_SHIP"
]
if not advisories:
red_print(f"Build {nvr} is not attached to any advisories.")
continue
for advisory in advisories:
if advisory["status"] == "SHIPPED_LIVE":
green_print(
f"Missing build {nvr} has been shipped with advisory {advisory}."
)
else:
yellow_print(
f"Missing build {nvr} is in another pending advisory.")
advisory_nvrs.setdefault(advisory["id"], []).append(nvr)
name = nvr.rsplit("-", 2)[0]
del missing_in_errata[name]
if advisory_nvrs:
click.echo(
f"Getting information of {len(advisory_nvrs)} advisories...")
for advisory, nvrs in advisory_nvrs.items():
advisory_obj = elliottlib.errata.get_raw_erratum(advisory)
adv_type, adv_info = next(iter(advisory_obj["errata"].items()))
item = {
"id": advisory,
"type": adv_type.upper(),
"url": elliottlib.constants.errata_url + f"/{advisory}",
"summary": adv_info["synopsis"],
"state": adv_info["status"],
"nvrs": nvrs,
}
if adv_info["status"] == "SHIPPED_LIVE":
in_shipped_advisory.append(item)
else:
in_pending_advisory.append(item)
green_print("Summary results:")
click.echo(json.dumps(output, indent=4))
with open('summary_results.json', 'w') as fp:
json.dump(output, fp, indent=4)
green_prefix("Wrote out summary results: ")
click.echo("summary_results.json")
