def find_bugs(runtime, advisory, default_advisory_type, mode, status, id, from_diff, flag, report):
"""Find Red Hat Bugzilla bugs or add them to ADVISORY. Bugs can be
"swept" into the advisory either automatically (--mode sweep), or by
manually specifying one or more bugs using --mode list and the --id option.
Use cases are described below:
Note: Using --id without --add is basically pointless
SWEEP: For this use-case the --group option MUST be provided. The
--group automatically determines the correct target-releases to search
for MODIFIED bugs in.
LIST: The --group option is not required if you are specifying bugs
manually. Provide one or more --id's for manual bug addition. In LIST
mode you must provide a list of IDs to attach with the --id option.
DIFF: For this use case, you must provide the --between option using two
URLs to payloads.
Using --use-default-advisory without a value set for the matching key
in the build-data will cause an error and elliott will exit in a
non-zero state. Use of this option silently overrides providing an
advisory with the --add option.
Automatically add bugs with target-release matching 3.7.Z or 3.7.0
to advisory 123456:
\b
$ elliott --group openshift-3.7 find-bugs --mode sweep --add 123456
List bugs that WOULD be added to an advisory and have set the bro_ok flag on them (NOOP):
\b
$ elliott --group openshift-3.7 find-bugs --mode sweep --flag bro_ok
Add two bugs to advisory 123456. Note that --group is not required
because we're not auto searching:
\b
$ elliott find-bugs --mode list --id 8675309 --id 7001337 --add 123456
Automatically find bugs for openshift-4.1 and attach them to the
rpm advisory defined in ocp-build-data:
\b
$ elliott --group=openshift-4.1 --mode sweep --use-default-advisory rpm
"""
if mode != 'list' and len(id) > 0:
raise click.BadParameter("Combining the automatic and manual bug attachment options is not supported")
if mode == 'list' and len(id) == 0:
raise click.BadParameter("When using mode=list, you must provide a list of bug IDs")
if mode == 'payload' and not len(from_diff) == 2:
raise click.BadParameter("If using mode=payload, you must provide two payloads to compare")
if advisory and default_advisory_type:
raise click.BadParameter("Use only one of --use-default-advisory or --add")
runtime.initialize()
bz_data = runtime.gitdata.load_data(key='bugzilla').data
bzapi = elliottlib.bzutil.get_bzapi(bz_data)
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
if mode == 'sweep':
green_prefix("Searching for bugs with target release(s):")
click.echo(" {tr}".format(tr=", ".join(bz_data['target_release'])))
bug_ids = elliottlib.bzutil.search_for_bugs(bz_data, status)
elif mode == 'list':
bug_ids = [bzapi.getbug(i) for i in cli_opts.id_convert(id)]
elif mode == "diff":
click.echo(runtime.working_dir)
bug_id_strings = elliottlib.openshiftclient.get_bug_list(runtime.working_dir, from_diff[0], from_diff[1])
bug_ids = [bzapi.getbug(i) for i in bug_id_strings]
green_prefix("Found {} bugs:".format(len(bug_ids)))
click.echo(" {}".format(", ".join([str(b.bug_id) for b in bug_ids])))
if report:
green_print("{:<8s} {:<25s} {:<12s} {:<7s} {:<10s} {:60s}".format("ID", "COMPONENT", "STATUS", "SCORE", "AGE", "SUMMARY"))
for bug in bug_ids:
created_date = datetime.datetime.strptime(str(bug.creation_time), '%Y%m%dT%H:%M:%S')
days_ago = (datetime.datetime.today() - created_date).days
click.echo("{:<8d} {:<25s} {:<12s} {:<7s} {:<3d} days {:60s} ".format(bug.id,
bug.component,
bug.status,
bug.cf_pm_score,
days_ago,
bug.summary[:60].encode('ascii', 'replace')))
if len(flag) > 0:
for bug in bug_ids:
for f in flag:
bug.updateflags({f: "+"})
if advisory is not False:
elliottlib.errata.add_bugs_with_retry(advisory, [bug.id for bug in bug_ids], False)
def verify_payload(ctx, payload, advisory):
"""Cross-check that the builds present in PAYLOAD match the builds
attached to ADVISORY. The payload is treated as the source of
truth. If something is absent or different in the advisory it is
treated as an error with the advisory.
\b
PAYLOAD - Full pullspec of the payload to verify
ADVISORY - Numerical ID of the advisory
Two checks are made:
\b
1. Missing in Advisory - No payload components are absent from the given advisory
2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory
Results are summarily printed at the end of the run. They are also
written out to summary_results.json.
Verify builds in the given payload match the builds attached to
advisory 41567
\b
$ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567
"""
try:
green_prefix("Fetching advisory builds: ")
click.echo("Advisory - {}".format(advisory))
builds = elliottlib.errata.get_builds(advisory)
except GSSError:
exit_unauthenticated()
except elliottlib.exceptions.ErrataToolError as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
all_advisory_nvrs = {}
# Results come back with top level keys which are brew tags
green_prefix("Looping over tags: ")
click.echo("{} tags to check".format(len(builds.keys())))
for tag in builds.keys():
# Each top level has a key 'builds' which is a list of dicts
green_prefix("Looping over builds in tag: ")
click.echo("{} with {} builds".format(tag, len(builds[tag]['builds'])))
for build in builds[tag]['builds']:
# Each dict has a top level key which might be the actual
# 'nvr' but I don't have enough data to know for sure
# yet. Also I don't know when there might be more than one
# key in the build dict. We'll loop over it to be sure.
for name in build.keys():
n, v, r = name.rsplit('-', 2)
version_release = "{}-{}".format(v, r)
all_advisory_nvrs[n] = version_release
click.echo("Found {} builds".format(len(all_advisory_nvrs)))
all_payload_nvrs = {}
click.echo("Fetching release info")
release_export_cmd = 'oc adm release info {} -o json'.format(payload)
rc, stdout, stderr = exectools.cmd_gather(release_export_cmd)
if rc != 0:
# Probably no point in continuing.. can't contact brew?
print("Unable to run oc release info: out={} ; err={}".format(stdout, stderr))
exit(1)
else:
click.echo("Got release info")
payload_json = json.loads(stdout)
green_prefix("Looping over payload images: ")
click.echo("{} images to check".format(len(payload_json['references']['spec']['tags'])))
for image in payload_json['references']['spec']['tags']:
click.echo("----")
green_prefix("Getting payload image metadata: ")
click.echo("{}".format(image['from']['name']))
pullspec = image['from']['name']
image_name = image['name']
pullspec_cmd = 'oc image info {} -o json'.format(pullspec)
rc, stdout, stderr = exectools.cmd_gather(pullspec_cmd)
if rc != 0:
# Probably no point in continuing.. can't contact brew?
red_prefix("Unable to run oc image info: ")
click.echo("out={} ; err={}".format(stdout, stderr))
exit(1)
image_info = json.loads(stdout)
labels = image_info['config']['config']['Labels']
# The machine-os-content image doesn't follow the standard
# pattern. We need to skip that image when we find it, it is
# not attached to advisories.
if 'com.coreos.ostree-commit' in labels:
yellow_prefix("Skipping machine-os-content image: ")
click.echo("Not required for checks")
continue
component = labels['com.redhat.component']
n = image_name
click.echo("Payload name: {}".format(n))
click.echo("Brew name: {}".format(component))
if labels:
v = labels['version']
r = labels['release']
all_payload_nvrs[component] = "{}-{}".format(v, r)
else:
print("For image {} Labels doesn't exist, image_info: {}".format(image_name, image_info))
missing_in_errata = {}
payload_doesnt_match_errata = {}
output = {
'missing_in_advisory': missing_in_errata,
'payload_advisory_mismatch': payload_doesnt_match_errata,
}
green_prefix("Analyzing data: ")
click.echo("{} images to consider from payload".format(len(all_payload_nvrs)))
for image, vr in all_payload_nvrs.items():
yellow_prefix("Cross-checking from payload: ")
click.echo("{}-{}".format(image, vr))
if image not in all_advisory_nvrs:
missing_in_errata[image] = "{}-{}".format(image, vr)
click.echo("{} in payload not found in advisory".format("{}-{}".format(image, vr)))
elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]:
click.echo("{} from payload has version {} which does not match {} from advisory".format(
image, vr, all_advisory_nvrs[image]))
payload_doesnt_match_errata[image] = {
'payload': vr,
'errata': all_advisory_nvrs[image]
}
green_print("Summary results:")
click.echo(json.dumps(output, indent=4))
with open('summary_results.json', 'w') as fp:
json.dump(output, fp, indent=4)
green_prefix("Wrote out summary results: ")
click.echo("summary_results.json")
def add_bugs_with_retry(advisory, bugs, noop=False, batch_size=100):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bugs: iterable of bzutil.bug to attach to advisory
:return:
"""
print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}')
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
existing_bugs = advs.errata_bugs
new_bugs = set(bug.id for bug in bugs) - set(existing_bugs)
print(f'Bugs already attached: {len(existing_bugs)}')
print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}')
if not new_bugs:
print('No new bugs to attach. Exiting.')
return
bugs = list(new_bugs)
batches = list(range(0, len(bugs), batch_size))
if len(bugs) % batch_size != 0:
batches.append(len(bugs))
green_prefix(
f"Adding bugs in batches of {batch_size}. Number of batches: {len(batches)-1}\n"
)
for i in range(len(batches) - 1):
start, end = batches[i], batches[i + 1]
print(f"Attaching Batch {i+1}")
if noop:
print('Dry run: Would have attached bugs')
continue
try:
advs.addBugs(bugs[start:end])
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
block_list = parse_exception_error_message(e)
retry_list = [x for x in bugs[start:end] if x not in block_list]
if len(retry_list) == 0:
continue
try:
advs = Erratum(errata_id=advisory)
advs.addBugs(retry_list)
advs.commit()
except ErrataException as e:
raise exceptions.ElliottFatalError(
getattr(e, 'message', repr(e)))
print("remaining bugs attached")
async def verify_cvp_cli(runtime: Runtime, all_images, nvrs, optional_checks,
all_optional_checks, fix, message):
""" Verify CVP test results
Example 1: Verify CVP test results for all latest 4.4 image builds, also warn those with failed content_set_check
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check
Example 2: Apply patches to ocp-build-data to fix the redundant content sets error:
$ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check --fix
Note:
1. If `--message` is not given, `--fix` will leave changed ocp-build-data files uncommitted.
2. Make sure your ocp-build-data directory is clean before running `--fix`.
"""
if bool(all_images) + bool(nvrs) != 1:
raise click.BadParameter('You must use one of --all or --build.')
if all_optional_checks and optional_checks:
raise click.BadParameter(
'Use only one of --all-optional-checks or --include-optional-check.'
)
runtime.initialize(mode='images')
tag_pv_map = runtime.gitdata.load_data(
key='erratatool',
replace_vars=runtime.group_config.vars.primitive()
if runtime.group_config.vars else
{}).data.get('brew_tag_product_version_mapping')
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
builds = []
if all_images:
runtime.logger.info("Getting latest image builds from Brew...")
# TODO: This does not honor overrides
builds = get_latest_image_builds(brew_session,
tag_pv_map.keys(),
runtime.image_metas,
event=runtime.brew_event)
elif nvrs:
runtime.logger.info(f"Finding {len(builds)} builds from Brew...")
builds = brew.get_build_objects(nvrs, brew_session)
runtime.logger.info(f"Found {len(builds)} image builds.")
resultsdb_api = ResultsDBAPI()
nvrs = [b["nvr"] for b in builds]
runtime.logger.info(
f"Getting CVP test results for {len(builds)} image builds...")
latest_cvp_results = await get_latest_cvp_results(runtime, resultsdb_api,
nvrs)
# print a summary for all CVP results
good_results = [] # good means PASSED or INFO
bad_results = [] # bad means NEEDS_INSPECTION or FAILED
incomplete_nvrs = []
for nvr, result in zip(nvrs, latest_cvp_results):
if not result:
incomplete_nvrs.append(nvr)
continue
outcome = result.get(
"outcome"
) # only PASSED, FAILED, INFO, NEEDS_INSPECTION are now valid outcome values (https://resultsdb20.docs.apiary.io/#introduction/changes-since-1.0)
if outcome in {"PASSED", "INFO"}:
good_results.append(result)
elif outcome in {"NEEDS_INSPECTION", "FAILED"}:
bad_results.append(result)
green_prefix("good: {}".format(len(good_results)))
click.echo(", ", nl=False)
red_prefix("bad: {}".format(len(bad_results)))
click.echo(", ", nl=False)
yellow_print("incomplete: {}".format(len(incomplete_nvrs)))
if bad_results:
red_print("The following builds didn't pass CVP tests:")
for r in bad_results:
nvr = r["data"]["item"][0]
red_print(f"{nvr} {r['outcome']}: {r['ref_url']}")
if incomplete_nvrs:
yellow_print(
"We couldn't find CVP test results for the following builds:")
for nvr in incomplete_nvrs:
yellow_print(nvr)
if not optional_checks and not all_optional_checks:
return # no need to print failed optional CVP checks
# Find failed optional CVP checks in case some of the tiem *will* become required.
optional_checks = set(optional_checks)
complete_results = good_results + bad_results
runtime.logger.info(
f"Getting optional checks for {len(complete_results)} CVP tests...")
optional_check_results = await get_optional_checks(runtime,
complete_results)
component_distgit_keys = {
} # a dict of brew component names to distgit keys
content_set_to_repo_names = {
} # a map of content set names to group.yml repo names
for image in runtime.image_metas():
component_distgit_keys[image.get_component_name()] = image.distgit_key
for repo_name, repo_info in runtime.group_config.get("repos", {}).items():
for arch, cs_name in repo_info.get('content_set', {}).items():
if arch == "optional":
continue # not a real arch name
content_set_to_repo_names[cs_name] = repo_name
nvr_to_builds = {build["nvr"]: build for build in builds}
ocp_build_data_updated = False
failed_with_not_covered_rpms = set()
failed_with_redundant_repos = set()
only_failed_in_non_x86_with_not_covered_rpms = set()
only_failed_in_non_x86_with_redundant_repos = set()
for cvp_result, checks in zip(complete_results, optional_check_results):
# example optional checks: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/hive-container-v4.6.0-202008010302.p0/da01e36c-8c69-4a19-be7d-ba4593a7b085/sanity-tests-optional-results.json
bad_checks = [
check for check in checks["checks"]
if check["status"] != "PASS" and (
all_optional_checks or check["name"] in optional_checks)
]
if not bad_checks:
continue
nvr = cvp_result["data"]["item"][0]
build = nvr_to_builds[nvr]
yellow_print("----------")
yellow_print(
f"Build {nvr} (https://brewweb.engineering.redhat.com/brew/buildinfo?buildID={nvr_to_builds[nvr]['id']}) has {len(bad_checks)} problematic CVP optional checks:"
)
for check in bad_checks:
yellow_print(f"* {check['name']} {check['status']}")
try:
amd64_result = list(
filter(lambda item: item.get("arch") == "amd64",
check["logs"][-1]))
except AttributeError:
red_print("CVP result malformed.")
if len(amd64_result) != 1:
red_print(
"WHAT?! This build doesn't include an amd64 image? This shouldn't happen. Check Brew and CVP logs with the CVP team!"
)
continue
amd64_result = amd64_result[0]
image_component_name = nvr.rsplit('-', 2)[0]
distgit_key = component_distgit_keys.get(image_component_name)
amd64_redundant_cs = amd64_result.get("redundant_cs", [])
amd64_redundant_repos = {
content_set_to_repo_names[cs]
for cs in amd64_redundant_cs
}
def _strip_arch_suffix(rpm):
# rh-nodejs10-3.2-3.el7.x86_64 -> rh-nodejs10-3.2-3.el7
rpm_split = rpm.rsplit(".", 1)
return rpm_split[0]
amd64_not_covered_rpms = {
_strip_arch_suffix(rpm)
for rpm in amd64_result.get("not_covered_rpms", [])
}
if check["name"] == "content_set_check":
details = check["logs"][
-1] # example: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/logging-fluentd-container-v4.6.0-202008261251.p0/dd9f2024-5440-4f33-b508-472ccf258439/sanity-tests-optional-results.json
if not details:
red_print(
"content_set_check failed without any explanation. Report to CVP team!"
)
continue
if len(
details
) > 1: # if this build is multi-arch, check if all per-arch results are consistent
for result in details:
if result["arch"] == "amd64":
continue
redundant_repos = {
content_set_to_repo_names[cs]
for cs in result.get("redundant_cs", [])
}
if redundant_repos != amd64_redundant_repos:
only_failed_in_non_x86_with_redundant_repos.add(
nvr)
red_print(
f"""content_set_check for {nvr} arch {result["arch"]} has different redundant_cs result from the one for amd64:
{result["arch"]} has redundant_cs {result.get("redundant_cs")},
but amd64 has redundant_cs {amd64_redundant_cs}.
Not sure what happened. Please see Brew and CVP logs and/or check with the CVP team."""
)
not_covered_rpms = {
_strip_arch_suffix(rpm)
for rpm in result.get("not_covered_rpms", [])
}
if not_covered_rpms != amd64_not_covered_rpms:
only_failed_in_non_x86_with_not_covered_rpms.add(
nvr)
red_print(
f"""content_set_check for {nvr} arch {result["arch"]} has different not_covered_rpms result from the one for amd64:
{result["arch"]} has extra not_covered_rpms {not_covered_rpms - amd64_not_covered_rpms},
and missing not_covered_rpms {amd64_not_covered_rpms - not_covered_rpms}.
Not sure what happened. Check Brew and CVP logs with the CVP team!"""
)
if amd64_not_covered_rpms: # This build has not_covered_rpms
failed_with_not_covered_rpms.add(nvr)
yellow_print(
f"Image {distgit_key} has not_covered_rpms: {amd64_not_covered_rpms}"
)
brew_repos = await find_repos_for_rpms(
amd64_not_covered_rpms, build)
yellow_print(
f"Those repos shown in Brew logs might be a good hint: {brew_repos}"
)
runtime.logger.info(
"Looking for parent image's content_sets...")
parent = get_parent_build_ids([build])[0]
if parent:
parent_build = brew.get_build_objects([parent])[0]
parent_cs = await get_content_sets_for_build(
parent_build)
parent_enabled_repos = {
content_set_to_repo_names[cs]
for cs in parent_cs.get("x86_64", [])
}
enabled_repos = set(
runtime.image_map[distgit_key].config.get(
"enabled_repos", []))
missing_repos = parent_enabled_repos - enabled_repos
yellow_print(
f"""The following repos are defined in parent {parent_build["nvr"]} {component_distgit_keys.get(parent_build["name"], "?")}.yml but not in
{component_distgit_keys[build["name"]]}.yml: {missing_repos}"""
)
if fix and missing_repos:
runtime.logger.info(
"Trying to merge parent image's content_sets..."
)
fix_missing_content_set(runtime, distgit_key,
missing_repos)
ocp_build_data_updated = True
runtime.logger.info(f"{distgit_key}.yml patched")
if amd64_redundant_repos: # This build has redundant_cs
failed_with_redundant_repos.add(nvr)
yellow_print(
f"Image {distgit_key} has redundant repos: {amd64_redundant_repos}"
)
if not fix:
yellow_print(
f"Please add the following repos to non_shipping_repos in {distgit_key}.yml: {amd64_redundant_repos}"
)
else:
runtime.logger.info(
f"Applying redundant content sets fix to {distgit_key}.yml..."
)
fix_redundant_content_set(runtime, distgit_key,
amd64_redundant_repos)
ocp_build_data_updated = True
runtime.logger.info(f"{distgit_key}.yml patched")
print(
f"See {cvp_result['ref_url']}sanity-tests-optional-results.json for more details."
)
if failed_with_not_covered_rpms or failed_with_redundant_repos:
yellow_print(
f"{len(failed_with_not_covered_rpms | failed_with_redundant_repos)} images failed content_sets.\n Where"
)
if failed_with_not_covered_rpms:
yellow_print(
f"\t{len(failed_with_not_covered_rpms)} images failed content_sets check because of not_covered_rpms:"
)
for rpm in failed_with_not_covered_rpms:
line = f"\t\t{rpm}"
if rpm in only_failed_in_non_x86_with_not_covered_rpms:
line += " - non-x86 arches are different from x86 one"
yellow_print(line)
if failed_with_redundant_repos:
yellow_print(
f"\t{len(failed_with_redundant_repos)} images failed content_sets check because of redundant_repos:"
)
for rpm in failed_with_redundant_repos:
line = f"\t\t{rpm}"
if rpm in only_failed_in_non_x86_with_redundant_repos:
line += " - non-x86 arches are different from x86 one"
yellow_print(line)
if message and ocp_build_data_updated:
runtime.gitdata.commit(message)
def poll_signed(runtime, minutes, advisory, default_advisory_type, noop):
"""Poll for the signed-status of RPM builds attached to
ADVISORY. Returns rc=0 when all builds have been signed. Returns non-0
after MINUTES have passed and all builds have not been signed. This
non-0 return code is the number of unsigned builds remaining. All
builds must show 'signed' for this command to succeed.
NOTE: The two advisory options are mutually exclusive.
For testing in pipeline scripts this sub-command accepts a --noop
option. When --noop is used the value of --minutes is irrelevant. This
command will print out the signed state of all attached builds and
then exit with rc=0 if all builds are signed and non-0 if builds are
still unsigned. In the non-0 case the return code is the number of
unsigned builds.
Wait 15 minutes for the default 4.2 advisory to show all RPMS have
been signed:
$ elliott -g openshift-4.2 poll-signed --use-default-advisory rpm
Wait 5 mintes for the provided 4.2 advisory to show all RPMs have
been signed:
$ elliott -g openshift-4.2 poll-signed -m 5 --advisory 123456
Print the signed status of all attached builds, exit
immediately. Return code is the number of unsigned builds.
\b
$ elliott -g openshift-4.2 poll-signed --noop --use-default-advisory rpm
"""
if not (bool(advisory) ^ bool(default_advisory_type)):
raise click.BadParameter(
"Use only one of --use-default-advisory or --advisory")
runtime.initialize(no_group=default_advisory_type is None)
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
if not noop:
click.echo("Polling up to {} minutes for all RPMs to be signed".format(
minutes))
try:
e = elliottlib.errata.Advisory(errata_id=advisory)
all_builds = set([])
all_signed = False
# `errata_builds` is a dict with brew tags as keys, values are
# lists of builds on the advisory with that tag
for k, v in e.errata_builds.items():
all_builds = all_builds.union(set(v))
green_prefix("Fetching initial states: ")
click.echo("{} builds to check".format(len(all_builds)))
start_time = datetime.datetime.now()
while datetime.datetime.now() - start_time < datetime.timedelta(
minutes=minutes):
pbar_header("Getting build signatures: ", "Should be pretty quick",
all_builds)
pool = ThreadPool(cpu_count())
# Look up builds concurrently
click.secho("[", nl=False)
build_sigs = pool.map(
lambda build: progress_func(
lambda: elliottlib.errata.build_signed(build), '*'),
all_builds)
# Wait for results
pool.close()
pool.join()
click.echo(']')
if all(build_sigs):
all_signed = True
break
elif noop:
# Escape the time-loop
break
else:
yellow_prefix("Not all builds signed: ")
click.echo("re-checking")
continue
if not all_signed:
red_prefix("Signing incomplete: ")
if noop:
click.echo("All builds not signed. ")
else:
click.echo(
"All builds not signed in given window ({} minutes). ".
format(minutes))
exit(1)
else:
green_prefix("All builds signed: ")
click.echo("Enjoy!")
except ErrataException as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
async def verify_payload(ctx, payload, advisory):
"""Cross-check that the builds present in PAYLOAD match the builds
attached to ADVISORY. The payload is treated as the source of
truth. If something is absent or different in the advisory it is
treated as an error with the advisory.
\b
PAYLOAD - Full pullspec of the payload to verify
ADVISORY - Numerical ID of the advisory
Two checks are made:
\b
1. Missing in Advisory - No payload components are absent from the given advisory
2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory
Results are summarily printed at the end of the run. They are also
written out to summary_results.json.
Verify builds in the given payload match the builds attached to
advisory 41567
\b
$ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567
"""
all_advisory_nvrs = elliottlib.errata.get_advisory_nvrs(advisory)
click.echo("Found {} builds".format(len(all_advisory_nvrs)))
all_payload_nvrs = {}
click.echo("Fetching release info")
release_export_cmd = 'oc adm release info {} -o json'.format(payload)
rc, stdout, stderr = exectools.cmd_gather(release_export_cmd)
if rc != 0:
# Probably no point in continuing.. can't contact brew?
print("Unable to run oc release info: out={} ; err={}".format(
stdout, stderr))
exit(1)
else:
click.echo("Got release info")
payload_json = json.loads(stdout)
green_prefix("Looping over payload images: ")
click.echo("{} images to check".format(
len(payload_json['references']['spec']['tags'])))
cmds = [['oc', 'image', 'info', '-o', 'json', tag['from']['name']]
for tag in payload_json['references']['spec']['tags']]
green_prefix("Querying image infos...")
cmd_results = await asyncio.gather(
*[exectools.cmd_gather_async(cmd) for cmd in cmds])
for image, cmd, cmd_result in zip(
payload_json['references']['spec']['tags'], cmds, cmd_results):
click.echo("----")
image_name = image['name']
rc, stdout, stderr = cmd_result
if rc != 0:
# Probably no point in continuing.. can't contact brew?
red_prefix("Unable to run oc image info: ")
red_print(f"cmd={cmd!r}, out={stdout} ; err={stderr}")
exit(1)
image_info = json.loads(stdout)
labels = image_info['config']['config']['Labels']
# The machine-os-content image doesn't follow the standard
# pattern. We need to skip that image when we find it, it is
# not attached to advisories.
if 'com.coreos.ostree-commit' in labels:
yellow_prefix("Skipping machine-os-content image: ")
click.echo("Not required for checks")
continue
component = labels['com.redhat.component']
n = image_name
click.echo("Payload name: {}".format(n))
click.echo("Brew name: {}".format(component))
if labels:
v = labels['version']
r = labels['release']
all_payload_nvrs[component] = "{}-{}".format(v, r)
else:
print("For image {} Labels doesn't exist, image_info: {}".format(
image_name, image_info))
missing_in_errata = {}
payload_doesnt_match_errata = {}
in_pending_advisory = []
in_shipped_advisory = []
output = {
'missing_in_advisory': missing_in_errata,
'payload_advisory_mismatch': payload_doesnt_match_errata,
"in_pending_advisory": in_pending_advisory,
"in_shipped_advisory": in_shipped_advisory,
}
green_prefix("Analyzing data: ")
click.echo("{} images to consider from payload".format(
len(all_payload_nvrs)))
for image, vr in all_payload_nvrs.items():
yellow_prefix("Cross-checking from payload: ")
click.echo("{}-{}".format(image, vr))
if image not in all_advisory_nvrs:
missing_in_errata[image] = "{}-{}".format(image, vr)
click.echo("{} in payload not found in advisory".format(
"{}-{}".format(image, vr)))
elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]:
click.echo(
"{} from payload has version {} which does not match {} from advisory"
.format(image, vr, all_advisory_nvrs[image]))
payload_doesnt_match_errata[image] = {
'payload': vr,
'errata': all_advisory_nvrs[image]
}
if missing_in_errata: # check if missing images are already shipped or pending to ship
advisory_nvrs: Dict[int, List[str]] = {
} # a dict mapping advisory numbers to lists of NVRs
green_print(
f"Checking if {len(missing_in_errata)} missing images are shipped..."
)
for nvr in missing_in_errata.copy().values():
# get the list of advisories that this build has been attached to
build = elliottlib.errata.get_brew_build(nvr)
# filter out dropped advisories
advisories = [
ad for ad in build.all_errata
if ad["status"] != "DROPPED_NO_SHIP"
]
if not advisories:
red_print(f"Build {nvr} is not attached to any advisories.")
continue
for advisory in advisories:
if advisory["status"] == "SHIPPED_LIVE":
green_print(
f"Missing build {nvr} has been shipped with advisory {advisory}."
)
else:
yellow_print(
f"Missing build {nvr} is in another pending advisory.")
advisory_nvrs.setdefault(advisory["id"], []).append(nvr)
name = nvr.rsplit("-", 2)[0]
del missing_in_errata[name]
if advisory_nvrs:
click.echo(
f"Getting information of {len(advisory_nvrs)} advisories...")
for advisory, nvrs in advisory_nvrs.items():
advisory_obj = elliottlib.errata.get_raw_erratum(advisory)
adv_type, adv_info = next(iter(advisory_obj["errata"].items()))
item = {
"id": advisory,
"type": adv_type.upper(),
"url": elliottlib.constants.errata_url + f"/{advisory}",
"summary": adv_info["synopsis"],
"state": adv_info["status"],
"nvrs": nvrs,
}
if adv_info["status"] == "SHIPPED_LIVE":
in_shipped_advisory.append(item)
else:
in_pending_advisory.append(item)
green_print("Summary results:")
click.echo(json.dumps(output, indent=4))
with open('summary_results.json', 'w') as fp:
json.dump(output, fp, indent=4)
green_prefix("Wrote out summary results: ")
click.echo("summary_results.json")
def find_builds_cli(runtime, advisory, default_advisory_type, builds, kind,
from_diff, as_json, allow_attached, remove, clean,
no_cdn_repos, payload, non_payload):
'''Automatically or manually find or attach/remove viable rpm or image builds
to ADVISORY. Default behavior searches Brew for viable builds in the
given group. Provide builds manually by giving one or more --build
(-b) options. Manually provided builds are verified against the Errata
Tool API.
\b
* Attach the builds to ADVISORY by giving --attach
* Remove the builds to ADVISORY by giving --remove
* Specify the build type using --kind KIND
Example: Assuming --group=openshift-3.7, then a build is a VIABLE
BUILD IFF it meets ALL of the following criteria:
\b
* HAS the tag in brew: rhaos-3.7-rhel7-candidate
* DOES NOT have the tag in brew: rhaos-3.7-rhel7
* IS NOT attached to ANY existing RHBA, RHSA, or RHEA
That is to say, a viable build is tagged as a "candidate", has NOT
received the "shipped" tag yet, and is NOT attached to any PAST or
PRESENT advisory. Here are some examples:
SHOW the latest OSE 3.6 image builds that would be attached to a
3.6 advisory:
$ elliott --group openshift-3.6 find-builds -k image
ATTACH the latest OSE 3.6 rpm builds to advisory 123456:
\b
$ elliott --group openshift-3.6 find-builds -k rpm --attach 123456
VERIFY (no --attach) that the manually provided RPM NVR and build
ID are viable builds:
$ elliott --group openshift-3.6 find-builds -k rpm -b megafrobber-1.0.1-2.el7 -a 93170
\b
Remove specific RPM NVR and build ID from advisory:
$ elliott --group openshift-4.3 find-builds -k image -b oauth-server-container-v4.3.22-202005212137 -a 55017 --remove
'''
if from_diff and builds:
raise click.BadParameter(
'Use only one of --build or --from-diff/--between.')
if clean and (remove or from_diff or builds):
raise click.BadParameter(
'Option --clean cannot be used with --build or --from-diff/--between.'
)
if not builds and remove:
raise click.BadParameter(
'Option --remove only support removing specific build with -b.')
if from_diff and kind != "image":
raise click.BadParameter(
'Option --from-diff/--between should be used with --kind/-k image.'
)
if advisory and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --attach')
if payload and non_payload:
raise click.BadParameter('Use only one of --payload or --non-payload.')
runtime.initialize(mode='images' if kind == 'image' else 'none')
replace_vars = runtime.group_config.vars.primitive(
) if runtime.group_config.vars else {}
et_data = runtime.gitdata.load_data(key='erratatool',
replace_vars=replace_vars).data
tag_pv_map = et_data.get('brew_tag_product_version_mapping')
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
ensure_erratatool_auth(
) # before we waste time looking up builds we can't process
# get the builds we want to add
unshipped_nvrps = []
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
if builds:
green_prefix('Fetching builds...')
unshipped_nvrps = _fetch_nvrps_by_nvr_or_id(
builds, tag_pv_map, ignore_product_version=remove)
elif clean:
unshipped_builds = errata.get_brew_builds(advisory)
elif from_diff:
unshipped_nvrps = _fetch_builds_from_diff(from_diff[0], from_diff[1],
tag_pv_map)
else:
if kind == 'image':
unshipped_nvrps = _fetch_builds_by_kind_image(
runtime, tag_pv_map, brew_session, payload, non_payload)
elif kind == 'rpm':
unshipped_nvrps = _fetch_builds_by_kind_rpm(
tag_pv_map, brew_session)
pbar_header('Fetching builds from Errata: ',
'Hold on a moment, fetching buildinfos from Errata Tool...',
unshipped_builds if clean else unshipped_nvrps)
if not clean and not remove:
# if is --clean then batch fetch from Erratum no need to fetch them individually
# if is not for --clean fetch individually using nvrp tuples then get specific
# elliottlib.brew.Build Objects by get_brew_build()
# e.g. :
# ('atomic-openshift-descheduler-container', 'v4.3.23', '202005250821', 'RHEL-7-OSE-4.3').
# Build(atomic-openshift-descheduler-container-v4.3.23-202005250821).
unshipped_builds = parallel_results_with_progress(
unshipped_nvrps, lambda nvrp: elliottlib.errata.get_brew_build(
'{}-{}-{}'.format(nvrp[0], nvrp[1], nvrp[2]),
nvrp[3],
session=requests.Session()))
if not (allow_attached or builds):
unshipped_builds = _filter_out_inviable_builds(
kind, unshipped_builds, elliottlib.errata)
_json_dump(as_json, unshipped_builds, kind, tag_pv_map)
if not unshipped_builds:
green_print('No builds needed to be attached.')
return
if advisory:
if remove:
_detach_builds(
advisory,
[f"{nvrp[0]}-{nvrp[1]}-{nvrp[2]}" for nvrp in unshipped_nvrps])
elif clean:
_detach_builds(advisory, [b.nvr for b in unshipped_builds])
else: # attach
erratum = _update_to_advisory(unshipped_builds, kind, advisory,
remove, clean)
if not no_cdn_repos and kind == "image" and not (remove or clean):
cdn_repos = et_data.get('cdn_repos')
if cdn_repos:
# set up CDN repos
click.echo(
f"Configuring CDN repos {', '.join(cdn_repos)}...")
erratum.metadataCdnRepos(enable=cdn_repos)
click.echo("Done")
else:
click.echo('The following {n} builds '.format(n=len(unshipped_builds)),
nl=False)
if not (remove or clean):
click.secho('may be attached', bold=True, nl=False)
click.echo(' to an advisory:')
else:
click.secho('may be removed from', bold=True, nl=False)
click.echo(' from an advisory:')
for b in sorted(unshipped_builds):
click.echo(' ' + b.nvr)
def find_bugs_sweep(runtime: Runtime, advisory_id, default_advisory_type,
check_builds, major_version, find_bugs_obj, report, output,
brew_event, noop, count_advisory_attach_flags,
bug_tracker):
if output == 'text':
statuses = sorted(find_bugs_obj.status)
tr = bug_tracker.target_release()
green_prefix(
f"Searching {bug_tracker.type} for bugs with status {statuses} and target releases: {tr}\n"
)
bugs = find_bugs_obj.search(bug_tracker_obj=bug_tracker,
verbose=runtime.debug)
sweep_cutoff_timestamp = get_sweep_cutoff_timestamp(
runtime, cli_brew_event=brew_event)
if sweep_cutoff_timestamp:
utc_ts = datetime.utcfromtimestamp(sweep_cutoff_timestamp)
green_print(
f"Filtering bugs that have changed ({len(bugs)}) to one of the desired statuses before the "
f"cutoff time {utc_ts}...")
qualified_bugs = []
for chunk_of_bugs in chunk(bugs, constants.BUG_LOOKUP_CHUNK_SIZE):
b = bug_tracker.filter_bugs_by_cutoff_event(
chunk_of_bugs, find_bugs_obj.status, sweep_cutoff_timestamp)
qualified_bugs.extend(b)
click.echo(
f"{len(qualified_bugs)} of {len(bugs)} bugs are qualified for the cutoff time "
f"{utc_ts}...")
bugs = qualified_bugs
included_bug_ids, excluded_bug_ids = get_assembly_bug_ids(runtime)
if included_bug_ids & excluded_bug_ids:
raise ValueError(
"The following bugs are defined in both 'include' and 'exclude': "
f"{included_bug_ids & excluded_bug_ids}")
if included_bug_ids:
yellow_print(
"The following bugs will be additionally included because they are "
f"explicitly defined in the assembly config: {included_bug_ids}")
included_bugs = bug_tracker.get_bugs(included_bug_ids)
bugs.extend(included_bugs)
if excluded_bug_ids:
yellow_print(
"The following bugs will be excluded because they are explicitly "
f"defined in the assembly config: {excluded_bug_ids}")
bugs = [bug for bug in bugs if bug.id not in excluded_bug_ids]
if output == 'text':
green_prefix(f"Found {len(bugs)} bugs: ")
click.echo(", ".join(sorted(str(b.id) for b in bugs)))
if report:
print_report(bugs, output)
if count_advisory_attach_flags < 1:
return
# `--add ADVISORY_NUMBER` should respect the user's wish
# and attach all available bugs to whatever advisory is specified.
if advisory_id and not default_advisory_type:
bug_tracker.attach_bugs(advisory_id, [b.id for b in bugs], noop=noop)
return
rpm_advisory_id = common.find_default_advisory(
runtime, 'rpm') if check_builds else None
bugs_by_type = categorize_bugs_by_type(bugs,
rpm_advisory_id=rpm_advisory_id,
major_version=major_version,
check_builds=check_builds)
advisory_types_to_attach = [
default_advisory_type
] if default_advisory_type else bugs_by_type.keys()
for advisory_type in sorted(advisory_types_to_attach):
bugs = bugs_by_type.get(advisory_type)
green_prefix(f'{advisory_type} advisory: ')
if bugs:
adv_id = common.find_default_advisory(runtime, advisory_type)
bug_tracker.attach_bugs(adv_id, [b.id for b in bugs], noop=noop)
else:
click.echo("0 bugs found")
def advisory_commons_cli(runtime, advisories, field, new, version_replace,
yes):
"""Display or Change a common field (like date) across multiple advisories.
Advisories created for an OCP version have common fields, that sometimes
will need updating. This command helps with that.
NOTE: The two advisory input options (--assembly and --advisories)
are mutually exclusive and can not be used together.
Show the field "publish_date" for all advisories for an assembly/group/advisories
$ elliott -g openshift-4.8 --assembly 4.8.8 advisory-commons --field "publish_date"
$ elliott -g openshift-3.11 advisory-commons --field "publish_date"
$ elliott advisory-commons 80825 80824 --field "publish_date"
(Preview) update field "publish_date" for all advisories for an assembly
$ elliott -g openshift-4.8 --assembly 4.8.8 advisory-commons --field "publish_date" --new "2021-Aug-31"
(Commit) update field "publish_date" for all advisories for an assembly
$ elliott -g openshift-4.8 --assembly 4.8.8 advisory-commons --field "publish_date" --new "2021-Aug-31" --yes
"""
noop = not yes
count_flags = sum(map(bool, [runtime.group, advisories]))
if count_flags > 1:
raise click.BadParameter("Use only one of --group or advisories param")
count_flags = sum(map(bool, [field, version_replace]))
if count_flags > 1:
raise click.BadParameter(
"Use only one of --field or --version-replace")
if new:
if field not in supported_update_fields:
raise click.BadParameter(
f"Only these fields are supported for update: {supported_update_fields}"
)
if not advisories:
runtime.initialize()
advisories = runtime.group_config.advisories.values()
errors = []
for advisory_id in advisories:
update = False
try:
advisory = Erratum(errata_id=advisory_id)
green_prefix(f"{advisory_id}: ")
if field:
current = getattr(advisory, field)
click.echo(f"{field} = {current}")
if new:
if new == current:
click.echo(
f"No change. New value is same as current value: {field} = {current}"
)
else:
click.echo(
f"Preparing update to {field}: {current} ➔ {new}")
advisory = _update_advisory(field, new, advisory)
update = True
elif version_replace:
rex = r"^(\d.\d+.\d+):(\d.\d+.\d+)$"
match = re.search(rex, version_replace)
if not match:
click.BadParameter(
f"--version-replace needs to be of format {rex}. example '4.5.6:4.7.8'"
)
search_version, replace_version = match.groups()
# special case for description
f = "description"
click.echo(f"<{f}>")
current = getattr(advisory, f)
lines_to_match = [
"This advisory contains the RPM packages for Red Hat OpenShift Container Platform {version}",
"quay.io/openshift-release-dev/ocp-release:{version}"
]
for line in lines_to_match:
search_line = line.format(version=search_version)
if search_line in current:
replace_line = line.format(version=replace_version)
new_value = current.replace(search_line, replace_line)
click.echo(
f"Preparing line update: {search_line} ➔ {replace_line}"
)
update = True
if update:
_update_advisory(f, new_value, advisory)
else:
click.echo("No change. New value is same as current value")
# rest of the fields
fields = ["synopsis", "topic"]
for f in fields:
click.echo(f"<{f}>")
current = getattr(advisory, f)
new_value = current.replace(search_version,
replace_version)
if current == new_value:
click.echo(
"No change. New value is same as current value")
else:
click.echo(
f"Preparing update: {current} ➔ {new_value}")
_update_advisory(f, new_value, advisory)
update = True
if not noop and update:
advisory.commit()
click.echo("Committed change")
except ErrataException as ex:
click.echo(f'Error fetching/changing {advisory_id}: {ex}')
errors.append(ex)
if errors:
raise Exception(errors)