def categorize_bugs_by_type(bugs: List, rpm_advisory_id: Optional[int] = None, major_version: int = 4, check_builds: bool = True): # key is type ("rpm", "image", "extras"), value is a set of bug IDs. bugs_by_type = {"rpm": set(), "image": set(), "extras": set()} # for 3.x, all bugs should go to the rpm advisory if int(major_version) < 4: bugs_by_type["rpm"] = set(bugs) else: # for 4.x, sweep rpm tracker bugs into rpm advisory rpm_bugs = Bug.get_valid_rpm_cves(bugs) if rpm_bugs: green_prefix("RPM Tracker Bugs found: ") click.echo(sorted(b.id for b in rpm_bugs)) # if --check-builds flag is set # only attach bugs that have corresponding brew builds attached to rpm advisory if check_builds and rpm_advisory_id: click.echo( "Validating bugs with builds attached to the rpm advisory") attached_builds = errata.get_advisory_nvrs(rpm_advisory_id) packages = attached_builds.keys() not_found = [] for bug, package_name in rpm_bugs.items(): if package_name not in packages: not_found.append((bug.id, package_name)) else: click.echo( f"Build found for #{bug.id}, {package_name}") bugs_by_type["rpm"].add(bug) if not_found: red_prefix("RPM CVE Tracker Error: ") click.echo( "The following (tracker bug, package) were found but not attached, because " "no corresponding brew builds were found attached to the rpm advisory. " "First attach builds and then rerun to attach the bugs, or exclude the " "bug ids in the assembly definition") click.echo(not_found) raise ValueError( f'No builds found for CVE (bug, package): {not_found}. Either attach ' f'builds or exclude the bugs in the assembly definition: {[b for b, p in not_found]}' ) else: click.echo( "Skipping attaching RPM Tracker Bugs. Use --check-builds flag to validate with builds." ) bugs_by_type["extras"] = extras_bugs(bugs) # all other bugs should go into "image" advisory bugs_by_type["image"] = set( bugs) - bugs_by_type["extras"] - rpm_bugs.keys() return bugs_by_type
def find_default_advisory(runtime, default_advisory_type, quiet=False): '''The `quiet` parameter will disable printing the informational message''' default_advisory = runtime.group_config.advisories.get(default_advisory_type, None) if default_advisory is None: red_prefix('No value defined for default advisory:') click.echo(' The key advisories.{} is not defined for group {} in group.yml'.format( default_advisory_type, runtime.group)) exit(1) if not quiet: green_prefix('Default advisory detected: ') click.echo(default_advisory) return default_advisory
def show(ctx, advisory, yaml, json): """ Show RPMDiff failures for an advisory. """ runtime = ctx.obj # type: Runtime if not advisory: runtime.initialize() advisory = runtime.group_config.advisories.get("rpm", 0) if not advisory: raise ElliottFatalError( "No RPM advisory number configured in ocp-build-data.") else: runtime.initialize(no_group=True) logger = runtime.logger logger.info( "Fetching RPMDiff runs from Errata Tool for advisory {}...".format( advisory)) rpmdiff_runs = list(errata.get_rpmdiff_runs(advisory)) logger.info("Found {} RPMDiff runs.".format(len(rpmdiff_runs))) # "good" means PASSED, INFO, or WAIVED good_runs = [] # "bad" means NEEDS_INSPECTION or FAILED bad_runs = [] incomplete_runs = [] for rpmdiff_run in rpmdiff_runs: attr = rpmdiff_run['attributes'] if attr["status"] in constants.ET_GOOD_EXTERNAL_TEST_STATUSES: good_runs.append(rpmdiff_run) elif attr["status"] in constants.ET_BAD_EXTERNAL_TEST_STATUSES: bad_runs.append(rpmdiff_run) else: incomplete_runs.append(rpmdiff_run) util.green_prefix("good: {}".format(len(good_runs))) click.echo(", ", nl=False) util.red_prefix("bad: {}".format(len(bad_runs))) click.echo(", ", nl=False) util.yellow_print("incomplete: {}".format(len(incomplete_runs))) if not bad_runs: return logger.info( "Fetching detailed information from RPMDiff for bad RPMDiff runs...") rpmdiff_client = RPMDiffClient(constants.RPMDIFF_HUB_URL) rpmdiff_client.authenticate() if yaml or json: _structured_output(bad_runs, rpmdiff_client, yaml) else: _unstructured_output(bad_runs, rpmdiff_client)
def show(ctx, advisory): """ Show RPMDiff failures for an advisory. """ runtime = ctx.obj # type: Runtime if not advisory: runtime.initialize() advisory = runtime.group_config.advisories.get("rpm", 0) if not advisory: raise ElliottFatalError( "No RPM advisory number configured in ocp-build-data.") else: runtime.initialize(no_group=True) logger = runtime.logger logger.info( "Fetching RPMDiff runs from Errata Tool for advisory {}...".format( advisory)) rpmdiff_runs = list(errata.get_rpmdiff_runs(advisory)) logger.info("Found {} RPMDiff runs.".format(len(rpmdiff_runs))) # "good" means PASSED, INFO, or WAIVED good_runs = [] # "bad" means NEEDS_INSPECTION or FAILED bad_runs = [] incomplete_runs = [] for rpmdiff_run in rpmdiff_runs: attr = rpmdiff_run['attributes'] if attr["status"] in constants.ET_GOOD_EXTERNAL_TEST_STATUSES: good_runs.append(rpmdiff_run) elif attr["status"] in constants.ET_BAD_EXTERNAL_TEST_STATUSES: bad_runs.append(rpmdiff_run) else: incomplete_runs.append(rpmdiff_run) util.green_prefix("good: {}".format(len(good_runs))) click.echo(", ", nl=False) util.red_prefix("bad:{}".format(len(bad_runs))) click.echo(", ", nl=False) util.yellow_print("incomplete: {}".format(len(incomplete_runs))) if not bad_runs: return logger.info( "Fetching detailed information from RPMDiff for bad RPMDiff runs...") rpmdiff_client = RPMDiffClient(constants.RPMDIFF_HUB_URL) rpmdiff_client.authenticate() for run in bad_runs: attr = run["attributes"] run_id = attr["external_id"] run_url = "{}/run/{}/".format(constants.RPMDIFF_WEB_URL, run_id) print("----------------") msg = "{0} {1}".format(run["relationships"]["brew_build"]["nvr"], attr["status"]) if attr["status"] == "NEEDS_INSPECTION": util.yellow_print(msg) else: util.red_print(msg) test_results = rpmdiff_client.get_test_results(run_id) run_obj = rpmdiff_client.get_run(run_id) for result in test_results: score = result["score"] if score >= 0 and score < 3: # good test result continue result_id = result["result_id"] test = result["test"] details = result["details"] test_id = test["test_id"] package_name = run_obj["package_name"] result_url = run_url + str(test_id) + "/" result_msg = "* TEST {0} {2} {1} {3}".format( result_id, constants.RPMDIFF_SCORE_NAMES[score], test["description"], result_url) if score == 3: # NEEDS_INSPECTION util.yellow_print(result_msg) else: util.red_print(result_msg) # get last waiver message waivers = rpmdiff_client.list_waivers(package_name, test_id, limit=1) if waivers: util.green_print(" Last waiver: @" + waivers[0]["owner"]["username"] + ": " + waivers[0]["description"]) else: util.yellow_print(" No last waiver found.") for detail in details: detail_msg = " * {1} {0}".format( constants.RPMDIFF_SCORE_NAMES[detail["score"]], detail["subpackage"]) if detail["score"] == 3: util.yellow_print(detail_msg) else: util.red_print(detail_msg) content = re.sub('^', ' ', detail["content"], flags=re.MULTILINE) print(content) print()
def poll_signed(runtime, minutes, advisory, default_advisory_type, noop): """Poll for the signed-status of RPM builds attached to ADVISORY. Returns rc=0 when all builds have been signed. Returns non-0 after MINUTES have passed and all builds have not been signed. This non-0 return code is the number of unsigned builds remaining. All builds must show 'signed' for this command to succeed. NOTE: The two advisory options are mutually exclusive. For testing in pipeline scripts this sub-command accepts a --noop option. When --noop is used the value of --minutes is irrelevant. This command will print out the signed state of all attached builds and then exit with rc=0 if all builds are signed and non-0 if builds are still unsigned. In the non-0 case the return code is the number of unsigned builds. Wait 15 minutes for the default 4.2 advisory to show all RPMS have been signed: $ elliott -g openshift-4.2 poll-signed --use-default-advisory rpm Wait 5 mintes for the provided 4.2 advisory to show all RPMs have been signed: $ elliott -g openshift-4.2 poll-signed -m 5 --advisory 123456 Print the signed status of all attached builds, exit immediately. Return code is the number of unsigned builds. \b $ elliott -g openshift-4.2 poll-signed --noop --use-default-advisory rpm """ if not (bool(advisory) ^ bool(default_advisory_type)): raise click.BadParameter( "Use only one of --use-default-advisory or --advisory") runtime.initialize(no_group=default_advisory_type is None) if default_advisory_type is not None: advisory = find_default_advisory(runtime, default_advisory_type) if not noop: click.echo("Polling up to {} minutes for all RPMs to be signed".format( minutes)) try: e = elliottlib.errata.Advisory(errata_id=advisory) all_builds = set([]) all_signed = False # `errata_builds` is a dict with brew tags as keys, values are # lists of builds on the advisory with that tag for k, v in e.errata_builds.items(): all_builds = all_builds.union(set(v)) green_prefix("Fetching initial states: ") click.echo("{} builds to check".format(len(all_builds))) start_time = datetime.datetime.now() while datetime.datetime.now() - start_time < datetime.timedelta( minutes=minutes): pbar_header("Getting build signatures: ", "Should be pretty quick", all_builds) pool = ThreadPool(cpu_count()) # Look up builds concurrently click.secho("[", nl=False) build_sigs = pool.map( lambda build: progress_func( lambda: elliottlib.errata.build_signed(build), '*'), all_builds) # Wait for results pool.close() pool.join() click.echo(']') if all(build_sigs): all_signed = True break elif noop: # Escape the time-loop break else: yellow_prefix("Not all builds signed: ") click.echo("re-checking") continue if not all_signed: red_prefix("Signing incomplete: ") if noop: click.echo("All builds not signed. ") else: click.echo( "All builds not signed in given window ({} minutes). ". format(minutes)) exit(1) else: green_prefix("All builds signed: ") click.echo("Enjoy!") except ErrataException as ex: raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
async def verify_payload(ctx, payload, advisory): """Cross-check that the builds present in PAYLOAD match the builds attached to ADVISORY. The payload is treated as the source of truth. If something is absent or different in the advisory it is treated as an error with the advisory. \b PAYLOAD - Full pullspec of the payload to verify ADVISORY - Numerical ID of the advisory Two checks are made: \b 1. Missing in Advisory - No payload components are absent from the given advisory 2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory Results are summarily printed at the end of the run. They are also written out to summary_results.json. Verify builds in the given payload match the builds attached to advisory 41567 \b $ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567 """ all_advisory_nvrs = elliottlib.errata.get_advisory_nvrs(advisory) click.echo("Found {} builds".format(len(all_advisory_nvrs))) all_payload_nvrs = {} click.echo("Fetching release info") release_export_cmd = 'oc adm release info {} -o json'.format(payload) rc, stdout, stderr = exectools.cmd_gather(release_export_cmd) if rc != 0: # Probably no point in continuing.. can't contact brew? print("Unable to run oc release info: out={} ; err={}".format( stdout, stderr)) exit(1) else: click.echo("Got release info") payload_json = json.loads(stdout) green_prefix("Looping over payload images: ") click.echo("{} images to check".format( len(payload_json['references']['spec']['tags']))) cmds = [['oc', 'image', 'info', '-o', 'json', tag['from']['name']] for tag in payload_json['references']['spec']['tags']] green_prefix("Querying image infos...") cmd_results = await asyncio.gather( *[exectools.cmd_gather_async(cmd) for cmd in cmds]) for image, cmd, cmd_result in zip( payload_json['references']['spec']['tags'], cmds, cmd_results): click.echo("----") image_name = image['name'] rc, stdout, stderr = cmd_result if rc != 0: # Probably no point in continuing.. can't contact brew? red_prefix("Unable to run oc image info: ") red_print(f"cmd={cmd!r}, out={stdout} ; err={stderr}") exit(1) image_info = json.loads(stdout) labels = image_info['config']['config']['Labels'] # The machine-os-content image doesn't follow the standard # pattern. We need to skip that image when we find it, it is # not attached to advisories. if 'com.coreos.ostree-commit' in labels: yellow_prefix("Skipping machine-os-content image: ") click.echo("Not required for checks") continue component = labels['com.redhat.component'] n = image_name click.echo("Payload name: {}".format(n)) click.echo("Brew name: {}".format(component)) if labels: v = labels['version'] r = labels['release'] all_payload_nvrs[component] = "{}-{}".format(v, r) else: print("For image {} Labels doesn't exist, image_info: {}".format( image_name, image_info)) missing_in_errata = {} payload_doesnt_match_errata = {} in_pending_advisory = [] in_shipped_advisory = [] output = { 'missing_in_advisory': missing_in_errata, 'payload_advisory_mismatch': payload_doesnt_match_errata, "in_pending_advisory": in_pending_advisory, "in_shipped_advisory": in_shipped_advisory, } green_prefix("Analyzing data: ") click.echo("{} images to consider from payload".format( len(all_payload_nvrs))) for image, vr in all_payload_nvrs.items(): yellow_prefix("Cross-checking from payload: ") click.echo("{}-{}".format(image, vr)) if image not in all_advisory_nvrs: missing_in_errata[image] = "{}-{}".format(image, vr) click.echo("{} in payload not found in advisory".format( "{}-{}".format(image, vr))) elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]: click.echo( "{} from payload has version {} which does not match {} from advisory" .format(image, vr, all_advisory_nvrs[image])) payload_doesnt_match_errata[image] = { 'payload': vr, 'errata': all_advisory_nvrs[image] } if missing_in_errata: # check if missing images are already shipped or pending to ship advisory_nvrs: Dict[int, List[str]] = { } # a dict mapping advisory numbers to lists of NVRs green_print( f"Checking if {len(missing_in_errata)} missing images are shipped..." ) for nvr in missing_in_errata.copy().values(): # get the list of advisories that this build has been attached to build = elliottlib.errata.get_brew_build(nvr) # filter out dropped advisories advisories = [ ad for ad in build.all_errata if ad["status"] != "DROPPED_NO_SHIP" ] if not advisories: red_print(f"Build {nvr} is not attached to any advisories.") continue for advisory in advisories: if advisory["status"] == "SHIPPED_LIVE": green_print( f"Missing build {nvr} has been shipped with advisory {advisory}." ) else: yellow_print( f"Missing build {nvr} is in another pending advisory.") advisory_nvrs.setdefault(advisory["id"], []).append(nvr) name = nvr.rsplit("-", 2)[0] del missing_in_errata[name] if advisory_nvrs: click.echo( f"Getting information of {len(advisory_nvrs)} advisories...") for advisory, nvrs in advisory_nvrs.items(): advisory_obj = elliottlib.errata.get_raw_erratum(advisory) adv_type, adv_info = next(iter(advisory_obj["errata"].items())) item = { "id": advisory, "type": adv_type.upper(), "url": elliottlib.constants.errata_url + f"/{advisory}", "summary": adv_info["synopsis"], "state": adv_info["status"], "nvrs": nvrs, } if adv_info["status"] == "SHIPPED_LIVE": in_shipped_advisory.append(item) else: in_pending_advisory.append(item) green_print("Summary results:") click.echo(json.dumps(output, indent=4)) with open('summary_results.json', 'w') as fp: json.dump(output, fp, indent=4) green_prefix("Wrote out summary results: ") click.echo("summary_results.json")
def change_state_cli(runtime, state, advisory, default_advisories, default_advisory_type, noop): """Change the state of an ADVISORY. Additional permissions may be required to change an advisory to certain states. An advisory may not move between some states until all criteria have been met. For example, an advisory can not move from NEW_FILES to QE unless Bugzilla Bugs or JIRA Issues have been attached. NOTE: The two advisory options are mutually exclusive and can not be used together. See the find-bugs help for additional information on adding Bugs. Move assembly release advisories to QE $ elliott -g openshift-4.10 --assembly 4.10.4 change-state -s QE Move group release advisories to QE: $ elliott -g openshift-4.5 change-state -s QE --default-advisories Move the advisory 123456 to QE: $ elliott change-state --state QE --advisory 123456 Move the advisory 123456 back to NEW_FILES (short option flag): $ elliott change-state -s NEW_FILES -a 123456 Do not actually change state, just check that the command could have ran (for example, when testing out pipelines) $ elliott change-state -s NEW_FILES -a 123456 --noop """ count_flags = sum( map(bool, [advisory, default_advisory_type, default_advisories])) if count_flags > 1: raise click.BadParameter( "Use only one of --use-default-advisory or --advisory or --default-advisories" ) runtime.initialize(no_group=bool(advisory)) advisories = [] if default_advisory_type is not None: advisory = find_default_advisory(runtime, default_advisory_type) if advisory: advisories.append(advisory) if not advisories: advisories = list(runtime.group_config.advisories.values()) click.echo(f"Attempting to move advisories {advisories} to {state}") errors = [] for advisory in advisories: try: e = Erratum(errata_id=advisory) if e.errata_state == state: green_prefix(f"No Change ({advisory}): ") click.echo(f"Target state is same as current state: {state}") # we have 5 different states we can only change the state if it's in NEW_FILES or QE # "NEW_FILES", # "QE", # "REL_PREP", # "PUSH_READY", # "IN_PUSH" elif e.errata_state != 'NEW_FILES' and e.errata_state != 'QE': red_prefix(f"Error ({advisory}): ") if default_advisory_type is not None: click.echo( f"Could not change '{e.errata_state}', group.yml is probably pointing at old one" ) else: click.echo( f"Can only change the state if it's in NEW_FILES or QE, current state is {e.errata_state}" ) else: if noop: green_prefix(f"NOOP ({advisory}): ") click.echo( f"Would have changed state {e.errata_state} ➔ {state}") else: # Capture current state because `e.commit()` will # refresh the `e.errata_state` attribute old_state = e.errata_state e.setState(state) e.commit() green_prefix(f"Changed state ({advisory}): ") click.echo(f"{old_state} ➔ {state}") except ErrataException as ex: click.echo(f'Error fetching/changing state of {advisory}: {ex}') errors.append(ex) if errors: raise Exception(errors)
def verify_cvp_cli(runtime: Runtime, all_images, nvrs, optional_checks, all_optional_checks, fix, message): """ Verify CVP test results Example 1: Verify CVP test results for all latest 4.4 image builds, also warn those with failed content_set_check $ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check Example 2: Apply patches to ocp-build-data to fix the redundant content sets error: $ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check --fix Note: 1. If `--message` is not given, `--fix` will leave changed ocp-build-data files uncommitted. 2. Make sure your ocp-build-data directory is clean before running `--fix`. """ if bool(all_images) + bool(nvrs) != 1: raise click.BadParameter('You must use one of --all or --build.') if all_optional_checks and optional_checks: raise click.BadParameter( 'Use only one of --all-optional-checks or --include-optional-check.' ) runtime.initialize(mode='images') tag_pv_map = runtime.gitdata.load_data( key='erratatool', replace_vars=runtime.group_config.vars.primitive() if runtime.group_config.vars else {}).data.get('brew_tag_product_version_mapping') brew_session = koji.ClientSession(runtime.group_config.urls.brewhub or constants.BREW_HUB) builds = [] if all_images: runtime.logger.info("Getting latest image builds from Brew...") builds = get_latest_image_builds(brew_session, tag_pv_map.keys(), runtime.image_metas) elif nvrs: runtime.logger.info(f"Finding {len(builds)} builds from Brew...") builds = brew.get_build_objects(nvrs, brew_session) runtime.logger.info(f"Found {len(builds)} image builds.") resultsdb_api = ResultsDBAPI() nvrs = [b["nvr"] for b in builds] runtime.logger.info( f"Getting CVP test results for {len(builds)} image builds...") latest_cvp_results = get_latest_cvp_results(runtime, resultsdb_api, nvrs) # print a summary for all CVP results good_results = [] # good means PASSED or INFO bad_results = [] # bad means NEEDS_INSPECTION or FAILED incomplete_nvrs = [] for nvr, result in zip(nvrs, latest_cvp_results): if not result: incomplete_nvrs.append(nvr) continue outcome = result.get( "outcome" ) # only PASSED, FAILED, INFO, NEEDS_INSPECTION are now valid outcome values (https://resultsdb20.docs.apiary.io/#introduction/changes-since-1.0) if outcome in {"PASSED", "INFO"}: good_results.append(result) elif outcome in {"NEEDS_INSPECTION", "FAILED"}: bad_results.append(result) green_prefix("good: {}".format(len(good_results))) click.echo(", ", nl=False) red_prefix("bad: {}".format(len(bad_results))) click.echo(", ", nl=False) yellow_print("incomplete: {}".format(len(incomplete_nvrs))) if bad_results: red_print("The following builds didn't pass CVP tests:") for r in bad_results: nvr = r["data"]["item"][0] red_print(f"{nvr} {r['outcome']}: {r['ref_url']}") if incomplete_nvrs: yellow_print( "We couldn't find CVP test results for the following builds:") for nvr in incomplete_nvrs: yellow_print(nvr) if not optional_checks and not all_optional_checks: return # no need to print failed optional CVP checks # Find failed optional CVP checks in case some of the tiem *will* become required. optional_checks = set(optional_checks) complete_results = good_results + bad_results runtime.logger.info( f"Getting optional checks for {len(complete_results)} CVP tests...") optional_check_results = get_optional_checks(runtime, complete_results) component_distgit_keys = { } # a dict of brew component names to distgit keys content_set_repo_names = { } # a map of x86_64 content set names to group.yml repo names if fix: # Fixing redundant content sets requires those dicts for image in runtime.image_metas(): component_distgit_keys[ image.get_component_name()] = image.distgit_key for repo_name, repo_info in runtime.group_config.get("repos", {}).items(): content_set_name = repo_info.get( 'content_set', {}).get('x86_64') or repo_info.get( 'content_set', {}).get('default') if content_set_name: content_set_repo_names[content_set_name] = repo_name ocp_build_data_updated = False for cvp_result, checks in zip(complete_results, optional_check_results): # example optional checks: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/hive-container-v4.6.0-202008010302.p0/da01e36c-8c69-4a19-be7d-ba4593a7b085/sanity-tests-optional-results.json bad_checks = [ check for check in checks["checks"] if check["status"] != "PASS" and ( all_optional_checks or check["name"] in optional_checks) ] if not bad_checks: continue nvr = cvp_result["data"]["item"][0] yellow_print("----------") yellow_print( f"Build {nvr} has {len(bad_checks)} problematic CVP optional checks:" ) for check in bad_checks: yellow_print(f"* {check['name']} {check['status']}") if fix and check["name"] == "content_set_check": if "Some content sets are redundant." in check["logs"]: # fix redundant content sets name = nvr.rsplit('-', 2)[0] distgit_keys = component_distgit_keys.get(name) if not distgit_keys: runtime.logger.warning( f"Will not apply the redundant content sets fix to image {name}: We don't know its distgit key." ) continue amd64_content_sets = list( filter(lambda item: item.get("arch") == "amd64", check["logs"][-1]) ) # seems only x86_64 (amd64) content sets are defined in ocp-build-data. if not amd64_content_sets: runtime.logger.warning( f"Will not apply the redundant content sets fix to image {name}: It doesn't have redundant x86_64 (amd64) content sets" ) continue amd64_redundant_cs = amd64_content_sets[0]["redundant_cs"] redundant_repos = [ content_set_repo_names[cs] for cs in amd64_redundant_cs if cs in content_set_repo_names ] if len(redundant_repos) != len(amd64_redundant_cs): runtime.logger.error( f"Not all content sets have a repo entry in group.yml: #content_sets is {len(amd64_redundant_cs)}, #repos is {len(redundant_repos)}" ) runtime.logger.info( f"Applying redundant content sets fix to {distgit_keys}..." ) fix_redundant_content_set(runtime, distgit_keys, redundant_repos) ocp_build_data_updated = True runtime.logger.info( f"Fixed redundant content sets for {distgit_keys}") yellow_print( f"See {cvp_result['ref_url']}sanity-tests-optional-results.json for more details." ) if message and ocp_build_data_updated: runtime.gitdata.commit(message)
async def verify_cvp_cli(runtime: Runtime, all_images, nvrs, optional_checks, all_optional_checks, fix, message): """ Verify CVP test results Example 1: Verify CVP test results for all latest 4.4 image builds, also warn those with failed content_set_check $ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check Example 2: Apply patches to ocp-build-data to fix the redundant content sets error: $ elliott --group openshift-4.4 verify-cvp --all --include-optional-check content_set_check --fix Note: 1. If `--message` is not given, `--fix` will leave changed ocp-build-data files uncommitted. 2. Make sure your ocp-build-data directory is clean before running `--fix`. """ if bool(all_images) + bool(nvrs) != 1: raise click.BadParameter('You must use one of --all or --build.') if all_optional_checks and optional_checks: raise click.BadParameter('Use only one of --all-optional-checks or --include-optional-check.') runtime.initialize(mode='images') brew_session = koji.ClientSession(runtime.group_config.urls.brewhub or constants.BREW_HUB) builds = [] if all_images: image_metas = runtime.image_metas() builds = await get_latest_image_builds(image_metas) elif nvrs: runtime.logger.info(f"Finding {len(builds)} builds from Brew...") builds = brew.get_build_objects(nvrs, brew_session) runtime.logger.info(f"Found {len(builds)} image builds.") resultsdb_api = ResultsDBAPI() nvrs = [b["nvr"] for b in builds] runtime.logger.info(f"Getting CVP test results for {len(builds)} image builds...") latest_cvp_results = await get_latest_cvp_results(runtime, resultsdb_api, nvrs) # print a summary for all CVP results good_results = [] # good means PASSED or INFO bad_results = [] # bad means NEEDS_INSPECTION or FAILED incomplete_nvrs = [] for nvr, result in zip(nvrs, latest_cvp_results): if not result: incomplete_nvrs.append(nvr) continue outcome = result.get("outcome") # only PASSED, FAILED, INFO, NEEDS_INSPECTION are now valid outcome values (https://resultsdb20.docs.apiary.io/#introduction/changes-since-1.0) if outcome in {"PASSED", "INFO"}: good_results.append(result) elif outcome in {"NEEDS_INSPECTION", "FAILED"}: bad_results.append(result) green_prefix("good: {}".format(len(good_results))) click.echo(", ", nl=False) red_prefix("bad: {}".format(len(bad_results))) click.echo(", ", nl=False) yellow_print("incomplete: {}".format(len(incomplete_nvrs))) if bad_results: red_print("The following builds didn't pass CVP tests:") for r in bad_results: nvr = r["data"]["item"][0] red_print(f"{nvr} {r['outcome']}: {r['ref_url']}") if incomplete_nvrs: yellow_print("We couldn't find CVP test results for the following builds:") for nvr in incomplete_nvrs: yellow_print(nvr) if not optional_checks and not all_optional_checks: return # no need to print failed optional CVP checks # Find failed optional CVP checks in case some of the tiem *will* become required. optional_checks = set(optional_checks) complete_results = good_results + bad_results runtime.logger.info(f"Getting optional checks for {len(complete_results)} CVP tests...") optional_check_results = await get_optional_checks(runtime, complete_results) component_distgit_keys = {} # a dict of brew component names to distgit keys content_set_to_repo_names = {} # a map of content set names to group.yml repo names for image in runtime.image_metas(): component_distgit_keys[image.get_component_name()] = image.distgit_key for repo_name, repo_info in runtime.group_config.get("repos", {}).items(): for arch, cs_name in repo_info.get('content_set', {}).items(): if arch == "optional": continue # not a real arch name content_set_to_repo_names[cs_name] = repo_name nvr_to_builds = {build["nvr"]: build for build in builds} ocp_build_data_updated = False failed_with_not_covered_rpms = set() failed_with_redundant_repos = set() only_failed_in_non_x86_with_not_covered_rpms = set() only_failed_in_non_x86_with_redundant_repos = set() for cvp_result, checks in zip(complete_results, optional_check_results): # example optional checks: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/hive-container-v4.6.0-202008010302.p0/da01e36c-8c69-4a19-be7d-ba4593a7b085/sanity-tests-optional-results.json bad_checks = [check for check in checks["checks"] if check["status"] != "PASS" and (all_optional_checks or check["name"] in optional_checks)] if not bad_checks: continue nvr = cvp_result["data"]["item"][0] build = nvr_to_builds[nvr] yellow_print("----------") yellow_print(f"Build {nvr} (https://brewweb.engineering.redhat.com/brew/buildinfo?buildID={nvr_to_builds[nvr]['id']}) has {len(bad_checks)} problematic CVP optional checks:") for check in bad_checks: yellow_print(f"* {check['name']} {check['status']}") try: amd64_result = list(filter(lambda item: item.get("arch") == "amd64", check["logs"][-1])) except AttributeError: red_print("CVP result malformed.") if len(amd64_result) != 1: red_print("WHAT?! This build doesn't include an amd64 image? This shouldn't happen. Check Brew and CVP logs with the CVP team!") continue amd64_result = amd64_result[0] image_component_name = nvr.rsplit('-', 2)[0] distgit_key = component_distgit_keys.get(image_component_name) amd64_redundant_cs = amd64_result.get("redundant_cs", []) amd64_redundant_repos = {content_set_to_repo_names[cs] for cs in amd64_redundant_cs} def _strip_arch_suffix(rpm): # rh-nodejs10-3.2-3.el7.x86_64 -> rh-nodejs10-3.2-3.el7 rpm_split = rpm.rsplit(".", 1) return rpm_split[0] amd64_not_covered_rpms = {_strip_arch_suffix(rpm) for rpm in amd64_result.get("not_covered_rpms", [])} if check["name"] == "content_set_check": details = check["logs"][-1] # example: http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/logging-fluentd-container-v4.6.0-202008261251.p0/dd9f2024-5440-4f33-b508-472ccf258439/sanity-tests-optional-results.json if not details: red_print("content_set_check failed without any explanation. Report to CVP team!") continue if len(details) > 1: # if this build is multi-arch, check if all per-arch results are consistent for result in details: if result["arch"] == "amd64": continue redundant_repos = {content_set_to_repo_names[cs] for cs in result.get("redundant_cs", [])} if redundant_repos != amd64_redundant_repos: only_failed_in_non_x86_with_redundant_repos.add(nvr) red_print(f"""content_set_check for {nvr} arch {result["arch"]} has different redundant_cs result from the one for amd64: {result["arch"]} has redundant_cs {result.get("redundant_cs")}, but amd64 has redundant_cs {amd64_redundant_cs}. Not sure what happened. Please see Brew and CVP logs and/or check with the CVP team.""") not_covered_rpms = {_strip_arch_suffix(rpm) for rpm in result.get("not_covered_rpms", [])} if not_covered_rpms != amd64_not_covered_rpms: only_failed_in_non_x86_with_not_covered_rpms.add(nvr) red_print(f"""content_set_check for {nvr} arch {result["arch"]} has different not_covered_rpms result from the one for amd64: {result["arch"]} has extra not_covered_rpms {not_covered_rpms - amd64_not_covered_rpms}, and missing not_covered_rpms {amd64_not_covered_rpms - not_covered_rpms}. Not sure what happened. Check Brew and CVP logs with the CVP team!""") if amd64_not_covered_rpms: # This build has not_covered_rpms failed_with_not_covered_rpms.add(nvr) yellow_print(f"Image {distgit_key} has not_covered_rpms: {amd64_not_covered_rpms}") brew_repos = await find_repos_for_rpms(amd64_not_covered_rpms, build) yellow_print(f"Those repos shown in Brew logs might be a good hint: {brew_repos}") runtime.logger.info("Looking for parent image's content_sets...") parent = get_parent_build_ids([build])[0] if parent: parent_build = brew.get_build_objects([parent])[0] parent_cs = await get_content_sets_for_build(parent_build) parent_enabled_repos = {content_set_to_repo_names[cs] for cs in parent_cs.get("x86_64", [])} enabled_repos = set(runtime.image_map[distgit_key].config.get("enabled_repos", [])) missing_repos = parent_enabled_repos - enabled_repos yellow_print(f"""The following repos are defined in parent {parent_build["nvr"]} {component_distgit_keys.get(parent_build["name"], "?")}.yml but not in {component_distgit_keys[build["name"]]}.yml: {missing_repos}""") if fix and missing_repos: runtime.logger.info("Trying to merge parent image's content_sets...") fix_missing_content_set(runtime, distgit_key, missing_repos) ocp_build_data_updated = True runtime.logger.info(f"{distgit_key}.yml patched") if amd64_redundant_repos: # This build has redundant_cs failed_with_redundant_repos.add(nvr) yellow_print(f"Image {distgit_key} has redundant repos: {amd64_redundant_repos}") if not fix: yellow_print(f"Please add the following repos to non_shipping_repos in {distgit_key}.yml: {amd64_redundant_repos}") else: runtime.logger.info(f"Applying redundant content sets fix to {distgit_key}.yml...") fix_redundant_content_set(runtime, distgit_key, amd64_redundant_repos) ocp_build_data_updated = True runtime.logger.info(f"{distgit_key}.yml patched") print(f"See {cvp_result['ref_url']}sanity-tests-optional-results.json for more details.") if failed_with_not_covered_rpms or failed_with_redundant_repos: yellow_print(f"{len(failed_with_not_covered_rpms | failed_with_redundant_repos)} images failed content_sets.\n Where") if failed_with_not_covered_rpms: yellow_print(f"\t{len(failed_with_not_covered_rpms)} images failed content_sets check because of not_covered_rpms:") for rpm in failed_with_not_covered_rpms: line = f"\t\t{rpm}" if rpm in only_failed_in_non_x86_with_not_covered_rpms: line += " - non-x86 arches are different from x86 one" yellow_print(line) if failed_with_redundant_repos: yellow_print(f"\t{len(failed_with_redundant_repos)} images failed content_sets check because of redundant_repos:") for rpm in failed_with_redundant_repos: line = f"\t\t{rpm}" if rpm in only_failed_in_non_x86_with_redundant_repos: line += " - non-x86 arches are different from x86 one" yellow_print(line) if message and ocp_build_data_updated: runtime.gitdata.commit(message)
async def verify_payload(ctx, payload, advisory): """Cross-check that the builds present in PAYLOAD match the builds attached to ADVISORY. The payload is treated as the source of truth. If something is absent or different in the advisory it is treated as an error with the advisory. \b PAYLOAD - Full pullspec of the payload to verify ADVISORY - Numerical ID of the advisory Two checks are made: \b 1. Missing in Advisory - No payload components are absent from the given advisory 2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory Results are summarily printed at the end of the run. They are also written out to summary_results.json. Verify builds in the given payload match the builds attached to advisory 41567 \b $ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567 """ try: green_prefix("Fetching advisory builds: ") click.echo("Advisory - {}".format(advisory)) builds = elliottlib.errata.get_builds(advisory) except GSSError: exit_unauthenticated() except elliottlib.exceptions.ErrataToolError as ex: raise ElliottFatalError(getattr(ex, 'message', repr(ex))) all_advisory_nvrs = {} # Results come back with top level keys which are brew tags green_prefix("Looping over tags: ") click.echo("{} tags to check".format(len(builds))) for tag in builds.keys(): # Each top level has a key 'builds' which is a list of dicts green_prefix("Looping over builds in tag: ") click.echo("{} with {} builds".format(tag, len(builds[tag]['builds']))) for build in builds[tag]['builds']: # Each dict has a top level key which might be the actual # 'nvr' but I don't have enough data to know for sure # yet. Also I don't know when there might be more than one # key in the build dict. We'll loop over it to be sure. for name in build.keys(): n, v, r = name.rsplit('-', 2) version_release = "{}-{}".format(v, r) all_advisory_nvrs[n] = version_release click.echo("Found {} builds".format(len(all_advisory_nvrs))) all_payload_nvrs = {} click.echo("Fetching release info") release_export_cmd = 'oc adm release info {} -o json'.format(payload) rc, stdout, stderr = exectools.cmd_gather(release_export_cmd) if rc != 0: # Probably no point in continuing.. can't contact brew? print("Unable to run oc release info: out={} ; err={}".format( stdout, stderr)) exit(1) else: click.echo("Got release info") payload_json = json.loads(stdout) green_prefix("Looping over payload images: ") click.echo("{} images to check".format( len(payload_json['references']['spec']['tags']))) cmds = [['oc', 'image', 'info', '-o', 'json', tag['from']['name']] for tag in payload_json['references']['spec']['tags']] green_prefix("Querying image infos...") cmd_results = await asyncio.gather( *[exectools.cmd_gather_async(cmd) for cmd in cmds]) for image, cmd, cmd_result in zip( payload_json['references']['spec']['tags'], cmds, cmd_results): click.echo("----") image_name = image['name'] rc, stdout, stderr = cmd_result if rc != 0: # Probably no point in continuing.. can't contact brew? red_prefix("Unable to run oc image info: ") red_print(f"cmd={cmd!r}, out={stdout} ; err={stderr}") exit(1) image_info = json.loads(stdout) labels = image_info['config']['config']['Labels'] # The machine-os-content image doesn't follow the standard # pattern. We need to skip that image when we find it, it is # not attached to advisories. if 'com.coreos.ostree-commit' in labels: yellow_prefix("Skipping machine-os-content image: ") click.echo("Not required for checks") continue component = labels['com.redhat.component'] n = image_name click.echo("Payload name: {}".format(n)) click.echo("Brew name: {}".format(component)) if labels: v = labels['version'] r = labels['release'] all_payload_nvrs[component] = "{}-{}".format(v, r) else: print("For image {} Labels doesn't exist, image_info: {}".format( image_name, image_info)) missing_in_errata = {} payload_doesnt_match_errata = {} in_other_advisories = {} output = { 'missing_in_advisory': missing_in_errata, 'payload_advisory_mismatch': payload_doesnt_match_errata, "in_other_advisories": in_other_advisories, } green_prefix("Analyzing data: ") click.echo("{} images to consider from payload".format( len(all_payload_nvrs))) for image, vr in all_payload_nvrs.items(): yellow_prefix("Cross-checking from payload: ") click.echo("{}-{}".format(image, vr)) if image not in all_advisory_nvrs: missing_in_errata[image] = "{}-{}".format(image, vr) click.echo("{} in payload not found in advisory".format( "{}-{}".format(image, vr))) elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]: click.echo( "{} from payload has version {} which does not match {} from advisory" .format(image, vr, all_advisory_nvrs[image])) payload_doesnt_match_errata[image] = { 'payload': vr, 'errata': all_advisory_nvrs[image] } if missing_in_errata: green_print( f"Checking if {len(missing_in_errata)} missing images are shipped..." ) nvrs = list(missing_in_errata.values()) tag_lists = elliottlib.brew.get_builds_tags(nvrs) for nvr, tags in zip(nvrs, tag_lists): name = nvr.rsplit("-", 2)[0] if any(map(lambda tag: tag["name"].endswith('-released'), tags)): green_print(f"Build {nvr} is shipped. Skipping...") del missing_in_errata[name] elif any(map(lambda tag: tag["name"].endswith('-pending'), tags)): green_print(f"Build {nvr} is in another advisory.") del missing_in_errata[name] in_other_advisories[name] = nvr green_print("Summary results:") click.echo(json.dumps(output, indent=4)) with open('summary_results.json', 'w') as fp: json.dump(output, fp, indent=4) green_prefix("Wrote out summary results: ") click.echo("summary_results.json")
def find_bugs_cli(runtime, advisory, default_advisory_type, mode, check_builds, status, exclude_status, id, cve_trackers, from_diff, flag, report, into_default_advisories, noop): """Find Red Hat Bugzilla bugs or add them to ADVISORY. Bugs can be "swept" into the advisory either automatically (--mode sweep), or by manually specifying one or more bugs using --mode list with the --id option. Use cases are described below: Note: Using --id without --add is basically pointless SWEEP: For this use-case the --group option MUST be provided. The --group automatically determines the correct target-releases to search for bugs claimed to be fixed, but not yet attached to advisories. --check-builds flag forces bug validation with attached builds to rpm advisory. It assumes builds have been attached and only attaches bugs with matching builds. default --status: ['MODIFIED', 'ON_QA', 'VERIFIED'] LIST: The --group option is not required if you are specifying bugs manually. Provide one or more --id's for manual bug addition. In LIST mode you must provide a list of IDs to attach with the --id option. DIFF: For this use case, you must provide the --between option using two URLs to payloads. QE: Find MODIFIED bugs for the target-releases, and set them to ON_QA. The --group option MUST be provided. Cannot be used in combination with --add, --use-default-advisory, --into-default-advisories, --exclude-status. BLOCKER: List active blocker+ bugs for the target-releases. The --group option MUST be provided. Cannot be used in combination with --add, --use-default-advisory, --into-default-advisories. default --status: ['NEW', 'ASSIGNED', 'POST', 'MODIFIED', 'ON_DEV', 'ON_QA'] Use --exclude_status to filter out from default status list. By default --cve-trackers is True. Using --use-default-advisory without a value set for the matching key in the build-data will cause an error and elliott will exit in a non-zero state. Use of this option silently overrides providing an advisory with the --add option. Automatically add bugs with target-release matching 3.7.Z or 3.7.0 to advisory 123456: \b $ elliott --group openshift-3.7 find-bugs --mode sweep --add 123456 List bugs that WOULD be added to an advisory and have set the bro_ok flag on them (NOOP): \b $ elliott --group openshift-3.7 find-bugs --mode sweep --flag bro_ok Attach bugs to their correct default advisories, e.g. operator-related bugs go to "extras" instead of the default "image": \b $ elliott --group=openshift-4.4 find-bugs --mode=sweep --into-default-advisories Add two bugs to advisory 123456. Note that --group is not required because we're not auto searching: \b $ elliott find-bugs --mode list --id 8675309 --id 7001337 --add 123456 Automatically find bugs for openshift-4.1 and attach them to the rpm advisory defined in ocp-build-data: \b $ elliott --group=openshift-4.1 --mode sweep --use-default-advisory rpm Find bugs for 4.6 that are in MODIFIED state, and set them to ON_QA: \b $ elliott --group=openshift-4.6 --mode qe \b $ elliott --group=openshift-4.6 --mode blocker --report """ count_advisory_attach_flags = sum( map(bool, [advisory, default_advisory_type, into_default_advisories])) if mode != 'list' and len(id) > 0: raise click.BadParameter( "Combining the automatic and manual bug attachment options is not supported" ) if mode == 'list' and len(id) == 0: raise click.BadParameter( "When using mode=list, you must provide a list of bug IDs") if mode == 'list' and into_default_advisories: raise click.BadParameter( "Cannot use --into-default-advisories with mode=list") if mode == 'diff' and not len(from_diff) == 2: raise click.BadParameter( "If using mode=diff, you must provide two payloads to compare") if count_advisory_attach_flags > 1: raise click.BadParameter( "Use only one of --use-default-advisory, --add, or --into-default-advisories" ) if mode in ['qe', 'blocker'] and count_advisory_attach_flags > 0: raise click.BadParameter( "Mode does not operate on an advisory. Do not specify any of " "`--use-default-advisory`, `--add`, or `--into-default-advisories`" ) runtime.initialize() bz_data = runtime.gitdata.load_data(key='bugzilla').data bzapi = bzutil.get_bzapi(bz_data) # filter out bugs ART does not manage m = re.match( r"rhaos-(\d+).(\d+)", runtime.branch ) # extract OpenShift version from the branch name. there should be a better way... if not m: raise ElliottFatalError( f"Unable to determine OpenShift version from branch name {runtime.branch}." ) major_version = int(m[1]) minor_version = int(m[2]) if default_advisory_type is not None: advisory = find_default_advisory(runtime, default_advisory_type) if mode in ['sweep', 'qe', 'blocker']: if not cve_trackers: if mode == 'blocker': cve_trackers = True else: cve_trackers = False if not status: # use default status filter according to mode if mode == 'sweep': status = ['MODIFIED', 'ON_QA', 'VERIFIED'] if mode == 'qe': status = ['MODIFIED'] if mode == 'blocker': status = [ 'NEW', 'ASSIGNED', 'POST', 'MODIFIED', 'ON_DEV', 'ON_QA' ] if mode != 'qe' and exclude_status: status = set(status) - set(exclude_status) green_prefix( f"Searching for bugs with status {' '.join(status)} and target release(s):" ) click.echo(" {tr}".format(tr=", ".join(bz_data['target_release']))) search_flag = 'blocker+' if mode == 'blocker' else None bugs = bzutil.search_for_bugs( bz_data, status, flag=search_flag, filter_out_security_bugs=not (cve_trackers), verbose=runtime.debug) elif mode == 'list': bugs = [bzapi.getbug(i) for i in cli_opts.id_convert(id)] mode_list(advisory=advisory, bugs=bugs, flags=flag, report=report, noop=noop) return elif mode == 'diff': click.echo(runtime.working_dir) bug_id_strings = openshiftclient.get_bug_list(runtime.working_dir, from_diff[0], from_diff[1]) bugs = [bzapi.getbug(i) for i in bug_id_strings] filtered_bugs = filter_bugs(bugs, major_version, minor_version, runtime) green_prefix( f"Found {len(filtered_bugs)} bugs ({len(bugs) - len(filtered_bugs)} ignored): " ) bugs = filtered_bugs click.echo(", ".join(sorted(str(b.bug_id) for b in bugs))) if mode == 'qe': for bug in bugs: bzutil.set_state(bug, 'ON_QA', noop=noop) if len(flag) > 0: add_flags(bugs=bugs, flags=flag, noop=noop) if report: print_report(bugs) if advisory and not default_advisory_type: # `--add ADVISORY_NUMBER` should respect the user's wish and attach all available bugs to whatever advisory is specified. errata.add_bugs_with_retry(advisory, bugs, noop=noop) return # If --use-default-advisory or --into-default-advisories is given, we need to determine which bugs should be swept into which advisory. # Otherwise we don't need to sweep bugs at all. if not (into_default_advisories or default_advisory_type): return # key is impetus ("rpm", "image", "extras"), value is a set of bug IDs. impetus_bugs = {"rpm": set(), "image": set(), "extras": set()} # @lmeyer: simple and stupid would still be keeping the logic in python, # possibly with config flags for branched logic. # until that logic becomes too ugly to keep in python, i suppose.. if major_version < 4: # for 3.x, all bugs should go to the rpm advisory impetus_bugs["rpm"] = set(bugs) else: # for 4.x # sweep rpm cve trackers into "rpm" advisory rpm_bugs = dict() if mode == 'sweep' and cve_trackers: rpm_bugs = bzutil.get_valid_rpm_cves(bugs) green_prefix("RPM CVEs found: ") click.echo(sorted(b.id for b in rpm_bugs)) if rpm_bugs: # if --check-builds flag is set # only attach bugs that have corresponding brew builds attached to rpm advisory if check_builds: click.echo( "Validating bugs with builds attached to the rpm advisory" ) attached_builds = errata.get_advisory_nvrs( runtime.group_config.advisories["rpm"]) packages = attached_builds.keys() not_found = [] for bug, package_name in rpm_bugs.items(): if package_name not in packages: not_found.append((bug.id, package_name)) else: click.echo( f"Build found for #{bug.id}, {package_name}") impetus_bugs["rpm"].add(bug) if not_found: red_prefix("RPM CVE Warning: ") click.echo( "The following CVE (bug, package) were found but not attached, because no corresponding brew builds were found attached to the rpm advisory. First attach builds and then rerun to attach the bugs" ) click.echo(not_found) else: click.echo( "Skipping attaching RPM CVEs. Use --check-builds flag to validate with builds." ) # optional operators bugs should be swept to the "extras" advisory # a way to identify operator-related bugs is by its "Component" value. # temporarily hardcode here until we need to move it to ocp-build-data. extra_components = { "Logging", "Service Brokers", "Metering Operator", "Node Feature Discovery Operator" } # we will probably find more impetus_bugs["extras"] = { b for b in bugs if b.component in extra_components } # all other bugs should go into "image" advisory impetus_bugs["image"] = set( bugs) - impetus_bugs["extras"] - rpm_bugs.keys() if default_advisory_type and impetus_bugs.get(default_advisory_type): errata.add_bugs_with_retry(advisory, impetus_bugs[default_advisory_type], noop=noop) elif into_default_advisories: for impetus, bugs in impetus_bugs.items(): if bugs: green_prefix(f'{impetus} advisory: ') errata.add_bugs_with_retry( runtime.group_config.advisories[impetus], bugs, noop=noop)