def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, builds,
kind, from_diff, as_json, allow_attached, remove, clean,
no_cdn_repos, payload, non_payload):
'''Automatically or manually find or attach/remove viable rpm or image builds
to ADVISORY. Default behavior searches Brew for viable builds in the
given group. Provide builds manually by giving one or more --build
(-b) options. Manually provided builds are verified against the Errata
Tool API.
\b
* Attach the builds to ADVISORY by giving --attach
* Remove the builds to ADVISORY by giving --remove
* Specify the build type using --kind KIND
Example: Assuming --group=openshift-3.7, then a build is a VIABLE
BUILD IFF it meets ALL of the following criteria:
\b
* HAS the tag in brew: rhaos-3.7-rhel7-candidate
* DOES NOT have the tag in brew: rhaos-3.7-rhel7
* IS NOT attached to ANY existing RHBA, RHSA, or RHEA
That is to say, a viable build is tagged as a "candidate", has NOT
received the "shipped" tag yet, and is NOT attached to any PAST or
PRESENT advisory. Here are some examples:
SHOW the latest OSE 3.6 image builds that would be attached to a
3.6 advisory:
$ elliott --group openshift-3.6 find-builds -k image
ATTACH the latest OSE 3.6 rpm builds to advisory 123456:
\b
$ elliott --group openshift-3.6 find-builds -k rpm --attach 123456
VERIFY (no --attach) that the manually provided RPM NVR and build
ID are viable builds:
$ elliott --group openshift-3.6 find-builds -k rpm -b megafrobber-1.0.1-2.el7 -a 93170
\b
Remove specific RPM NVR and build ID from advisory:
$ elliott --group openshift-4.3 find-builds -k image -b oauth-server-container-v4.3.22-202005212137 -a 55017 --remove
'''
if from_diff and builds:
raise click.BadParameter(
'Use only one of --build or --from-diff/--between.')
if clean and (remove or from_diff or builds):
raise click.BadParameter(
'Option --clean cannot be used with --build or --from-diff/--between.'
)
if not builds and remove:
raise click.BadParameter(
'Option --remove only support removing specific build with -b.')
if from_diff and kind != "image":
raise click.BadParameter(
'Option --from-diff/--between should be used with --kind/-k image.'
)
if advisory and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --attach')
if payload and non_payload:
raise click.BadParameter('Use only one of --payload or --non-payload.')
runtime.initialize(mode='images' if kind == 'image' else 'rpms')
replace_vars = runtime.group_config.vars.primitive(
) if runtime.group_config.vars else {}
et_data = runtime.gitdata.load_data(key='erratatool',
replace_vars=replace_vars).data
tag_pv_map = et_data.get('brew_tag_product_version_mapping')
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
ensure_erratatool_auth(
) # before we waste time looking up builds we can't process
unshipped_nvrps = []
unshipped_builds = []
to_remove = []
# get the builds we want to add
brew_session = runtime.build_retrying_koji_client(caching=True)
if builds:
green_prefix('Fetching builds...')
unshipped_nvrps = _fetch_nvrps_by_nvr_or_id(
builds,
tag_pv_map,
ignore_product_version=remove,
brew_session=brew_session)
elif clean:
unshipped_builds = errata.get_brew_builds(advisory)
elif from_diff:
unshipped_nvrps = _fetch_builds_from_diff(from_diff[0], from_diff[1],
tag_pv_map)
else:
if kind == 'image':
unshipped_nvrps = _fetch_builds_by_kind_image(
runtime, tag_pv_map, brew_session, payload, non_payload)
elif kind == 'rpm':
unshipped_nvrps = _fetch_builds_by_kind_rpm(
runtime, tag_pv_map, brew_session)
pbar_header('Fetching builds from Errata: ',
'Hold on a moment, fetching buildinfos from Errata Tool...',
unshipped_builds if clean else unshipped_nvrps)
if not clean and not remove:
# if is --clean then batch fetch from Erratum no need to fetch them individually
# if is not for --clean fetch individually using nvrp tuples then get specific
# elliottlib.brew.Build Objects by get_brew_build()
# e.g. :
# ('atomic-openshift-descheduler-container', 'v4.3.23', '202005250821', 'RHEL-7-OSE-4.3').
# Build(atomic-openshift-descheduler-container-v4.3.23-202005250821).
unshipped_builds = parallel_results_with_progress(
unshipped_nvrps, lambda nvrp: elliottlib.errata.get_brew_build(
'{}-{}-{}'.format(nvrp[0], nvrp[1], nvrp[2]),
nvrp[3],
session=requests.Session()))
if not allow_attached:
unshipped_builds = _filter_out_inviable_builds(
kind, unshipped_builds, elliottlib.errata)
_json_dump(as_json, unshipped_builds, kind, tag_pv_map)
if not unshipped_builds:
green_print('No builds needed to be attached.')
return
if not advisory:
click.echo('The following {n} builds '.format(n=len(unshipped_builds)),
nl=False)
if not (remove or clean):
click.secho('may be attached', bold=True, nl=False)
click.echo(' to an advisory:')
else:
click.secho('may be removed from', bold=True, nl=False)
click.echo(' from an advisory:')
for b in sorted(unshipped_builds):
click.echo(' ' + b.nvr)
return
if not unshipped_builds and not (remove and unshipped_nvrps):
# Do not change advisory state unless strictly necessary
return
try:
erratum = elliottlib.errata.Advisory(errata_id=advisory)
erratum.ensure_state('NEW_FILES')
if remove:
to_remove = [
f"{nvrp[0]}-{nvrp[1]}-{nvrp[2]}" for nvrp in unshipped_nvrps
]
elif clean:
to_remove = [b.nvr for b in unshipped_builds]
if to_remove:
erratum.remove_builds(to_remove)
else: # attach
erratum.attach_builds(unshipped_builds, kind)
cdn_repos = et_data.get('cdn_repos')
if cdn_repos and not no_cdn_repos and kind == "image":
erratum.set_cdn_repos(cdn_repos)
except GSSError:
exit_unauthenticated()
except ErrataException as e:
red_print(f'Cannot change advisory {advisory}: {e}')
exit(1)
def find_bugs_cli(runtime: Runtime, advisory, default_advisory_type, mode, check_builds, status, exclude_status, id, cve_trackers, from_diff,
flag, report, into_default_advisories, brew_event, noop):
"""Find Red Hat Bugzilla bugs or add them to ADVISORY. Bugs can be
"swept" into the advisory either automatically (--mode sweep), or by
manually specifying one or more bugs using --mode list with the --id option.
Use cases are described below:
Note: Using --id without --add is basically pointless
SWEEP: For this use-case the --group option MUST be provided. The
--group automatically determines the correct target-releases to search
for bugs claimed to be fixed, but not yet attached to advisories.
--check-builds flag forces bug validation with attached builds to rpm advisory.
It assumes builds have been attached and only attaches bugs with matching builds.
default --status: ['MODIFIED', 'ON_QA', 'VERIFIED']
LIST: The --group option is not required if you are specifying advisory
manually. Provide one or more --id's for manual bug addition. In LIST
mode you must provide a list of IDs to perform operation on with the --id option.
Supported operations: report with --report, attach with --attach and --into-default-advisories
DIFF: For this use case, you must provide the --between option using two
URLs to payloads.
QE: Find MODIFIED bugs for the target-releases, and set them to ON_QA.
The --group option MUST be provided. Cannot be used in combination
with --add, --use-default-advisory, --into-default-advisories, --exclude-status.
BLOCKER: List active blocker+ bugs for the target-releases.
The --group option MUST be provided. Cannot be used in combination
with --add, --use-default-advisory, --into-default-advisories.
default --status: ['NEW', 'ASSIGNED', 'POST', 'MODIFIED', 'ON_DEV', 'ON_QA']
Use --exclude_status to filter out from default status list.
By default --cve-trackers is True.
Using --use-default-advisory without a value set for the matching key
in the build-data will cause an error and elliott will exit in a
non-zero state. Use of this option silently overrides providing an
advisory with the --add option.
Automatically add bugs with target-release matching 3.7.Z or 3.7.0
to advisory 123456:
\b
$ elliott --group openshift-3.7 find-bugs --mode sweep --add 123456
List bugs that WOULD be added to an advisory and have set the bro_ok flag on them (NOOP):
\b
$ elliott --group openshift-3.7 find-bugs --mode sweep --flag bro_ok
Attach bugs to their correct default advisories, e.g. operator-related bugs go to "extras" instead of the default "image":
\b
$ elliott --group=openshift-4.4 find-bugs --mode=sweep --into-default-advisories
Add two bugs to advisory 123456. Note that --group is not required
because we're not auto searching:
\b
$ elliott find-bugs --mode list --id 8675309 --id 7001337 --add 123456
Add given list of bugs to the appropriate advisories. This would apply sweep logic to the given bugs
grouping them to be attached to rpm/extras/image advisories
\b
$ elliott -g openshift-4.8 find-bugs --mode list --id 8675309,7001337 --into-default-advisories
Automatically find bugs for openshift-4.1 and attach them to the
rpm advisory defined in ocp-build-data:
\b
$ elliott --group=openshift-4.1 --mode sweep --use-default-advisory rpm
Find bugs for 4.6 that are in MODIFIED state, and set them to ON_QA:
\b
$ elliott --group=openshift-4.6 --mode qe
\b
$ elliott --group=openshift-4.6 --mode blocker --report
"""
count_advisory_attach_flags = sum(map(bool, [advisory, default_advisory_type, into_default_advisories]))
if mode != 'list' and len(id) > 0:
raise click.BadParameter("Combining the automatic and manual bug attachment options is not supported")
if mode == 'list' and len(id) == 0:
raise click.BadParameter("When using mode=list, you must provide a list of bug IDs")
if mode == 'diff' and not len(from_diff) == 2:
raise click.BadParameter("If using mode=diff, you must provide two payloads to compare")
if count_advisory_attach_flags > 1:
raise click.BadParameter("Use only one of --use-default-advisory, --add, or --into-default-advisories")
if mode in ['qe', 'blocker'] and count_advisory_attach_flags > 0:
raise click.BadParameter("Mode does not operate on an advisory. Do not specify any of "
"`--use-default-advisory`, `--add`, or `--into-default-advisories`")
runtime.initialize(mode="both")
bz_data = runtime.gitdata.load_data(key='bugzilla').data
bzapi = bzutil.get_bzapi(bz_data)
# filter out bugs ART does not manage
m = re.match(r"rhaos-(\d+).(\d+)",
runtime.branch) # extract OpenShift version from the branch name. there should be a better way...
if not m:
raise ElliottFatalError(f"Unable to determine OpenShift version from branch name {runtime.branch}.")
major_version = int(m[1])
minor_version = int(m[2])
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
if mode in ['sweep', 'qe', 'blocker']:
if not cve_trackers:
if mode == 'blocker':
cve_trackers = True
else:
cve_trackers = False
if not status: # use default status filter according to mode
if mode == 'sweep':
status = ['MODIFIED', 'ON_QA', 'VERIFIED']
if mode == 'qe':
status = ['MODIFIED']
if mode == 'blocker':
status = ['NEW', 'ASSIGNED', 'POST', 'MODIFIED', 'ON_DEV', 'ON_QA']
if mode != 'qe' and exclude_status:
status = set(status) - set(exclude_status)
green_prefix(f"Searching for bugs with status {' '.join(status)} and target release(s):")
click.echo(" {tr}".format(tr=", ".join(bz_data['target_release'])))
search_flag = 'blocker+' if mode == 'blocker' else None
bugs = bzutil.search_for_bugs(bz_data, status, flag=search_flag, filter_out_security_bugs=not(cve_trackers),
verbose=runtime.debug)
sweep_cutoff_timestamp = 0
if brew_event:
green_print(f"Using command line specified cutoff event {runtime.assembly_basis_event}...")
sweep_cutoff_timestamp = runtime.build_retrying_koji_client().getEvent(brew_event)["ts"]
elif runtime.assembly_basis_event:
green_print(f"Determining approximate cutoff timestamp from basis event {runtime.assembly_basis_event}...")
brew_api = runtime.build_retrying_koji_client()
sweep_cutoff_timestamp = bzutil.approximate_cutoff_timestamp(runtime.assembly_basis_event, brew_api, runtime.rpm_metas() + runtime.image_metas())
if sweep_cutoff_timestamp:
green_print(f"Filtering bugs that have changed to one of the desired statuses before the cutoff time {datetime.utcfromtimestamp(sweep_cutoff_timestamp)}...")
qualified_bugs = bzutil.filter_bugs_by_cutoff_event(bzapi, bugs, status, sweep_cutoff_timestamp)
click.echo(f"{len(qualified_bugs)} of {len(bugs)} bugs are qualified for the cutoff time {datetime.utcfromtimestamp(sweep_cutoff_timestamp)}...")
bugs = qualified_bugs
# Loads included/excluded bugs from assembly config
issues_config = assembly_issues_config(runtime.get_releases_config(), runtime.assembly)
# JIRA issues are not supported yet. Only loads issues with integer IDs.
included_bug_ids: Set[int] = {int(issue["id"]) for issue in issues_config.include if isinstance(issue["id"], int) or issue["id"].isdigit()}
excluded_bug_ids: Set[int] = {int(issue["id"]) for issue in issues_config.exclude if isinstance(issue["id"], int) or issue["id"].isdigit()}
if included_bug_ids & excluded_bug_ids:
raise ValueError(f"The following bugs are defined in both 'include' and 'exclude': {included_bug_ids & excluded_bug_ids}")
if included_bug_ids:
yellow_print(f"The following bugs will be additionally included because they are explicitly defined in the assembly config: {included_bug_ids}")
included_bugs = bzapi.getbugs(included_bug_ids)
bugs.extend(included_bugs)
if excluded_bug_ids:
yellow_print(f"The following bugs will be excluded because they are explicitly defined in the assembly config: {excluded_bug_ids}")
bugs = [bug for bug in bugs if bug.id not in excluded_bug_ids]
elif mode == 'list':
bugs = [bzapi.getbug(i) for i in cli_opts.id_convert(id)]
if not into_default_advisories:
mode_list(advisory=advisory, bugs=bugs, flags=flag, report=report, noop=noop)
return
elif mode == 'diff':
click.echo(runtime.working_dir)
bug_id_strings = openshiftclient.get_bug_list(runtime.working_dir, from_diff[0], from_diff[1])
bugs = [bzapi.getbug(i) for i in bug_id_strings]
filtered_bugs = filter_bugs(bugs, major_version, minor_version, runtime)
green_prefix(f"Found {len(filtered_bugs)} bugs ({len(bugs) - len(filtered_bugs)} ignored): ")
bugs = filtered_bugs
click.echo(", ".join(sorted(str(b.bug_id) for b in bugs)))
if mode == 'qe':
for bug in bugs:
bzutil.set_state(bug, 'ON_QA', noop=noop, comment_for_release=f"{major_version}.{minor_version}")
if len(flag) > 0:
add_flags(bugs=bugs, flags=flag, noop=noop)
if report:
print_report(bugs)
if advisory and not default_advisory_type: # `--add ADVISORY_NUMBER` should respect the user's wish and attach all available bugs to whatever advisory is specified.
errata.add_bugs_with_retry(advisory, bugs, noop=noop)
return
# If --use-default-advisory or --into-default-advisories is given, we need to determine which bugs should be swept into which advisory.
# Otherwise we don't need to sweep bugs at all.
if not (into_default_advisories or default_advisory_type):
return
# key is impetus ("rpm", "image", "extras"), value is a set of bug IDs.
impetus_bugs = {
"rpm": set(),
"image": set(),
"extras": set()
}
# @lmeyer: simple and stupid would still be keeping the logic in python,
# possibly with config flags for branched logic.
# until that logic becomes too ugly to keep in python, i suppose..
if major_version < 4: # for 3.x, all bugs should go to the rpm advisory
impetus_bugs["rpm"] = set(bugs)
else: # for 4.x
# sweep rpm cve trackers into "rpm" advisory
rpm_bugs = dict()
if mode == 'sweep' and cve_trackers:
rpm_bugs = bzutil.get_valid_rpm_cves(bugs)
green_prefix("RPM CVEs found: ")
click.echo(sorted(b.id for b in rpm_bugs))
if rpm_bugs:
# if --check-builds flag is set
# only attach bugs that have corresponding brew builds attached to rpm advisory
if check_builds:
click.echo("Validating bugs with builds attached to the rpm advisory")
attached_builds = errata.get_advisory_nvrs(runtime.group_config.advisories["rpm"])
packages = attached_builds.keys()
not_found = []
for bug, package_name in rpm_bugs.items():
if package_name not in packages:
not_found.append((bug.id, package_name))
else:
click.echo(f"Build found for #{bug.id}, {package_name}")
impetus_bugs["rpm"].add(bug)
if not_found:
red_prefix("RPM CVE Warning: ")
click.echo("The following CVE (bug, package) were found but not attached, because no corresponding brew builds were found attached to the rpm advisory. First attach builds and then rerun to attach the bugs")
click.echo(not_found)
else:
click.echo("Skipping attaching RPM CVEs. Use --check-builds flag to validate with builds.")
impetus_bugs["extras"] = extras_bugs(bugs)
# all other bugs should go into "image" advisory
impetus_bugs["image"] = set(bugs) - impetus_bugs["extras"] - rpm_bugs.keys()
if default_advisory_type and impetus_bugs.get(default_advisory_type):
errata.add_bugs_with_retry(advisory, impetus_bugs[default_advisory_type], noop=noop)
elif into_default_advisories:
for impetus, bugs in impetus_bugs.items():
if bugs:
green_prefix(f'{impetus} advisory: ')
errata.add_bugs_with_retry(runtime.group_config.advisories[impetus], bugs, noop=noop)