def _fetch_builds_by_kind_image(runtime, default_product_version, session):
image_metadata = []
product_version_overide = {}
for b in runtime.image_metas():
# filter out non_release builds
if b not in runtime.group_config.get('non_release', []):
product_version_overide[b.name] = default_product_version
if b.branch() != runtime.branch:
product_version_overide[b.name] = override_product_version(
default_product_version, b.branch())
image_metadata.append(b)
pbar_header('Generating list of images: ',
'Hold on a moment, fetching Brew buildinfo', image_metadata)
# Returns a list of (n, v, r, pv) tuples of each build
image_tuples = parallel_results_with_progress(
image_metadata,
lambda build: build.get_latest_build_info(product_version_overide))
pbar_header('Generating build metadata: ',
'Fetching data for {n} builds '.format(n=len(image_tuples)),
image_tuples)
# By 'meta' I mean the lil bits of meta data given back from
# get_latest_build_info
#
# TODO: Update the ImageMetaData class to include the NVR as
# an object attribute.
results = parallel_results_with_progress(
image_tuples,
lambda meta: elliottlib.brew.get_brew_build('{}-{}-{}'.format(
meta[0], meta[1], meta[2]),
product_version=meta[3],
session=session))
return [
b for b in results if not b.attached_to_open_erratum
# filter out 'openshift-enterprise-base-container' since it's not needed in advisory
if 'openshift-enterprise-base-container' not in b.nvr
]
def _fetch_builds_by_kind_rpm(builds, base_tag, product_version, session):
green_prefix('Generating list of rpms: ')
click.echo('Hold on a moment, fetching Brew builds')
candidates = elliottlib.brew.find_unshipped_build_candidates(
base_tag, product_version, kind='rpm')
pbar_header('Gathering additional information: ',
'Brew buildinfo is required to continue', candidates)
# We could easily be making scores of requests, one for each build
# we need information about. May as well do it in parallel.
results = parallel_results_with_progress(
candidates, lambda nvr: elliottlib.brew.get_brew_build(
nvr, product_version, session=session))
return _attached_to_open_erratum_with_correct_product_version(
results, product_version, elliottlib.errata)
def get_latest_cvp_results(runtime: Runtime, resultsdb_api: ResultsDBAPI,
nvrs: List[str]):
all_results = []
results = parallel_results_with_progress(
nvrs,
lambda nvr: resultsdb_api.get_latest_results(
["rhproduct.default.sanity"], [nvr]),
file=sys.stderr)
for nvr, result in zip(nvrs, results):
data = result.get("data")
if not data: # Couldn't find a CVP test result for the given Brew build
runtime.logger.warning(
f"Couldn't find a CVP test result for {nvr}. Is the CVP test still running?"
)
all_results.append(None)
continue
all_results.append(data[0])
return all_results
def get_optional_checks(runtime: Runtime, test_results):
# Each CVP test result stored in ResultsDB has a link to an external storage with more CVP test details
# e.g. http://external-ci-coldstorage.datahub.redhat.com/cvp/cvp-product-test/ose-insights-operator-container-v4.5.0-202007240519.p0/fb9dd365-e886-46c9-9661-fd13c0d29c49/
external_urls = [
r["ref_url"] + "sanity-tests-optional-results.json"
for r in test_results
]
optional_check_results = []
with requests.session() as session:
responses = parallel_results_with_progress(
external_urls, lambda url: session.get(url), file=sys.stderr)
for cvp_result, r in zip(test_results, responses):
if r.status_code != 200:
nvr = cvp_result["data"]["item"][0]
runtime.logger.warning(
f"Couldn't find sanity-tests-optional-results.json for {nvr}")
optional_check_results.append(None)
continue
optional_check_results.append(r.json())
return optional_check_results
def find_builds_cli(runtime, advisory, default_advisory_type, builds, kind,
from_diff, as_json, allow_attached, remove, clean,
no_cdn_repos, payload, non_payload, brew_event):
'''Automatically or manually find or attach/remove viable rpm or image builds
to ADVISORY. Default behavior searches Brew for viable builds in the
given group. Provide builds manually by giving one or more --build
(-b) options. Manually provided builds are verified against the Errata
Tool API.
\b
* Attach the builds to ADVISORY by giving --attach
* Remove the builds to ADVISORY by giving --remove
* Specify the build type using --kind KIND
Example: Assuming --group=openshift-3.7, then a build is a VIABLE
BUILD IFF it meets ALL of the following criteria:
\b
* HAS the tag in brew: rhaos-3.7-rhel7-candidate
* DOES NOT have the tag in brew: rhaos-3.7-rhel7
* IS NOT attached to ANY existing RHBA, RHSA, or RHEA
That is to say, a viable build is tagged as a "candidate", has NOT
received the "shipped" tag yet, and is NOT attached to any PAST or
PRESENT advisory. Here are some examples:
SHOW the latest OSE 3.6 image builds that would be attached to a
3.6 advisory:
$ elliott --group openshift-3.6 find-builds -k image
ATTACH the latest OSE 3.6 rpm builds to advisory 123456:
\b
$ elliott --group openshift-3.6 find-builds -k rpm --attach 123456
VERIFY (no --attach) that the manually provided RPM NVR and build
ID are viable builds:
$ elliott --group openshift-3.6 find-builds -k rpm -b megafrobber-1.0.1-2.el7 -a 93170
\b
Remove specific RPM NVR and build ID from advisory:
$ elliott --group openshift-4.3 find-builds -k image -b oauth-server-container-v4.3.22-202005212137 -a 55017 --remove
'''
if from_diff and builds:
raise click.BadParameter(
'Use only one of --build or --from-diff/--between.')
if clean and (remove or from_diff or builds):
raise click.BadParameter(
'Option --clean cannot be used with --build or --from-diff/--between.'
)
if not builds and remove:
raise click.BadParameter(
'Option --remove only support removing specific build with -b.')
if from_diff and kind != "image":
raise click.BadParameter(
'Option --from-diff/--between should be used with --kind/-k image.'
)
if advisory and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --attach')
if payload and non_payload:
raise click.BadParameter('Use only one of --payload or --non-payload.')
runtime.initialize(mode='images' if kind == 'image' else 'none')
replace_vars = runtime.group_config.vars.primitive(
) if runtime.group_config.vars else {}
et_data = runtime.gitdata.load_data(key='erratatool',
replace_vars=replace_vars).data
tag_pv_map = et_data.get('brew_tag_product_version_mapping')
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
ensure_erratatool_auth(
) # before we waste time looking up builds we can't process
# get the builds we want to add
unshipped_nvrps = []
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
if builds:
green_prefix('Fetching builds...')
unshipped_nvrps = _fetch_nvrps_by_nvr_or_id(
builds, tag_pv_map, ignore_product_version=remove)
elif clean:
unshipped_builds = errata.get_brew_builds(advisory)
elif from_diff:
unshipped_nvrps = _fetch_builds_from_diff(from_diff[0], from_diff[1],
tag_pv_map)
else:
if kind == 'image':
unshipped_nvrps = _fetch_builds_by_kind_image(
runtime, tag_pv_map, brew_event, brew_session, payload,
non_payload)
elif kind == 'rpm':
unshipped_nvrps = _fetch_builds_by_kind_rpm(
runtime, tag_pv_map, brew_event, brew_session)
pbar_header('Fetching builds from Errata: ',
'Hold on a moment, fetching buildinfos from Errata Tool...',
unshipped_builds if clean else unshipped_nvrps)
if not clean and not remove:
# if is --clean then batch fetch from Erratum no need to fetch them individually
# if is not for --clean fetch individually using nvrp tuples then get specific
# elliottlib.brew.Build Objects by get_brew_build()
# e.g. :
# ('atomic-openshift-descheduler-container', 'v4.3.23', '202005250821', 'RHEL-7-OSE-4.3').
# Build(atomic-openshift-descheduler-container-v4.3.23-202005250821).
unshipped_builds = parallel_results_with_progress(
unshipped_nvrps, lambda nvrp: elliottlib.errata.get_brew_build(
'{}-{}-{}'.format(nvrp[0], nvrp[1], nvrp[2]),
nvrp[3],
session=requests.Session()))
if not (allow_attached or builds):
unshipped_builds = _filter_out_inviable_builds(
kind, unshipped_builds, elliottlib.errata)
_json_dump(as_json, unshipped_builds, kind, tag_pv_map)
if not unshipped_builds:
green_print('No builds needed to be attached.')
return
if advisory:
if remove:
_detach_builds(
advisory,
[f"{nvrp[0]}-{nvrp[1]}-{nvrp[2]}" for nvrp in unshipped_nvrps])
elif clean:
_detach_builds(advisory, [b.nvr for b in unshipped_builds])
else: # attach
erratum = _update_to_advisory(unshipped_builds, kind, advisory,
remove, clean)
if not no_cdn_repos and kind == "image" and not (remove or clean):
cdn_repos = et_data.get('cdn_repos')
if cdn_repos:
# set up CDN repos
click.echo(
f"Configuring CDN repos {', '.join(cdn_repos)}...")
erratum.metadataCdnRepos(enable=cdn_repos)
click.echo("Done")
else:
click.echo('The following {n} builds '.format(n=len(unshipped_builds)),
nl=False)
if not (remove or clean):
click.secho('may be attached', bold=True, nl=False)
click.echo(' to an advisory:')
else:
click.secho('may be removed from', bold=True, nl=False)
click.echo(' from an advisory:')
for b in sorted(unshipped_builds):
click.echo(' ' + b.nvr)