def _fetch_builds_by_kind_image(runtime: Runtime, tag_pv_map: Dict[str, str],
brew_session: koji.ClientSession,
payload_only: bool, non_payload_only: bool):
image_metas: List[ImageMetadata] = []
for image in runtime.image_metas():
if image.base_only or not image.is_release:
continue
if (payload_only and not image.is_payload) or (non_payload_only
and image.is_payload):
continue
image_metas.append(image)
pbar_header(
'Generating list of images: ',
f'Hold on a moment, fetching Brew builds for {len(image_metas)} components...'
)
brew_latest_builds: List[Dict] = asyncio.get_event_loop(
).run_until_complete(
asyncio.gather(*[
exectools.to_thread(progress_func, image.get_latest_build)
for image in image_metas
]))
_ensure_accepted_tags(brew_latest_builds, brew_session, tag_pv_map)
shipped = _find_shipped_builds([b["id"] for b in brew_latest_builds],
brew_session)
unshipped = [b for b in brew_latest_builds if b["id"] not in shipped]
click.echo(
f'Found {len(shipped)+len(unshipped)} builds, of which {len(unshipped)} are new.'
)
nvrps = _gen_nvrp_tuples(unshipped, tag_pv_map)
return nvrps
def _fetch_builds_by_kind_image(runtime: Runtime, tag_pv_map: Dict[str, str],
brew_session: koji.ClientSession,
payload_only: bool, non_payload_only: bool):
image_metas: List[ImageMetadata] = []
for image in runtime.image_metas():
if image.base_only or not image.is_release:
continue
if (payload_only and not image.is_payload) or (non_payload_only
and image.is_payload):
continue
image_metas.append(image)
pbar_header(
'Generating list of images: ',
f'Hold on a moment, fetching Brew builds for {len(image_metas)} components...'
)
brew_latest_builds: List[Dict] = []
for image in image_metas:
LOGGER.info("Getting latest build for %s...", image.distgit_key)
brew_latest_builds.append(image.get_latest_build(brew_session))
_ensure_accepted_tags(brew_latest_builds, brew_session, tag_pv_map)
shipped = _find_shipped_builds([b["id"] for b in brew_latest_builds],
brew_session)
unshipped = [b for b in brew_latest_builds if b["id"] not in shipped]
click.echo(
f'Found {len(shipped)+len(unshipped)} builds, of which {len(unshipped)} are new.'
)
nvrps = _gen_nvrp_tuples(unshipped, tag_pv_map)
return nvrps
def _fetch_builds_by_kind_image(runtime, tag_pv_map, brew_event, brew_session, p, np):
# filter out image like 'openshift-enterprise-base'
image_metas = [i for i in runtime.image_metas() if not i.base_only]
# Returns a list of (name, version, release, product_version) tuples of each build
nvrps = []
# type judge
def tj(image, p, np):
if not image.is_release:
return False
if p:
return p == image.is_payload
if np:
# boolean xor.
return np != image.is_payload
else:
return True
tag_component_tuples = [(tag, image.get_component_name()) for tag in tag_pv_map for image in image_metas if tj(image, p, np)]
pbar_header(
'Generating list of images: ',
f'Hold on a moment, fetching Brew builds for {len(image_metas)} components with tags {", ".join(tag_pv_map.keys())}...',
tag_component_tuples)
latest_builds = brew.get_latest_builds(tag_component_tuples, event=brew_event, session=brew_session)
for i, build in enumerate(latest_builds):
if not build:
continue
tag = tag_component_tuples[i][0]
nvrps.append((build[0]['name'], build[0]['version'], build[0]['release'], tag_pv_map[tag]))
return nvrps
def repair_bugs(runtime, advisory, auto, id, original_state, new_state,
comment, close_placeholder, use_jira, noop,
default_advisory_type, bug_tracker):
changed_bug_count = 0
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
if auto:
click.echo("Fetching Advisory(errata_id={})".format(advisory))
if use_jira:
raw_bug_list = [
issue["key"]
for issue in errata.get_jira_issue_from_advisory(advisory)
]
else:
e = elliottlib.errata.Advisory(errata_id=advisory)
raw_bug_list = e.errata_bugs
else:
click.echo("Bypassed fetching erratum, using provided BZs")
raw_bug_list = cli_opts.id_convert(id)
green_print("Getting bugs for advisory")
# Fetch bugs in parallel because it can be really slow doing it
# one-by-one when you have hundreds of bugs
pbar_header("Fetching data for {} bugs: ".format(len(raw_bug_list)),
"Hold on a moment, we have to grab each one", raw_bug_list)
pool = ThreadPool(cpu_count())
click.secho("[", nl=False)
attached_bugs = pool.map(
lambda bug: progress_func(lambda: bug_tracker.get_bug(bug), '*'),
raw_bug_list)
# Wait for results
pool.close()
pool.join()
click.echo(']')
green_print("Got bugs for advisory")
for bug in attached_bugs:
if close_placeholder and "Placeholder" in bug.summary:
# if set close placeholder, ignore bug state
bug_tracker.update_bug_status(bug, "CLOSED")
changed_bug_count += 1
else:
if bug.status in original_state:
bug_tracker.update_bug_status(bug, new_state)
# only add comments for non-placeholder bug
if comment and not noop:
bug_tracker.add_comment(bug, comment, private=False)
changed_bug_count += 1
green_print("{} bugs successfully modified (or would have been)".format(
changed_bug_count))
def _fetch_builds_by_kind_rpm(builds, base_tag, product_version, session):
green_prefix('Generating list of rpms: ')
click.echo('Hold on a moment, fetching Brew builds')
candidates = elliottlib.brew.find_unshipped_build_candidates(
base_tag, product_version, kind='rpm')
pbar_header('Gathering additional information: ',
'Brew buildinfo is required to continue', candidates)
# We could easily be making scores of requests, one for each build
# we need information about. May as well do it in parallel.
results = parallel_results_with_progress(
candidates, lambda nvr: elliottlib.brew.get_brew_build(
nvr, product_version, session=session))
return _attached_to_open_erratum_with_correct_product_version(
results, product_version, elliottlib.errata)
def _fetch_builds_by_kind_image(runtime: Runtime, tag_pv_map: Dict[str, str],
brew_event: Optional[int],
brew_session: koji.ClientSession, p: bool,
np: bool):
# filter out image like 'openshift-enterprise-base'
image_metas = [i for i in runtime.image_metas() if not i.base_only]
# type judge
def tj(image):
if not image.is_release:
return False
if p:
return p == image.is_payload
if np:
# boolean xor.
return np != image.is_payload
else:
return True
tag_component_tuples = [(tag, image.get_component_name())
for tag in tag_pv_map for image in image_metas
if tj(image)]
pbar_header(
'Generating list of images: ',
f'Hold on a moment, fetching Brew builds for {len(image_metas)} components with tags {", ".join(tag_pv_map.keys())}...',
tag_component_tuples)
brew_builds = brew.get_tagged_builds(tag_component_tuples,
"image",
event=brew_event,
session=brew_session)
brew_latest_builds = list(
_find_latest_builds(brew_builds, runtime.assembly))
click.echo(
f'Found {len(brew_latest_builds)} builds. Filtering out shipped builds...'
)
shipped = _find_shipped_builds([b["id"] for b in brew_latest_builds],
brew_session)
unshipped = [b for b in brew_latest_builds if b["id"] not in shipped]
nvrps = _gen_nvrp_tuples(unshipped, tag_pv_map)
return nvrps
def _fetch_builds_by_kind_image(runtime, default_product_version, session):
image_metadata = []
product_version_overide = {}
for b in runtime.image_metas():
# filter out non_release builds
if b not in runtime.group_config.get('non_release', []):
product_version_overide[b.name] = default_product_version
if b.branch() != runtime.branch:
product_version_overide[b.name] = override_product_version(
default_product_version, b.branch())
image_metadata.append(b)
pbar_header('Generating list of images: ',
'Hold on a moment, fetching Brew buildinfo', image_metadata)
# Returns a list of (n, v, r, pv) tuples of each build
image_tuples = parallel_results_with_progress(
image_metadata,
lambda build: build.get_latest_build_info(product_version_overide))
pbar_header('Generating build metadata: ',
'Fetching data for {n} builds '.format(n=len(image_tuples)),
image_tuples)
# By 'meta' I mean the lil bits of meta data given back from
# get_latest_build_info
#
# TODO: Update the ImageMetaData class to include the NVR as
# an object attribute.
results = parallel_results_with_progress(
image_tuples,
lambda meta: elliottlib.brew.get_brew_build('{}-{}-{}'.format(
meta[0], meta[1], meta[2]),
product_version=meta[3],
session=session))
return [
b for b in results if not b.attached_to_open_erratum
# filter out 'openshift-enterprise-base-container' since it's not needed in advisory
if 'openshift-enterprise-base-container' not in b.nvr
]
def find_builds_cli(runtime, advisory, default_advisory_type, builds, kind,
from_diff, as_json, allow_attached, remove, clean,
no_cdn_repos, payload, non_payload, brew_event):
'''Automatically or manually find or attach/remove viable rpm or image builds
to ADVISORY. Default behavior searches Brew for viable builds in the
given group. Provide builds manually by giving one or more --build
(-b) options. Manually provided builds are verified against the Errata
Tool API.
\b
* Attach the builds to ADVISORY by giving --attach
* Remove the builds to ADVISORY by giving --remove
* Specify the build type using --kind KIND
Example: Assuming --group=openshift-3.7, then a build is a VIABLE
BUILD IFF it meets ALL of the following criteria:
\b
* HAS the tag in brew: rhaos-3.7-rhel7-candidate
* DOES NOT have the tag in brew: rhaos-3.7-rhel7
* IS NOT attached to ANY existing RHBA, RHSA, or RHEA
That is to say, a viable build is tagged as a "candidate", has NOT
received the "shipped" tag yet, and is NOT attached to any PAST or
PRESENT advisory. Here are some examples:
SHOW the latest OSE 3.6 image builds that would be attached to a
3.6 advisory:
$ elliott --group openshift-3.6 find-builds -k image
ATTACH the latest OSE 3.6 rpm builds to advisory 123456:
\b
$ elliott --group openshift-3.6 find-builds -k rpm --attach 123456
VERIFY (no --attach) that the manually provided RPM NVR and build
ID are viable builds:
$ elliott --group openshift-3.6 find-builds -k rpm -b megafrobber-1.0.1-2.el7 -a 93170
\b
Remove specific RPM NVR and build ID from advisory:
$ elliott --group openshift-4.3 find-builds -k image -b oauth-server-container-v4.3.22-202005212137 -a 55017 --remove
'''
if from_diff and builds:
raise click.BadParameter(
'Use only one of --build or --from-diff/--between.')
if clean and (remove or from_diff or builds):
raise click.BadParameter(
'Option --clean cannot be used with --build or --from-diff/--between.'
)
if not builds and remove:
raise click.BadParameter(
'Option --remove only support removing specific build with -b.')
if from_diff and kind != "image":
raise click.BadParameter(
'Option --from-diff/--between should be used with --kind/-k image.'
)
if advisory and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --attach')
if payload and non_payload:
raise click.BadParameter('Use only one of --payload or --non-payload.')
runtime.initialize(mode='images' if kind == 'image' else 'none')
replace_vars = runtime.group_config.vars.primitive(
) if runtime.group_config.vars else {}
et_data = runtime.gitdata.load_data(key='erratatool',
replace_vars=replace_vars).data
tag_pv_map = et_data.get('brew_tag_product_version_mapping')
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
ensure_erratatool_auth(
) # before we waste time looking up builds we can't process
# get the builds we want to add
unshipped_nvrps = []
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
if builds:
green_prefix('Fetching builds...')
unshipped_nvrps = _fetch_nvrps_by_nvr_or_id(
builds, tag_pv_map, ignore_product_version=remove)
elif clean:
unshipped_builds = errata.get_brew_builds(advisory)
elif from_diff:
unshipped_nvrps = _fetch_builds_from_diff(from_diff[0], from_diff[1],
tag_pv_map)
else:
if kind == 'image':
unshipped_nvrps = _fetch_builds_by_kind_image(
runtime, tag_pv_map, brew_event, brew_session, payload,
non_payload)
elif kind == 'rpm':
unshipped_nvrps = _fetch_builds_by_kind_rpm(
runtime, tag_pv_map, brew_event, brew_session)
pbar_header('Fetching builds from Errata: ',
'Hold on a moment, fetching buildinfos from Errata Tool...',
unshipped_builds if clean else unshipped_nvrps)
if not clean and not remove:
# if is --clean then batch fetch from Erratum no need to fetch them individually
# if is not for --clean fetch individually using nvrp tuples then get specific
# elliottlib.brew.Build Objects by get_brew_build()
# e.g. :
# ('atomic-openshift-descheduler-container', 'v4.3.23', '202005250821', 'RHEL-7-OSE-4.3').
# Build(atomic-openshift-descheduler-container-v4.3.23-202005250821).
unshipped_builds = parallel_results_with_progress(
unshipped_nvrps, lambda nvrp: elliottlib.errata.get_brew_build(
'{}-{}-{}'.format(nvrp[0], nvrp[1], nvrp[2]),
nvrp[3],
session=requests.Session()))
if not (allow_attached or builds):
unshipped_builds = _filter_out_inviable_builds(
kind, unshipped_builds, elliottlib.errata)
_json_dump(as_json, unshipped_builds, kind, tag_pv_map)
if not unshipped_builds:
green_print('No builds needed to be attached.')
return
if advisory:
if remove:
_detach_builds(
advisory,
[f"{nvrp[0]}-{nvrp[1]}-{nvrp[2]}" for nvrp in unshipped_nvrps])
elif clean:
_detach_builds(advisory, [b.nvr for b in unshipped_builds])
else: # attach
erratum = _update_to_advisory(unshipped_builds, kind, advisory,
remove, clean)
if not no_cdn_repos and kind == "image" and not (remove or clean):
cdn_repos = et_data.get('cdn_repos')
if cdn_repos:
# set up CDN repos
click.echo(
f"Configuring CDN repos {', '.join(cdn_repos)}...")
erratum.metadataCdnRepos(enable=cdn_repos)
click.echo("Done")
else:
click.echo('The following {n} builds '.format(n=len(unshipped_builds)),
nl=False)
if not (remove or clean):
click.secho('may be attached', bold=True, nl=False)
click.echo(' to an advisory:')
else:
click.secho('may be removed from', bold=True, nl=False)
click.echo(' from an advisory:')
for b in sorted(unshipped_builds):
click.echo(' ' + b.nvr)
def poll_signed(runtime, minutes, advisory, default_advisory_type, noop):
"""Poll for the signed-status of RPM builds attached to
ADVISORY. Returns rc=0 when all builds have been signed. Returns non-0
after MINUTES have passed and all builds have not been signed. This
non-0 return code is the number of unsigned builds remaining. All
builds must show 'signed' for this command to succeed.
NOTE: The two advisory options are mutually exclusive.
For testing in pipeline scripts this sub-command accepts a --noop
option. When --noop is used the value of --minutes is irrelevant. This
command will print out the signed state of all attached builds and
then exit with rc=0 if all builds are signed and non-0 if builds are
still unsigned. In the non-0 case the return code is the number of
unsigned builds.
Wait 15 minutes for the default 4.2 advisory to show all RPMS have
been signed:
$ elliott -g openshift-4.2 poll-signed --use-default-advisory rpm
Wait 5 mintes for the provided 4.2 advisory to show all RPMs have
been signed:
$ elliott -g openshift-4.2 poll-signed -m 5 --advisory 123456
Print the signed status of all attached builds, exit
immediately. Return code is the number of unsigned builds.
\b
$ elliott -g openshift-4.2 poll-signed --noop --use-default-advisory rpm
"""
if not (bool(advisory) ^ bool(default_advisory_type)):
raise click.BadParameter(
"Use only one of --use-default-advisory or --advisory")
runtime.initialize(no_group=default_advisory_type is None)
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
if not noop:
click.echo("Polling up to {} minutes for all RPMs to be signed".format(
minutes))
try:
e = elliottlib.errata.Advisory(errata_id=advisory)
all_builds = set([])
all_signed = False
# `errata_builds` is a dict with brew tags as keys, values are
# lists of builds on the advisory with that tag
for k, v in e.errata_builds.items():
all_builds = all_builds.union(set(v))
green_prefix("Fetching initial states: ")
click.echo("{} builds to check".format(len(all_builds)))
start_time = datetime.datetime.now()
while datetime.datetime.now() - start_time < datetime.timedelta(
minutes=minutes):
pbar_header("Getting build signatures: ", "Should be pretty quick",
all_builds)
pool = ThreadPool(cpu_count())
# Look up builds concurrently
click.secho("[", nl=False)
build_sigs = pool.map(
lambda build: progress_func(
lambda: elliottlib.errata.build_signed(build), '*'),
all_builds)
# Wait for results
pool.close()
pool.join()
click.echo(']')
if all(build_sigs):
all_signed = True
break
elif noop:
# Escape the time-loop
break
else:
yellow_prefix("Not all builds signed: ")
click.echo("re-checking")
continue
if not all_signed:
red_prefix("Signing incomplete: ")
if noop:
click.echo("All builds not signed. ")
else:
click.echo(
"All builds not signed in given window ({} minutes). ".
format(minutes))
exit(1)
else:
green_prefix("All builds signed: ")
click.echo("Enjoy!")
except ErrataException as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
async def get_latest_image_builds(image_metas: Iterable[ImageMetadata]):
pbar_header(
'Generating list of images: ',
f'Hold on a moment, fetching Brew builds for {len(image_metas)} components...')
builds: List[Dict] = await asyncio.gather(*[exectools.to_thread(progress_func, image.get_latest_build) for image in image_metas])
return builds
def repair_bugs(runtime, advisory, auto, id, original_state, new_state, noop,
default_advisory_type):
"""Move bugs attached to the advisory from one state to another
state. This is useful if the bugs have changed states *after* they
were attached. Similar to `find-bugs` but in reverse. `repair-bugs`
begins by reading bugs from an advisory, whereas `find-bugs` reads
from bugzilla.
This looks at attached bugs in the provided --from state and moves
them to the provided --to state.
\b
Background: This is intended for bugs which went to MODIFIED, were
attached to advisories, set to ON_QA, and then failed
testing. When this happens their state is reset back to ASSIGNED.
Using --use-default-advisory without a value set for the matching key
in the build-data will cause an error and elliott will exit in a
non-zero state. Most likely you will only want to use the `rpm` state,
but that could change in the future. Use of this option conflicts with
providing an advisory with the -a/--advisory option.
Move bugs on 123456 FROM the MODIFIED state back TO ON_QA state:
\b
$ elliott --group=openshift-4.1 repair-bugs --auto --advisory 123456 --from MODIFIED --to ON_QA
As above, but using the default RPM advisory defined in ocp-build-data:
\b
$ elliott --group=openshift-4.1 repair-bugs --auto --use-default-advisory rpm --from MODIFIED --to ON_QA
The previous examples could also be ran like this (MODIFIED and ON_QA are both defaults):
\b
$ elliott --group=openshift-4.1 repair-bugs --auto --use-default-advisory rpm
Bug ids may be given manually instead of using --auto:
\b
$ elliott --group=openshift-4.1 repair-bugs --id 170899 --id 8675309 --use-default-advisory rpm
"""
if auto and len(id) > 0:
raise click.BadParameter(
"Combining the automatic and manual bug modification options is not supported"
)
if not auto and len(id) == 0:
# No bugs were provided
raise click.BadParameter(
"If not using --auto then one or more --id's must be provided")
if advisory and default_advisory_type:
raise click.BadParameter(
"Use only one of --use-default-advisory or --advisory")
if len(id) == 0 and advisory is None and default_advisory_type is None:
# error, no bugs, advisory, or default selected
raise click.BadParameter(
"No input provided: Must use one of --id, --advisory, or --use-default-advisory"
)
# Load bugzilla infomation and get a reference to the api
runtime.initialize()
bz_data = runtime.gitdata.load_data(key='bugzilla').data
bzapi = elliottlib.bzutil.get_bzapi(bz_data)
changed_bug_count = 0
attached_bugs = []
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
raw_bug_list = []
if auto:
click.echo("Fetching Erratum(errata_id={})".format(advisory))
e = Erratum(errata_id=advisory)
raw_bug_list = e.errata_bugs
else:
click.echo("Bypassed fetching erratum, using provided BZs")
raw_bug_list = cli_opts.id_convert(id)
green_print("Getting bugs for advisory")
# Fetch bugs in parallel because it can be really slow doing it
# one-by-one when you have hundreds of bugs
pbar_header("Fetching data for {} bugs: ".format(len(raw_bug_list)),
"Hold on a moment, we have to grab each one", raw_bug_list)
pool = ThreadPool(cpu_count())
click.secho("[", nl=False)
attached_bugs = pool.map(
lambda bug: progress_func(lambda: bzapi.getbug(bug), '*'),
raw_bug_list)
# Wait for results
pool.close()
pool.join()
click.echo(']')
green_print("Got bugs for advisory")
for bug in attached_bugs:
if bug.status in original_state:
changed_bug_count += 1
elliottlib.bzutil.set_state(bug, new_state, noop=noop)
green_print("{} bugs successfullly modified (or would have been)".format(
changed_bug_count))
