def remove_bugs(advisory_id, bug_ids, remove_all, bug_tracker, noop):
try:
advisory = errata.Advisory(errata_id=advisory_id)
except GSSError:
exit_unauthenticated()
if not advisory:
raise ElliottFatalError(f"Error: Could not locate advisory {advisory_id}")
try:
attached_bug_ids = bug_tracker.advisory_bug_ids(advisory)
if not remove_all:
bug_ids = [b for b in bug_ids if b in attached_bug_ids]
else:
bug_ids = attached_bug_ids
green_prefix(f"Found {len(bug_ids)} bugs attached to advisory: ")
click.echo(f"{bug_ids}")
if not bug_ids:
return
green_prefix(f"Removing bugs from advisory {advisory_id}..")
bug_tracker.remove_bugs(advisory, bug_ids, noop)
except ErrataException as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
def get_advisory_nvrs(advisory):
"""
:return: dict, with keys as package names and values as strs in the form: '{version}-{release}'
"""
try:
green_prefix("Fetching advisory builds: ")
click.echo("Advisory - {}".format(advisory))
builds = get_builds(advisory)
except GSSError:
exit_unauthenticated()
except exceptions.ErrataToolError as ex:
raise exceptions.ElliottFatalError(getattr(ex, 'message', repr(ex)))
all_advisory_nvrs = {}
# Results come back with top level keys which are brew tags
green_prefix("Looping over tags: ")
click.echo("{} tags to check".format(len(builds)))
for tag in builds.keys():
# Each top level has a key 'builds' which is a list of dicts
green_prefix("Looping over builds in tag: ")
click.echo("{} with {} builds".format(tag, len(builds[tag]['builds'])))
for build in builds[tag]['builds']:
# Each dict has a top level key which might be the actual
# 'nvr' but I don't have enough data to know for sure
# yet. Also I don't know when there might be more than one
# key in the build dict. We'll loop over it to be sure.
for name in build.keys():
n, v, r = name.rsplit('-', 2)
version_release = "{}-{}".format(v, r)
all_advisory_nvrs[n] = version_release
return all_advisory_nvrs
def add_metadata_cli(runtime, kind, impetus, advisory):
"""Add metadata to an advisory. This is usually called by
create immediately after creation. It is only useful to you if
you are going back and adding metadata to older advisories.
Note: Requires you provide a --group
Example to add standard metadata to a 3.10 images release
\b
$ elliott --group=openshift-3.10 add-metadata --impetus standard --kind image
"""
runtime.initialize()
release = release_from_branch(runtime.group_config.branch)
try:
advisory = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
result = elliottlib.errata.add_comment(
advisory.errata_id, {'release': release, 'kind': kind, 'impetus': impetus})
if result.status_code == 201:
green_prefix("Added metadata successfully")
click.echo()
elif result.status_code == 403:
exit_unauthorized()
else:
red_print("Something weird may have happened")
raise ElliottFatalError(
"Unexpected response from ET API: {code}".format(code=result.status_code))
def list_cli(ctx, filter_id, n):
"""Print a list of one-line informational strings of RHOSE
advisories. By default the 5 most recently created advisories are
printed. Note, they are NOT sorted by release date.
NOTE: new filters must be created in the Errata Tool web
interface.
Default filter definition: RHBA; Active; Product: RHOSE; Devel Group:
ENG OpenShift Enterprise; sorted by newest. Browse this filter
yourself online: https://errata.devel.redhat.com/filter/1965
List 10 advisories instead of the default 6 with your custom
filter #1337:
$ elliott list -n 10 -f 1337
"""
try:
for erratum in elliottlib.errata.get_filtered_list(filter_id, limit=n):
click.echo(
"{release_date:11s} {state:15s} {synopsis:80s} {url}".format(
release_date=erratum.publish_date_override,
state=erratum.errata_state,
synopsis=erratum.synopsis,
url=erratum.url()))
except GSSError:
exit_unauthenticated()
except elliottlib.exceptions.ErrataToolError as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
def _update_to_advisory(builds, kind, advisory, remove, clean):
click.echo(f"Attaching to advisory {advisory}...")
if kind not in {"rpm", "image"}:
raise ValueError(f"{kind} should be one of 'rpm' or 'image'")
try:
erratum = Erratum(errata_id=advisory)
file_type = 'tar' if kind == 'image' else 'rpm'
product_version_set = {build.product_version for build in builds}
for pv in product_version_set:
erratum.addBuilds(buildlist=[
build.nvr for build in builds if build.product_version == pv
],
release=pv,
file_types={
build.nvr: [file_type]
for build in builds
})
erratum.commit()
build_nvrs = sorted(build.nvr for build in builds)
green_print('Attached build(s) successfully:')
for b in build_nvrs:
click.echo(' ' + b)
return erratum
except GSSError:
exit_unauthenticated()
except elliottlib.exceptions.BrewBuildException as ex:
raise ElliottFatalError(f'Error attaching/removing builds: {str(ex)}')
def add_bugs_with_retry(advisory, bug_list, retried):
"""
adding specified bug_list into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bug_list: bug id list which suppose to attach to advisory
:param retried: retry 2 times, first attempt fetch failed bugs sift out then attach again
:return:
"""
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
green_prefix("Adding {count} bugs to advisory {retry_times} times:".format(
count=len(bug_list), retry_times=1 if retried is False else 2))
print(" {advs}".format(advs=advs))
try:
advs.addBugs(bug_list)
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
if retried is not True:
black_list = parse_exception_error_message(e)
retry_list = [x for x in bug_list if x not in black_list]
if len(retry_list) > 0:
add_bugs_with_retry(advisory, retry_list, True)
else:
raise exceptions.ElliottFatalError(getattr(e, 'message', repr(e)))
def _attach_to_advisory(builds, kind, advisory):
if kind is None:
raise ElliottFatalError(
'Need to specify with --kind=image or --kind=rpm with packages: {}'
.format(builds))
try:
erratum = Erratum(errata_id=advisory)
file_type = 'tar' if kind == 'image' else 'rpm'
product_version_set = {build.product_version for build in builds}
for pv in product_version_set:
erratum.addBuilds(buildlist=[
build.nvr for build in builds if build.product_version == pv
],
release=pv,
file_types={
build.nvr: [file_type]
for build in builds
})
erratum.commit()
build_nvrs = sorted(build.nvr for build in builds)
green_print('Attached build(s) successfully:')
for b in build_nvrs:
click.echo(' ' + b)
except GSSError:
exit_unauthenticated()
except elliottlib.exceptions.BrewBuildException as ex:
raise ElliottFatalError('Error attaching builds: {}'.format(
getattr(ex, 'message', repr(ex))))
def add_bugzilla_bugs_with_retry(
advisory_id: int,
bugids: List,
noop: bool = False,
batch_size: int = constants.BUG_ATTACH_CHUNK_SIZE):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory_id: advisory id
:param bugids: iterable of bugzilla bug ids to attach to advisory
:param noop: do not modify anything
:param batch_size: perform operation in batches of given size
:return:
"""
logger.info(
f'Request to attach {len(bugids)} bugs to the advisory {advisory_id}')
try:
advisory = Erratum(errata_id=advisory_id)
except GSSError:
exit_unauthenticated()
if not advisory:
raise exceptions.ElliottFatalError(
f"Error: Could not locate advisory {advisory_id}")
existing_bugs = bzutil.BugzillaBugTracker.advisory_bug_ids(advisory)
new_bugs = set(bugids) - set(existing_bugs)
logger.info(
f'Bugs already attached: {len(existing_bugs)}. New bugs: {len(new_bugs)}'
)
if not new_bugs:
return
for chunk_of_bugs in chunk(list(new_bugs), batch_size):
if noop:
logger.info('Dry run: Would have attached bugs')
continue
try:
advisory.addBugs(chunk_of_bugs)
advisory.commit()
except ErrataException as e:
logger.info(f"ErrataException Message: {e}\nRetrying...")
block_list = parse_exception_error_message(e)
retry_list = [x for x in chunk_of_bugs if x not in block_list]
if len(retry_list) == 0:
continue
try:
advisory = Erratum(errata_id=advisory_id)
advisory.addBugs(retry_list)
advisory.commit()
except ErrataException as e:
raise exceptions.ElliottFatalError(
getattr(e, 'message', repr(e)))
logger.info("remaining bugs attached")
logger.info("All bugzilla bugs attached")
def _get_attached_bugs(self, advisories):
# get bugs attached to all advisories
bugs = set()
try:
for advisory in advisories:
green_print(f"Retrieving bugs for advisory {advisory}")
bugs.update(errata.get_bug_ids(advisory))
except GSSError:
exit_unauthenticated()
green_print(f"Found {len(bugs)} bugs")
return list(bzutil.get_bugs(self.bzapi, list(bugs)).values())
def add_bugs_with_retry(advisory, bugs, retried=False, noop=False):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bugs: iterable of bzutil.bug to attach to advisory
:param retried: retry 2 times, first attempt fetch failed bugs sift out then attach again
:return:
"""
print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}')
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
existing_bugs = advs.errata_bugs
new_bugs = set(bug.id for bug in bugs) - set(existing_bugs)
print(f'Bugs already attached: {len(existing_bugs)}')
print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}')
if noop:
print('Dry run. Exiting.')
return
if not new_bugs:
print('No new bugs to attach. Exiting.')
return
green_prefix("Adding {count} bugs to advisory {retry_times} times:".format(
count=len(bugs), retry_times=1 if retried is False else 2))
try:
advs.addBugs([bug.id for bug in bugs])
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
if retried is not True:
block_list = parse_exception_error_message(e)
retry_list = [x for x in bugs if x.id not in block_list]
if len(retry_list) > 0:
add_bugs_with_retry(advisory,
retry_list,
retried=True,
noop=noop)
else:
raise exceptions.ElliottFatalError(getattr(e, 'message', repr(e)))
def remove_bugs(runtime, advisory, default_advisory_type, id):
"""Remove given BUGS from ADVISORY.
Remove bugs that have been attached an advisory:
\b
$ elliott --group openshift-3.7 remove-bugs --id 123456 --advisory 1234123
Remove two bugs from default rpm advisory. Note that --group is required
because default advisory is from ocp-build-data:
\b
$ elliott --group openshift-3.7 remove-bugs --id 123456 --id 3412311 --use-default-advisory rpm
"""
if bool(advisory) == bool(default_advisory_type):
raise click.BadParameter(
"Specify exactly one of --use-default-advisory or advisory arg")
runtime.initialize()
bz_data = runtime.gitdata.load_data(key='bugzilla').data
bzapi = elliottlib.bzutil.get_bzapi(bz_data)
bug_ids = [bzapi.getbug(i) for i in cli_opts.id_convert(id)]
green_prefix("Found {} bugs:".format(len(bug_ids)))
click.echo(" {}".format(", ".join([str(b.bug_id) for b in bug_ids])))
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
if advisory is not False:
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise ElliottFatalError(
"Error: Could not locate advisory {advs}".format(
advs=advisory))
try:
green_prefix("Removing {count} bugs from advisory:".format(
count=len(bug_ids)))
click.echo(" {advs}".format(advs=advisory))
advs.removeBugs([bug.id for bug in bug_ids])
advs.commit()
except ErrataException as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
def _get_attached_image_builds(brew_session, advisories):
# get all attached image builds
build_nvrs = []
try:
for advisory in advisories:
green_print(f"Retrieving builds from advisory {advisory}")
advisory = Erratum(errata_id=advisory)
for build_list in advisory.errata_builds.values(): # one per product version
build_nvrs.extend(build_list)
except GSSError:
exit_unauthenticated()
green_print(f"Found {len(build_nvrs)} builds")
return [build for build in brew.get_build_objects(build_nvrs, brew_session) if _is_image(build)]
def create_placeholder_cli(runtime, kind, advisory, default_advisory_type):
"""Create a placeholder bug for attaching to an advisory.
KIND - The kind of placeholder to create ({}).
ADVISORY - Optional. The advisory to attach the bug to.
$ elliott --group openshift-4.1 create-placeholder --kind rpm --attach 12345
""".format('/'.join(elliottlib.constants.standard_advisory_types))
runtime.initialize()
if advisory and default_advisory_type:
raise click.BadParameter(
"Use only one of --use-default-advisory or --advisory")
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
kind = default_advisory_type
if kind is None:
raise click.BadParameter(
"--kind must be specified when not using --use-default-advisory")
bz_data = runtime.gitdata.load_data(key='bugzilla').data
target_release = bz_data['target_release'][0]
newbug = elliottlib.bzutil.create_placeholder(bz_data, kind,
target_release)
click.echo("Created BZ: {} {}".format(newbug.id, newbug.weburl))
if advisory is not False:
click.echo("Attaching to advisory...")
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise ElliottFatalError(
"Error: Could not locate advisory {advs}".format(
advs=advisory))
try:
green_prefix("Adding placeholder bug to advisory:")
click.echo(" {advs}".format(advs=advisory))
advs.addBugs([newbug.id])
advs.commit()
except ErrataException as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
async def verify_attached_bugs(runtime: Runtime, verify_bug_status: bool,
advisories: Tuple[int, ...], verify_flaws: bool,
use_jira: bool):
validator = BugValidator(runtime, use_jira, output="text")
try:
await validator.errata_api.login()
advisory_bugs = await validator.get_attached_bugs(advisories)
non_flaw_bugs = validator.filter_bugs_by_product(
{b
for bugs in advisory_bugs.values() for b in bugs})
validator.validate(non_flaw_bugs, verify_bug_status)
if verify_flaws:
await validator.verify_attached_flaws(advisory_bugs)
except GSSError:
exit_unauthenticated()
finally:
await validator.close()
def puddle_advisories_cli(runtime, filter_id, details):
"""Print a comma separated list of advisory numbers which can be used
when filling in the 'errata_whitelist' parameter in a signed puddle
config.
Uses an Errata Tool filter to find in-progress and being-released
advisories for OpenShift. This list is trimmed down to only advisories
matching the given --group by parsing the ART metadata embedded in the
first comment.
List advisories required to create a signed 4.2 puddle:
\b
$ elliott --group=openshift-4.1 puddle-advisories
44849, 44740
"""
use_in_puddle_conf = []
runtime.initialize()
major = major_from_branch(runtime.group_config.branch)
minor = minor_from_branch(runtime.group_config.branch)
release = "{}.{}".format(major, minor)
try:
for erratum in elliottlib.errata.get_filtered_list(filter_id,
limit=50):
metadata_comments_json = elliottlib.errata.get_metadata_comments_json(
erratum.errata_id)
if not metadata_comments_json:
# Does not contain ART metadata, skip it
sys.stderr.write("Does not contain ART metadata: {}\n".format(
erratum.errata_id))
continue
metadata = metadata_comments_json[0]
if str(metadata['release']) == str(release) and (
metadata['impetus'] != 'test'):
use_in_puddle_conf.append(str(erratum.errata_id))
if details:
sys.stderr.write(str(erratum))
sys.stderr.flush()
click.echo(", ".join(use_in_puddle_conf))
except GSSError:
exit_unauthenticated()
except elliottlib.exceptions.ErrataToolError as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
def add_jira_bugs_with_retry(
advisory_id: int,
bugids: List[str],
noop: bool = False,
batch_size: int = constants.BUG_ATTACH_CHUNK_SIZE):
"""
:param advisory_id: advisory id
:param bugids: iterable of jira bug ids to attach to advisory
:param noop: do not modify anything
:param batch_size: perform operation in batches of given size
"""
logger.info(
f'Request to attach {len(bugids)} bugs to the advisory {advisory_id}')
try:
advisory = Erratum(errata_id=advisory_id)
except GSSError:
exit_unauthenticated()
if not advisory:
raise exceptions.ElliottFatalError(
f"Error: Could not locate advisory {advisory_id}")
existing_bugs = bzutil.JIRABugTracker.advisory_bug_ids(advisory)
new_bugs = set(bugids) - set(existing_bugs)
logger.info(
f'Bugs already attached: {len(existing_bugs)}. New bugs: {len(new_bugs)}'
)
if not new_bugs:
return
for chunk_of_bugs in chunk(bugids, batch_size):
if noop:
logger.info('Dry run: Would have attached bugs')
continue
results = add_multi_jira_issues(advisory_id, chunk_of_bugs)
for i, result in enumerate(results):
if result.status_code != 201:
rt = add_jira_issue(advisory_id, chunk_of_bugs[i])
if rt.status_code != 201:
raise exceptions.ElliottFatalError(
f"attach jira bug {chunk_of_bugs[i]} failed with "
f"status={rt.status_code}: {rt.json()}")
logger.info("All jira bugs attached")
def create_placeholder(kind, advisory_id, bug_tracker, noop):
newbug = bug_tracker.create_placeholder(kind, noop)
if noop:
return
click.echo(f"Created Bug: {newbug.id} {newbug.weburl}")
if not advisory_id:
return
try:
advisory = Erratum(errata_id=advisory_id)
except GSSError:
exit_unauthenticated()
if advisory is False:
raise ElliottFatalError(
f"Error: Could not locate advisory {advisory_id}")
click.echo("Attaching bug to advisory...")
bug_tracker.attach_bugs(advisory_id, [newbug.id])
def get_all_advisory_nvrs(advisory):
"""
:return: list of tuples (name, version, release)
"""
try:
builds = get_builds(advisory)
except GSSError:
exit_unauthenticated()
except exceptions.ErrataToolError as ex:
raise exceptions.ElliottFatalError(getattr(ex, 'message', repr(ex)))
all_advisory_nvrs = []
# Results come back with top level keys which are brew tags
for tag in builds.keys():
# Each top level has a key 'builds' which is a list of dicts
for build in builds[tag]['builds']:
for name in build.keys():
n, v, r = name.rsplit('-', 2)
all_advisory_nvrs.append((n, v, r))
return all_advisory_nvrs
def validate_rhsa_cli(runtime, advisory):
"""
Validates an RHSA by returning the 'alerts' generated by Product Security's central SFM2 server.
Usage:
\b
$ elliott validate-rhsa ID
"""
try:
if Erratum(errata_id=advisory).errata_type != 'RHSA':
print(f"Advisory {advisory} is not an RHSA. Nothing to check.")
exit(0)
except GSSError:
exit_unauthenticated()
session = requests.Session()
url = constants.SFM2_ERRATA_ALERTS_URL.format(id=advisory)
print(url)
print("Validating..")
print()
resp = session.get(url)
# Will fail if id invalid, not an RHSA or RHSA has embargoed data
if resp.status_code != 200:
print(f"HTTP {resp.status_code}: Could not validate rhsa {advisory} (is it an RHSA? without embargo?)")
exit(1)
return
alerts = resp.json()
if alerts:
for a in alerts:
print("E: {text}".format(text=a.get("text")))
howto = a.get("how_to_resolve")
if howto:
print(textwrap.indent(howto, " > "))
print()
exit(2)
else:
print("No issues found")
def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, builds,
kind, from_diff, as_json, allow_attached, remove, clean,
no_cdn_repos, payload, non_payload):
'''Automatically or manually find or attach/remove viable rpm or image builds
to ADVISORY. Default behavior searches Brew for viable builds in the
given group. Provide builds manually by giving one or more --build
(-b) options. Manually provided builds are verified against the Errata
Tool API.
\b
* Attach the builds to ADVISORY by giving --attach
* Remove the builds to ADVISORY by giving --remove
* Specify the build type using --kind KIND
Example: Assuming --group=openshift-3.7, then a build is a VIABLE
BUILD IFF it meets ALL of the following criteria:
\b
* HAS the tag in brew: rhaos-3.7-rhel7-candidate
* DOES NOT have the tag in brew: rhaos-3.7-rhel7
* IS NOT attached to ANY existing RHBA, RHSA, or RHEA
That is to say, a viable build is tagged as a "candidate", has NOT
received the "shipped" tag yet, and is NOT attached to any PAST or
PRESENT advisory. Here are some examples:
SHOW the latest OSE 3.6 image builds that would be attached to a
3.6 advisory:
$ elliott --group openshift-3.6 find-builds -k image
ATTACH the latest OSE 3.6 rpm builds to advisory 123456:
\b
$ elliott --group openshift-3.6 find-builds -k rpm --attach 123456
VERIFY (no --attach) that the manually provided RPM NVR and build
ID are viable builds:
$ elliott --group openshift-3.6 find-builds -k rpm -b megafrobber-1.0.1-2.el7 -a 93170
\b
Remove specific RPM NVR and build ID from advisory:
$ elliott --group openshift-4.3 find-builds -k image -b oauth-server-container-v4.3.22-202005212137 -a 55017 --remove
'''
if from_diff and builds:
raise click.BadParameter(
'Use only one of --build or --from-diff/--between.')
if clean and (remove or from_diff or builds):
raise click.BadParameter(
'Option --clean cannot be used with --build or --from-diff/--between.'
)
if not builds and remove:
raise click.BadParameter(
'Option --remove only support removing specific build with -b.')
if from_diff and kind != "image":
raise click.BadParameter(
'Option --from-diff/--between should be used with --kind/-k image.'
)
if advisory and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --attach')
if payload and non_payload:
raise click.BadParameter('Use only one of --payload or --non-payload.')
runtime.initialize(mode='images' if kind == 'image' else 'rpms')
replace_vars = runtime.group_config.vars.primitive(
) if runtime.group_config.vars else {}
et_data = runtime.gitdata.load_data(key='erratatool',
replace_vars=replace_vars).data
tag_pv_map = et_data.get('brew_tag_product_version_mapping')
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
ensure_erratatool_auth(
) # before we waste time looking up builds we can't process
unshipped_nvrps = []
unshipped_builds = []
to_remove = []
# get the builds we want to add
brew_session = runtime.build_retrying_koji_client(caching=True)
if builds:
green_prefix('Fetching builds...')
unshipped_nvrps = _fetch_nvrps_by_nvr_or_id(
builds,
tag_pv_map,
ignore_product_version=remove,
brew_session=brew_session)
elif clean:
unshipped_builds = errata.get_brew_builds(advisory)
elif from_diff:
unshipped_nvrps = _fetch_builds_from_diff(from_diff[0], from_diff[1],
tag_pv_map)
else:
if kind == 'image':
unshipped_nvrps = _fetch_builds_by_kind_image(
runtime, tag_pv_map, brew_session, payload, non_payload)
elif kind == 'rpm':
unshipped_nvrps = _fetch_builds_by_kind_rpm(
runtime, tag_pv_map, brew_session)
pbar_header('Fetching builds from Errata: ',
'Hold on a moment, fetching buildinfos from Errata Tool...',
unshipped_builds if clean else unshipped_nvrps)
if not clean and not remove:
# if is --clean then batch fetch from Erratum no need to fetch them individually
# if is not for --clean fetch individually using nvrp tuples then get specific
# elliottlib.brew.Build Objects by get_brew_build()
# e.g. :
# ('atomic-openshift-descheduler-container', 'v4.3.23', '202005250821', 'RHEL-7-OSE-4.3').
# Build(atomic-openshift-descheduler-container-v4.3.23-202005250821).
unshipped_builds = parallel_results_with_progress(
unshipped_nvrps, lambda nvrp: elliottlib.errata.get_brew_build(
'{}-{}-{}'.format(nvrp[0], nvrp[1], nvrp[2]),
nvrp[3],
session=requests.Session()))
if not allow_attached:
unshipped_builds = _filter_out_inviable_builds(
kind, unshipped_builds, elliottlib.errata)
_json_dump(as_json, unshipped_builds, kind, tag_pv_map)
if not unshipped_builds:
green_print('No builds needed to be attached.')
return
if not advisory:
click.echo('The following {n} builds '.format(n=len(unshipped_builds)),
nl=False)
if not (remove or clean):
click.secho('may be attached', bold=True, nl=False)
click.echo(' to an advisory:')
else:
click.secho('may be removed from', bold=True, nl=False)
click.echo(' from an advisory:')
for b in sorted(unshipped_builds):
click.echo(' ' + b.nvr)
return
if not unshipped_builds and not (remove and unshipped_nvrps):
# Do not change advisory state unless strictly necessary
return
try:
erratum = elliottlib.errata.Advisory(errata_id=advisory)
erratum.ensure_state('NEW_FILES')
if remove:
to_remove = [
f"{nvrp[0]}-{nvrp[1]}-{nvrp[2]}" for nvrp in unshipped_nvrps
]
elif clean:
to_remove = [b.nvr for b in unshipped_builds]
if to_remove:
erratum.remove_builds(to_remove)
else: # attach
erratum.attach_builds(unshipped_builds, kind)
cdn_repos = et_data.get('cdn_repos')
if cdn_repos and not no_cdn_repos and kind == "image":
erratum.set_cdn_repos(cdn_repos)
except GSSError:
exit_unauthenticated()
except ErrataException as e:
red_print(f'Cannot change advisory {advisory}: {e}')
exit(1)
async def verify_payload(ctx, payload, advisory):
"""Cross-check that the builds present in PAYLOAD match the builds
attached to ADVISORY. The payload is treated as the source of
truth. If something is absent or different in the advisory it is
treated as an error with the advisory.
\b
PAYLOAD - Full pullspec of the payload to verify
ADVISORY - Numerical ID of the advisory
Two checks are made:
\b
1. Missing in Advisory - No payload components are absent from the given advisory
2. Payload Advisory Mismatch - The version-release of each payload item match what is in the advisory
Results are summarily printed at the end of the run. They are also
written out to summary_results.json.
Verify builds in the given payload match the builds attached to
advisory 41567
\b
$ elliott verify-payload quay.io/openshift-release-dev/ocp-release:4.1.0-rc.6 41567
"""
try:
green_prefix("Fetching advisory builds: ")
click.echo("Advisory - {}".format(advisory))
builds = elliottlib.errata.get_builds(advisory)
except GSSError:
exit_unauthenticated()
except elliottlib.exceptions.ErrataToolError as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
all_advisory_nvrs = {}
# Results come back with top level keys which are brew tags
green_prefix("Looping over tags: ")
click.echo("{} tags to check".format(len(builds)))
for tag in builds.keys():
# Each top level has a key 'builds' which is a list of dicts
green_prefix("Looping over builds in tag: ")
click.echo("{} with {} builds".format(tag, len(builds[tag]['builds'])))
for build in builds[tag]['builds']:
# Each dict has a top level key which might be the actual
# 'nvr' but I don't have enough data to know for sure
# yet. Also I don't know when there might be more than one
# key in the build dict. We'll loop over it to be sure.
for name in build.keys():
n, v, r = name.rsplit('-', 2)
version_release = "{}-{}".format(v, r)
all_advisory_nvrs[n] = version_release
click.echo("Found {} builds".format(len(all_advisory_nvrs)))
all_payload_nvrs = {}
click.echo("Fetching release info")
release_export_cmd = 'oc adm release info {} -o json'.format(payload)
rc, stdout, stderr = exectools.cmd_gather(release_export_cmd)
if rc != 0:
# Probably no point in continuing.. can't contact brew?
print("Unable to run oc release info: out={} ; err={}".format(
stdout, stderr))
exit(1)
else:
click.echo("Got release info")
payload_json = json.loads(stdout)
green_prefix("Looping over payload images: ")
click.echo("{} images to check".format(
len(payload_json['references']['spec']['tags'])))
cmds = [['oc', 'image', 'info', '-o', 'json', tag['from']['name']]
for tag in payload_json['references']['spec']['tags']]
green_prefix("Querying image infos...")
cmd_results = await asyncio.gather(
*[exectools.cmd_gather_async(cmd) for cmd in cmds])
for image, cmd, cmd_result in zip(
payload_json['references']['spec']['tags'], cmds, cmd_results):
click.echo("----")
image_name = image['name']
rc, stdout, stderr = cmd_result
if rc != 0:
# Probably no point in continuing.. can't contact brew?
red_prefix("Unable to run oc image info: ")
red_print(f"cmd={cmd!r}, out={stdout} ; err={stderr}")
exit(1)
image_info = json.loads(stdout)
labels = image_info['config']['config']['Labels']
# The machine-os-content image doesn't follow the standard
# pattern. We need to skip that image when we find it, it is
# not attached to advisories.
if 'com.coreos.ostree-commit' in labels:
yellow_prefix("Skipping machine-os-content image: ")
click.echo("Not required for checks")
continue
component = labels['com.redhat.component']
n = image_name
click.echo("Payload name: {}".format(n))
click.echo("Brew name: {}".format(component))
if labels:
v = labels['version']
r = labels['release']
all_payload_nvrs[component] = "{}-{}".format(v, r)
else:
print("For image {} Labels doesn't exist, image_info: {}".format(
image_name, image_info))
missing_in_errata = {}
payload_doesnt_match_errata = {}
in_other_advisories = {}
output = {
'missing_in_advisory': missing_in_errata,
'payload_advisory_mismatch': payload_doesnt_match_errata,
"in_other_advisories": in_other_advisories,
}
green_prefix("Analyzing data: ")
click.echo("{} images to consider from payload".format(
len(all_payload_nvrs)))
for image, vr in all_payload_nvrs.items():
yellow_prefix("Cross-checking from payload: ")
click.echo("{}-{}".format(image, vr))
if image not in all_advisory_nvrs:
missing_in_errata[image] = "{}-{}".format(image, vr)
click.echo("{} in payload not found in advisory".format(
"{}-{}".format(image, vr)))
elif image in all_advisory_nvrs and vr != all_advisory_nvrs[image]:
click.echo(
"{} from payload has version {} which does not match {} from advisory"
.format(image, vr, all_advisory_nvrs[image]))
payload_doesnt_match_errata[image] = {
'payload': vr,
'errata': all_advisory_nvrs[image]
}
if missing_in_errata:
green_print(
f"Checking if {len(missing_in_errata)} missing images are shipped..."
)
nvrs = list(missing_in_errata.values())
tag_lists = elliottlib.brew.get_builds_tags(nvrs)
for nvr, tags in zip(nvrs, tag_lists):
name = nvr.rsplit("-", 2)[0]
if any(map(lambda tag: tag["name"].endswith('-released'), tags)):
green_print(f"Build {nvr} is shipped. Skipping...")
del missing_in_errata[name]
elif any(map(lambda tag: tag["name"].endswith('-pending'), tags)):
green_print(f"Build {nvr} is in another advisory.")
del missing_in_errata[name]
in_other_advisories[name] = nvr
green_print("Summary results:")
click.echo(json.dumps(output, indent=4))
with open('summary_results.json', 'w') as fp:
json.dump(output, fp, indent=4)
green_prefix("Wrote out summary results: ")
click.echo("summary_results.json")
def get(ctx, runtime, default_advisory_type, details, id_only, as_json,
advisory):
"""Get details about a specific advisory from the Errata Tool. By
default a brief one-line informational string is printed. Use the
--details option to fetch and print the full details of the advisory.
Use of --id-only will override all other printing options. Requires
using --use-default-advisory. Only the ID of the advisory is printed
to standard out.
Fields for the short format: Release date, State, Synopsys, URL
Basic one-line output for advisory 123456:
\b
$ elliott get 123456
2018-02-23T18:34:40 NEW_FILES OpenShift Container Platform 3.9 bug fix and enhancement update - https://errata.devel.redhat.com/advisory/123456
Get the full JSON advisory object, use `jq` to print just the
errata portion of the advisory:
\b
$ elliott get --json - 123456 | jq '.errata'
{
"rhba": {
"actual_ship_date": null,
"assigned_to_id": 3002255,
"batch_id": null,
...
"""
runtime.initialize(no_group=default_advisory_type is None)
if bool(advisory) == bool(default_advisory_type):
raise click.BadParameter(
"Specify exactly one of --use-default-advisory or advisory arg")
if default_advisory_type is not None:
advisory = find_default_advisory(runtime,
default_advisory_type,
quiet=True)
if id_only:
click.echo(advisory)
return
try:
advisory = elliottlib.errata.Advisory(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if details:
click.echo(advisory)
return
if not as_json:
advisory_string = "{date} {state} {synopsis} {url}".format(
date=advisory.publish_date_override,
state=advisory.errata_state,
synopsis=advisory.synopsis,
url=advisory.url())
click.echo(advisory_string)
return
json_data = advisory.get_erratum_data()
json_data['bugs'] = advisory.errata_bugs
json_data['current_flags'] = advisory.current_flags
json_data['errata_builds'] = advisory.errata_builds
json_data['rpmdiffs'] = advisory.externalTests(test_type='rpmdiff')
if as_json == "-":
click.echo(json.dumps(json_data, indent=4, sort_keys=True))
return
with open(as_json, "w") as json_file:
json.dump(json_data, json_file, indent=4, sort_keys=True)
def add_bugs_with_retry(advisory, bugs, noop=False, batch_size=100):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bugs: iterable of bzutil.bug to attach to advisory
:return:
"""
print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}')
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
existing_bugs = advs.errata_bugs
new_bugs = set(bug.id for bug in bugs) - set(existing_bugs)
print(f'Bugs already attached: {len(existing_bugs)}')
print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}')
if not new_bugs:
print('No new bugs to attach. Exiting.')
return
bugs = list(new_bugs)
batches = list(range(0, len(bugs), batch_size))
if len(bugs) % batch_size != 0:
batches.append(len(bugs))
green_prefix(
f"Adding bugs in batches of {batch_size}. Number of batches: {len(batches)-1}\n"
)
for i in range(len(batches) - 1):
start, end = batches[i], batches[i + 1]
print(f"Attaching Batch {i+1}")
if noop:
print('Dry run: Would have attached bugs')
continue
try:
advs.addBugs(bugs[start:end])
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
block_list = parse_exception_error_message(e)
retry_list = [x for x in bugs[start:end] if x not in block_list]
if len(retry_list) == 0:
continue
try:
advs = Erratum(errata_id=advisory)
advs.addBugs(retry_list)
advs.commit()
except ErrataException as e:
raise exceptions.ElliottFatalError(
getattr(e, 'message', repr(e)))
print("remaining bugs attached")