def add_bugs_with_retry(advisory, bug_list, retried):
"""
adding specified bug_list into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bug_list: bug id list which suppose to attach to advisory
:param retried: retry 2 times, first attempt fetch failed bugs sift out then attach again
:return:
"""
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
green_prefix("Adding {count} bugs to advisory {retry_times} times:".format(
count=len(bug_list), retry_times=1 if retried is False else 2))
print(" {advs}".format(advs=advs))
try:
advs.addBugs(bug_list)
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
if retried is not True:
black_list = parse_exception_error_message(e)
retry_list = [x for x in bug_list if x not in black_list]
if len(retry_list) > 0:
add_bugs_with_retry(advisory, retry_list, True)
else:
raise exceptions.ElliottFatalError(getattr(e, 'message', repr(e)))
def add_bugzilla_bugs_with_retry(
advisory_id: int,
bugids: List,
noop: bool = False,
batch_size: int = constants.BUG_ATTACH_CHUNK_SIZE):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory_id: advisory id
:param bugids: iterable of bugzilla bug ids to attach to advisory
:param noop: do not modify anything
:param batch_size: perform operation in batches of given size
:return:
"""
logger.info(
f'Request to attach {len(bugids)} bugs to the advisory {advisory_id}')
try:
advisory = Erratum(errata_id=advisory_id)
except GSSError:
exit_unauthenticated()
if not advisory:
raise exceptions.ElliottFatalError(
f"Error: Could not locate advisory {advisory_id}")
existing_bugs = bzutil.BugzillaBugTracker.advisory_bug_ids(advisory)
new_bugs = set(bugids) - set(existing_bugs)
logger.info(
f'Bugs already attached: {len(existing_bugs)}. New bugs: {len(new_bugs)}'
)
if not new_bugs:
return
for chunk_of_bugs in chunk(list(new_bugs), batch_size):
if noop:
logger.info('Dry run: Would have attached bugs')
continue
try:
advisory.addBugs(chunk_of_bugs)
advisory.commit()
except ErrataException as e:
logger.info(f"ErrataException Message: {e}\nRetrying...")
block_list = parse_exception_error_message(e)
retry_list = [x for x in chunk_of_bugs if x not in block_list]
if len(retry_list) == 0:
continue
try:
advisory = Erratum(errata_id=advisory_id)
advisory.addBugs(retry_list)
advisory.commit()
except ErrataException as e:
raise exceptions.ElliottFatalError(
getattr(e, 'message', repr(e)))
logger.info("remaining bugs attached")
logger.info("All bugzilla bugs attached")
def add_bugs_with_retry(advisory, bugs, retried=False, noop=False):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bugs: iterable of bzutil.bug to attach to advisory
:param retried: retry 2 times, first attempt fetch failed bugs sift out then attach again
:return:
"""
print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}')
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
existing_bugs = advs.errata_bugs
new_bugs = set(bug.id for bug in bugs) - set(existing_bugs)
print(f'Bugs already attached: {len(existing_bugs)}')
print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}')
if noop:
print('Dry run. Exiting.')
return
if not new_bugs:
print('No new bugs to attach. Exiting.')
return
green_prefix("Adding {count} bugs to advisory {retry_times} times:".format(
count=len(bugs), retry_times=1 if retried is False else 2))
try:
advs.addBugs([bug.id for bug in bugs])
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
if retried is not True:
block_list = parse_exception_error_message(e)
retry_list = [x for x in bugs if x.id not in block_list]
if len(retry_list) > 0:
add_bugs_with_retry(advisory,
retry_list,
retried=True,
noop=noop)
else:
raise exceptions.ElliottFatalError(getattr(e, 'message', repr(e)))
def create_placeholder_cli(runtime, kind, advisory, default_advisory_type):
"""Create a placeholder bug for attaching to an advisory.
KIND - The kind of placeholder to create ({}).
ADVISORY - Optional. The advisory to attach the bug to.
$ elliott --group openshift-4.1 create-placeholder --kind rpm --attach 12345
""".format('/'.join(elliottlib.constants.standard_advisory_types))
runtime.initialize()
if advisory and default_advisory_type:
raise click.BadParameter(
"Use only one of --use-default-advisory or --advisory")
if default_advisory_type is not None:
advisory = find_default_advisory(runtime, default_advisory_type)
kind = default_advisory_type
if kind is None:
raise click.BadParameter(
"--kind must be specified when not using --use-default-advisory")
bz_data = runtime.gitdata.load_data(key='bugzilla').data
target_release = bz_data['target_release'][0]
newbug = elliottlib.bzutil.create_placeholder(bz_data, kind,
target_release)
click.echo("Created BZ: {} {}".format(newbug.id, newbug.weburl))
if advisory is not False:
click.echo("Attaching to advisory...")
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise ElliottFatalError(
"Error: Could not locate advisory {advs}".format(
advs=advisory))
try:
green_prefix("Adding placeholder bug to advisory:")
click.echo(" {advs}".format(advs=advisory))
advs.addBugs([newbug.id])
advs.commit()
except ErrataException as ex:
raise ElliottFatalError(getattr(ex, 'message', repr(ex)))
elif args.errata_type == "RHSA":
pass
# Ensure bugs are in proper state (MODIFIED or VERIFIED) and that they have
# the appropriate rhel-fast-datapath flag set.
s = utilities.open_session()
bz_bugs = utilities.get_bz_bugs(s, args.bugs)
for bug in bz_bugs:
if bug['status'] != 'MODIFIED' and bug['status'] != 'VERIFIED':
utilities.update_bz(s, bug['id'], json={'status': 'MODIFIED'})
utilities.set_bz_flag(s, bug['id'], f'fast-datapath-rhel-{release[-1]}',
'+')
e = Erratum(product='Fast-Datapath',
release=release,
errata_type=args.errata_type,
security_impact=args.security_impact,
synopsis=synopsis,
topic=topic,
description=description,
solution=solution,
qe_email='*****@*****.**',
qe_group='OVS QE',
owner_email=args.owner_email,
manager_email='*****@*****.**')
e.addBugs(args.bugs)
e.commit()
e.addBuilds([args.build], release=f"RHEL-{release[-1]}-Fast-Datapath")
print(e.url())
def create_textonly_cli(ctx, runtime, errata_type, date, assigned_to, manager,
package_owner, topic, synopsis, description, solution,
bugtitle, bugdescription, yes):
"""
Create a text only advisory with all required input passed from args, need to manually decide the statement for each release.
Also will create the notification bug along with the text only advisory, the bug also need some special comment and title.
These args need to be designated manually for text only advisory:
- topic
- synopsis
- description
- solution
- assigned
These args need to be designated manually for text only bug:
- bugtitle
- bugdescription
"""
runtime.initialize()
# create textonly bug
bz_data = runtime.gitdata.load_data(key='bugzilla').data
newbug = elliottlib.bzutil.create_textonly(bz_data, bugtitle,
bugdescription)
click.echo("Created BZ: {} {}".format(newbug.id, newbug.weburl))
# create textonly advisory
et_data = runtime.gitdata.load_data(key='erratatool').data
try:
erratum = Erratum(
product=et_data['product'],
release=et_data['release'],
qe_group=et_data['quality_responsibility_name'],
synopsis=synopsis,
topic=topic,
description=description,
solution=solution,
qe_email=assigned_to,
errata_type=errata_type,
owner_email=package_owner,
manager_email=manager,
date=date,
text_only=1,
)
except elliottlib.exceptions.ErrataToolUnauthorizedException:
exit_unauthorized()
except elliottlib.exceptions.ErrataToolError as ex:
raise repr(ex)
erratum.addBugs(newbug.id)
cdn_repos = et_data.get('cdn_repos')
if cdn_repos:
click.echo(f"Configuring CDN repos {', '.join(cdn_repos)}...")
erratum.textOnlyRepos(enable=cdn_repos)
if yes:
erratum.commit()
green_prefix("Created new text only advisory: ")
click.echo(str(erratum))
else:
green_prefix("Would have created advisory: ")
click.echo("")
click.echo(erratum)
def attach_cve_flaws_cli(runtime, advisory_id, noop, default_advisory_type):
"""Attach corresponding flaw bugs for trackers in advisory (first-fix only).
Also converts advisory to RHSA, if not already.
Example:
$ elliott --group openshift-4.6 attach-cve-flaws --use-default-advisory image
INFO Cloning config data from https://github.com/openshift/ocp-build-data.git
INFO Using branch from group.yml: rhaos-4.6-rhel-8
INFO found 114 tracker bugs attached to the advisory
INFO found 58 corresponding flaw bugs
INFO 23 out of 58 flaw bugs considered "first-fix"
INFO Adding the following BZs to the advisory: [1880456, 1858827, 1880460,
1847310, 1857682, 1857550, 1857551, 1857559, 1848089, 1848092, 1849503,
1851422, 1866148, 1858981, 1852331, 1861044, 1857081, 1857977, 1848647,
1849044, 1856529, 1843575, 1840253]
"""
runtime.initialize()
bzurl = runtime.gitdata.load_data(key='bugzilla').data['server']
bzapi = bugzilla.Bugzilla(bzurl)
if not advisory_id and default_advisory_type is not None:
advisory_id = find_default_advisory(runtime, default_advisory_type)
attached_tracker_bugs = get_attached_tracker_bugs(bzapi, advisory_id)
runtime.logger.info(
'found {} tracker bugs attached to the advisory'.format(
len(attached_tracker_bugs)))
corresponding_flaw_bugs = get_corresponding_flaw_bugs(
bzapi, attached_tracker_bugs)
runtime.logger.info('found {} corresponding flaw bugs'.format(
len(corresponding_flaw_bugs)))
attached_tracker_ids = [tracker.id for tracker in attached_tracker_bugs]
current_target_release = runtime.gitdata.load_data(
key='bugzilla').data['target_release']
first_fix_flaw_bugs = [
flaw_bug for flaw_bug in corresponding_flaw_bugs if is_first_fix(
bzapi, flaw_bug, current_target_release, attached_tracker_ids)
]
runtime.logger.info('{} out of {} flaw bugs considered "first-fix"'.format(
len(first_fix_flaw_bugs),
len(corresponding_flaw_bugs),
))
if not first_fix_flaw_bugs:
runtime.logger.info('No "first-fix" bugs found, exiting')
exit(0)
advisory = Erratum(errata_id=advisory_id)
if not is_security_advisory(advisory):
runtime.logger.info(
'Advisory type is {}, converting it to RHSA'.format(
advisory.errata_type))
cve_boilerplate = runtime.gitdata.load_data(
key='erratatool').data['boilerplates']['cve']
advisory.update(
errata_type='RHSA',
security_reviewer=cve_boilerplate['security_reviewer'],
synopsis=cve_boilerplate['synopsis'],
description=cve_boilerplate['description'],
topic=cve_boilerplate['topic'],
solution=cve_boilerplate['solution'],
security_impact='Low',
)
cves = ' '.join([flaw_bug.alias[0] for flaw_bug in first_fix_flaw_bugs])
advisory.update(cve_names="{} {}".format(advisory.cve_names, cves).strip())
print('List of *new* CVEs: {}'.format(cves))
highest_impact = get_highest_security_impact(first_fix_flaw_bugs)
if is_advisory_impact_smaller_than(advisory, highest_impact):
runtime.logger.info(
'Adjusting advisory security impact from {} to {}'.format(
advisory.security_impact, highest_impact))
advisory.update(security_impact=highest_impact)
flaw_ids = [flaw_bug.id for flaw_bug in first_fix_flaw_bugs]
runtime.logger.info(
'Adding the following BZs to the advisory: {}'.format(flaw_ids))
advisory.addBugs(flaw_ids)
if noop:
print(
'DRY-RUN: The following changes would have been applied to the advisory:'
)
print(advisory)
return True
return advisory.commit()
def add_bugs_with_retry(advisory, bugs, noop=False, batch_size=100):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bugs: iterable of bzutil.bug to attach to advisory
:return:
"""
print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}')
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
existing_bugs = advs.errata_bugs
new_bugs = set(bug.id for bug in bugs) - set(existing_bugs)
print(f'Bugs already attached: {len(existing_bugs)}')
print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}')
if not new_bugs:
print('No new bugs to attach. Exiting.')
return
bugs = list(new_bugs)
batches = list(range(0, len(bugs), batch_size))
if len(bugs) % batch_size != 0:
batches.append(len(bugs))
green_prefix(
f"Adding bugs in batches of {batch_size}. Number of batches: {len(batches)-1}\n"
)
for i in range(len(batches) - 1):
start, end = batches[i], batches[i + 1]
print(f"Attaching Batch {i+1}")
if noop:
print('Dry run: Would have attached bugs')
continue
try:
advs.addBugs(bugs[start:end])
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
block_list = parse_exception_error_message(e)
retry_list = [x for x in bugs[start:end] if x not in block_list]
if len(retry_list) == 0:
continue
try:
advs = Erratum(errata_id=advisory)
advs.addBugs(retry_list)
advs.commit()
except ErrataException as e:
raise exceptions.ElliottFatalError(
getattr(e, 'message', repr(e)))
print("remaining bugs attached")
def attach_cve_flaws_cli(runtime, advisory_id, noop, default_advisory_type):
"""Attach corresponding flaw bugs for trackers in advisory (first-fix only).
Also converts advisory to RHSA, if not already.
Example:
$ elliott --group openshift-4.6 attach-cve-flaws --use-default-advisory image
INFO Cloning config data from https://github.com/openshift/ocp-build-data.git
INFO Using branch from group.yml: rhaos-4.6-rhel-8
INFO found 114 tracker bugs attached to the advisory
INFO found 58 corresponding flaw bugs
INFO 23 out of 58 flaw bugs considered "first-fix"
INFO Adding the following BZs to the advisory: [1880456, 1858827, 1880460,
1847310, 1857682, 1857550, 1857551, 1857559, 1848089, 1848092, 1849503,
1851422, 1866148, 1858981, 1852331, 1861044, 1857081, 1857977, 1848647,
1849044, 1856529, 1843575, 1840253]
"""
runtime.initialize()
bzurl = runtime.gitdata.bz_server_url()
bzapi = bugzilla.Bugzilla(bzurl)
if not advisory_id and default_advisory_type is not None:
advisory_id = find_default_advisory(runtime, default_advisory_type)
# get attached bugs from advisory
attached_tracker_bugs = get_attached_tracker_bugs(bzapi, advisory_id)
runtime.logger.info(
'found {} tracker bugs attached to the advisory: {}'.format(
len(attached_tracker_bugs),
sorted(bug.id for bug in attached_tracker_bugs)))
if len(attached_tracker_bugs) == 0:
exit(0)
# validate and get target_release
current_target_release, err = util.get_target_release(
attached_tracker_bugs)
if err:
runtime.logger.error(err)
exit(1)
runtime.logger.info(
'current_target_release: {}'.format(current_target_release))
corresponding_flaw_bugs = get_corresponding_flaw_bugs(
bzapi, attached_tracker_bugs)
runtime.logger.info('found {} corresponding flaw bugs: {}'.format(
len(corresponding_flaw_bugs),
sorted(bug.id for bug in corresponding_flaw_bugs)))
# current_target_release is digit.digit.[z|0]
# if current_target_release is GA then run first-fix bug filtering
# for GA not every flaw bug is considered first-fix
# for z-stream every flaw bug is considered first-fix
if current_target_release[-1] == 'z':
runtime.logger.info(
"detected z-stream target release, every flaw bug is considered first-fix"
)
first_fix_flaw_bugs = corresponding_flaw_bugs
else:
runtime.logger.info(
"detected GA release, applying first-fix filtering..")
attached_tracker_ids = [
tracker.id for tracker in attached_tracker_bugs
]
first_fix_flaw_bugs = [
flaw_bug for flaw_bug in corresponding_flaw_bugs if is_first_fix(
bzapi, flaw_bug, current_target_release, attached_tracker_ids)
]
runtime.logger.info('{} out of {} flaw bugs considered "first-fix"'.format(
len(first_fix_flaw_bugs),
len(corresponding_flaw_bugs),
))
if not first_fix_flaw_bugs:
runtime.logger.info('No "first-fix" bugs found, exiting')
exit(0)
advisory = Erratum(errata_id=advisory_id)
if not is_security_advisory(advisory):
runtime.logger.info(
'Advisory type is {}, converting it to RHSA'.format(
advisory.errata_type))
cve_boilerplate = runtime.gitdata.load_data(
key='erratatool').data['boilerplates']['cve']
advisory.update(
errata_type='RHSA',
security_reviewer=cve_boilerplate['security_reviewer'],
synopsis=cve_boilerplate['synopsis'],
description=cve_boilerplate['description'],
topic=cve_boilerplate['topic'],
solution=cve_boilerplate['solution'],
security_impact='Low',
)
cves = ' '.join([flaw_bug.alias[0] for flaw_bug in first_fix_flaw_bugs])
cve_str = cves
if advisory.cve_names and cves not in advisory.cve_names:
cve_str = "{} {}".format(advisory.cve_names, cves).strip()
advisory.update(cve_names=cve_str)
runtime.logger.info('List of *new* CVEs: {}'.format(cves))
highest_impact = get_highest_security_impact(first_fix_flaw_bugs)
if is_advisory_impact_smaller_than(advisory, highest_impact):
runtime.logger.info(
'Adjusting advisory security impact from {} to {}'.format(
advisory.security_impact, highest_impact))
advisory.update(security_impact=highest_impact)
flaw_ids = [flaw_bug.id for flaw_bug in first_fix_flaw_bugs]
runtime.logger.info(
f'Request to attach {len(flaw_ids)} bugs to the advisory')
existing_bug_ids = advisory.errata_bugs
new_bugs = set(flaw_ids) - set(existing_bug_ids)
print(f'Bugs already attached: {len(existing_bug_ids)}')
print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}')
if new_bugs:
advisory.addBugs(flaw_ids)
if noop:
print(
'DRY-RUN: The following changes would have been applied to the advisory:'
)
print(advisory)
return True
return advisory.commit()