def add_bugs_with_retry(advisory, bug_list, retried):
"""
adding specified bug_list into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bug_list: bug id list which suppose to attach to advisory
:param retried: retry 2 times, first attempt fetch failed bugs sift out then attach again
:return:
"""
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
green_prefix("Adding {count} bugs to advisory {retry_times} times:".format(
count=len(bug_list), retry_times=1 if retried is False else 2))
print(" {advs}".format(advs=advs))
try:
advs.addBugs(bug_list)
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
if retried is not True:
black_list = parse_exception_error_message(e)
retry_list = [x for x in bug_list if x not in black_list]
if len(retry_list) > 0:
add_bugs_with_retry(advisory, retry_list, True)
else:
raise exceptions.ElliottFatalError(getattr(e, 'message', repr(e)))
def add_bugzilla_bugs_with_retry(
advisory_id: int,
bugids: List,
noop: bool = False,
batch_size: int = constants.BUG_ATTACH_CHUNK_SIZE):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory_id: advisory id
:param bugids: iterable of bugzilla bug ids to attach to advisory
:param noop: do not modify anything
:param batch_size: perform operation in batches of given size
:return:
"""
logger.info(
f'Request to attach {len(bugids)} bugs to the advisory {advisory_id}')
try:
advisory = Erratum(errata_id=advisory_id)
except GSSError:
exit_unauthenticated()
if not advisory:
raise exceptions.ElliottFatalError(
f"Error: Could not locate advisory {advisory_id}")
existing_bugs = bzutil.BugzillaBugTracker.advisory_bug_ids(advisory)
new_bugs = set(bugids) - set(existing_bugs)
logger.info(
f'Bugs already attached: {len(existing_bugs)}. New bugs: {len(new_bugs)}'
)
if not new_bugs:
return
for chunk_of_bugs in chunk(list(new_bugs), batch_size):
if noop:
logger.info('Dry run: Would have attached bugs')
continue
try:
advisory.addBugs(chunk_of_bugs)
advisory.commit()
except ErrataException as e:
logger.info(f"ErrataException Message: {e}\nRetrying...")
block_list = parse_exception_error_message(e)
retry_list = [x for x in chunk_of_bugs if x not in block_list]
if len(retry_list) == 0:
continue
try:
advisory = Erratum(errata_id=advisory_id)
advisory.addBugs(retry_list)
advisory.commit()
except ErrataException as e:
raise exceptions.ElliottFatalError(
getattr(e, 'message', repr(e)))
logger.info("remaining bugs attached")
logger.info("All bugzilla bugs attached")
def add_bugs_with_retry(advisory, bugs, retried=False, noop=False):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bugs: iterable of bzutil.bug to attach to advisory
:param retried: retry 2 times, first attempt fetch failed bugs sift out then attach again
:return:
"""
print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}')
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
existing_bugs = advs.errata_bugs
new_bugs = set(bug.id for bug in bugs) - set(existing_bugs)
print(f'Bugs already attached: {len(existing_bugs)}')
print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}')
if noop:
print('Dry run. Exiting.')
return
if not new_bugs:
print('No new bugs to attach. Exiting.')
return
green_prefix("Adding {count} bugs to advisory {retry_times} times:".format(
count=len(bugs), retry_times=1 if retried is False else 2))
try:
advs.addBugs([bug.id for bug in bugs])
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
if retried is not True:
block_list = parse_exception_error_message(e)
retry_list = [x for x in bugs if x.id not in block_list]
if len(retry_list) > 0:
add_bugs_with_retry(advisory,
retry_list,
retried=True,
noop=noop)
else:
raise exceptions.ElliottFatalError(getattr(e, 'message', repr(e)))
def get_advisory_nvrs(advisory):
"""
:return: dict, with keys as package names and values as strs in the form: '{version}-{release}'
"""
try:
green_prefix("Fetching advisory builds: ")
click.echo("Advisory - {}".format(advisory))
builds = get_builds(advisory)
except GSSError:
exit_unauthenticated()
except exceptions.ErrataToolError as ex:
raise exceptions.ElliottFatalError(getattr(ex, 'message', repr(ex)))
all_advisory_nvrs = {}
# Results come back with top level keys which are brew tags
green_prefix("Looping over tags: ")
click.echo("{} tags to check".format(len(builds)))
for tag in builds.keys():
# Each top level has a key 'builds' which is a list of dicts
green_prefix("Looping over builds in tag: ")
click.echo("{} with {} builds".format(tag, len(builds[tag]['builds'])))
for build in builds[tag]['builds']:
# Each dict has a top level key which might be the actual
# 'nvr' but I don't have enough data to know for sure
# yet. Also I don't know when there might be more than one
# key in the build dict. We'll loop over it to be sure.
for name in build.keys():
n, v, r = name.rsplit('-', 2)
version_release = "{}-{}".format(v, r)
all_advisory_nvrs[n] = version_release
return all_advisory_nvrs
def get_corresponding_flaw_bugs(bzapi, tracker_bugs, fields, strict=False):
"""
Get corresponding flaw bugs objects for the
given tracker bug objects
"""
# fields needed for is_flaw_bug()
if "product" not in fields:
fields.append("product")
if "component" not in fields:
fields.append("component")
blocking_bugs = bzapi.getbugs(unique(
flatten([t.blocks for t in tracker_bugs])),
include_fields=fields,
permissive=False)
flaw_bugs = [
flaw_bug for flaw_bug in blocking_bugs if bzutil.is_flaw_bug(flaw_bug)
]
# Validate that each tracker has a corresponding flaw bug
flaw_ids = set([b.id for b in flaw_bugs])
no_flaws = set()
for tracker in tracker_bugs:
if not set(tracker.blocks).intersection(flaw_ids):
no_flaws.add(tracker.id)
if no_flaws:
msg = f'No flaw bugs could be found for these trackers: {no_flaws}'
if strict:
raise exceptions.ElliottFatalError(msg)
else:
logger.warn(msg)
return flaw_bugs
def get_corresponding_flaw_bugs(self, tracker_bugs: List[Bug], flaw_bug_tracker=None,
strict: bool = False, verbose: bool = False):
"""Get corresponding flaw bug objects for given list of tracker bug objects.
Accepts a flaw_bug_tracker object to fetch flaw bugs from incase it's different from self
:return: (tracker_flaws, flaw_id_bugs): tracker_flaws is a dict with tracker bug id as key and list of flaw
bug id as value, flaw_id_bugs is a dict with flaw bug id as key and flaw bug object as value
"""
bug_tracker = flaw_bug_tracker if flaw_bug_tracker else self
flaw_bugs = bug_tracker.get_flaw_bugs(
list(set(sum([t.corresponding_flaw_bug_ids for t in tracker_bugs], []))),
verbose=verbose
)
flaw_id_bugs = {bug.id: bug for bug in flaw_bugs}
# Validate that each tracker has a corresponding flaw bug
flaw_ids = set(flaw_id_bugs.keys())
no_flaws = set()
for tracker in tracker_bugs:
if not set(tracker.corresponding_flaw_bug_ids).intersection(flaw_ids):
no_flaws.add(tracker.id)
if no_flaws:
msg = f'No flaw bugs could be found for these trackers: {no_flaws}'
if strict:
raise exceptions.ElliottFatalError(msg)
else:
logger.warn(msg)
tracker_flaws = {
tracker.id: [b for b in tracker.corresponding_flaw_bug_ids if b in flaw_id_bugs]
for tracker in tracker_bugs
}
return tracker_flaws, flaw_id_bugs
def add_jira_bugs_with_retry(
advisory_id: int,
bugids: List[str],
noop: bool = False,
batch_size: int = constants.BUG_ATTACH_CHUNK_SIZE):
"""
:param advisory_id: advisory id
:param bugids: iterable of jira bug ids to attach to advisory
:param noop: do not modify anything
:param batch_size: perform operation in batches of given size
"""
logger.info(
f'Request to attach {len(bugids)} bugs to the advisory {advisory_id}')
try:
advisory = Erratum(errata_id=advisory_id)
except GSSError:
exit_unauthenticated()
if not advisory:
raise exceptions.ElliottFatalError(
f"Error: Could not locate advisory {advisory_id}")
existing_bugs = bzutil.JIRABugTracker.advisory_bug_ids(advisory)
new_bugs = set(bugids) - set(existing_bugs)
logger.info(
f'Bugs already attached: {len(existing_bugs)}. New bugs: {len(new_bugs)}'
)
if not new_bugs:
return
for chunk_of_bugs in chunk(bugids, batch_size):
if noop:
logger.info('Dry run: Would have attached bugs')
continue
results = add_multi_jira_issues(advisory_id, chunk_of_bugs)
for i, result in enumerate(results):
if result.status_code != 201:
rt = add_jira_issue(advisory_id, chunk_of_bugs[i])
if rt.status_code != 201:
raise exceptions.ElliottFatalError(
f"attach jira bug {chunk_of_bugs[i]} failed with "
f"status={rt.status_code}: {rt.json()}")
logger.info("All jira bugs attached")
def get_all_advisory_nvrs(advisory):
"""
:return: list of tuples (name, version, release)
"""
try:
builds = get_builds(advisory)
except GSSError:
exit_unauthenticated()
except exceptions.ErrataToolError as ex:
raise exceptions.ElliottFatalError(getattr(ex, 'message', repr(ex)))
all_advisory_nvrs = []
# Results come back with top level keys which are brew tags
for tag in builds.keys():
# Each top level has a key 'builds' which is a list of dicts
for build in builds[tag]['builds']:
for name in build.keys():
n, v, r = name.rsplit('-', 2)
all_advisory_nvrs.append((n, v, r))
return all_advisory_nvrs
def tag_builds_cli(runtime: Runtime, advisories: Tuple[int],
default_advisory_type: str, product_version: str,
builds: Tuple[str], tag: str, dont_untag: bool,
dry_run: bool):
""" Tag builds into Brew tag and optionally untag unspecified builds.
Example 1: Tag RHEL7 RPMs that on ocp-build-data recorded advisory into rhaos-4.3-rhel-7-image-build
$ elliott --group=openshift-4.3 tag-builds --use-default-advisory rpm --product-version RHEL-7-OSE-4.3 --tag rhaos-4.3-rhel-7-image-build
Example 2: Tag RHEL8 RPMs that are on advisory 55016 into rhaos-4.3-rhel-8-image-build
$ elliott --group=openshift-4.3 tag-builds --advisory 55016 --product-version OSE-4.4-RHEL-8 --tag rhaos-4.3-rhel-8-image-build
Example 3: Tag specified builds into rhaos-4.3-rhel-8-image-build
$ elliott --group=openshift-4.3 tag-builds --build buildah-1.11.6-6.rhaos4.3.el8 --build openshift-4.3.23-202005230952.g1.b596217.el8 --tag rhaos-4.3-rhel-8-image-build
"""
if advisories and builds:
raise click.BadParameter('Use only one of --build or --advisory/-a.')
if advisories and default_advisory_type:
raise click.BadParameter(
'Use only one of --use-default-advisory or --advisory/-a.')
if default_advisory_type and builds:
raise click.BadParameter(
'Use only one of --build or --use-default-advisory.')
if product_version and not advisories and not default_advisory_type:
raise click.BadParameter(
'--product-version should only be used with --use-default-advisory or --advisory/-a.'
)
runtime.initialize()
logger = runtime.logger
if default_advisory_type:
advisories = (find_default_advisory(runtime, default_advisory_type), )
all_builds = set() # All Brew builds that should be in the tag
if advisories:
errata_session = requests.session()
for advisory in advisories:
logger.info(
f"Fetching attached Brew builds from advisory {advisory}...")
errata_builds = errata.get_builds(advisory, errata_session)
product_versions = list(errata_builds.keys())
logger.debug(
f"Advisory {advisory} has builds for {len(product_versions)} product versions: {product_versions}"
)
if product_version: # Only this product version should be concerned
product_versions = [product_version]
for pv in product_versions:
logger.debug(f"Extract Errata builds for product version {pv}")
nvrs = _extract_nvrs_from_errata_build_list(errata_builds, pv)
logger.info(
f"Found {len(nvrs)} builds from advisory {advisory} with product version {pv}"
)
logger.debug(
f"The following builds are found for product version {pv}:\n\t{list(nvrs)}"
)
all_builds |= set(nvrs)
brew_session = koji.ClientSession(runtime.group_config.urls.brewhub
or constants.BREW_HUB)
if builds: # NVRs are directly specified with --build
build_objs = brew.get_build_objects(list(builds), brew_session)
all_builds = {build["nvr"] for build in build_objs}
click.echo(
f"The following {len(all_builds)} build(s) should be in tag {tag}:")
for nvr in all_builds:
green_print(f"\t{nvr}")
# get NVRs that have been tagged
tagged_build_objs = brew_session.listTagged(tag,
latest=False,
inherit=False)
tagged_builds = {build["nvr"] for build in tagged_build_objs}
# get NVRs that should be tagged
missing_builds = all_builds - tagged_builds
click.echo(f"{len(missing_builds)} build(s) need to be tagged into {tag}:")
for nvr in missing_builds:
green_print(f"\t{nvr}")
# get NVRs that should be untagged
extra_builds = tagged_builds - all_builds
click.echo(f"{len(extra_builds)} build(s) need to be untagged from {tag}:")
for nvr in extra_builds:
green_print(f"\t{nvr}")
if dry_run:
yellow_print("Dry run: Do nothing.")
return
brew_session.gssapi_login()
if not dont_untag:
# untag extra builds
extra_builds = list(extra_builds)
logger.info(f"Untagging {len(extra_builds)} build(s) from {tag}...")
multicall_tasks = brew.untag_builds(tag, extra_builds, brew_session)
failed_to_untag = []
for index, task in enumerate(multicall_tasks):
try:
task.result
click.echo(f"{nvr} has been successfully untagged from {tag}")
except Exception as ex:
nvr = extra_builds[index]
failed_to_untag.append(nvr)
logger.error(f"Failed to untag {nvr}: {ex}")
# tag missing builds
missing_builds = list(missing_builds)
task_id_nvr_map = {}
logger.info(f"Tagging {len(missing_builds)} build(s) into {tag}...")
multicall_tasks = brew.tag_builds(tag, missing_builds, brew_session)
failed_to_tag = []
for index, task in enumerate(multicall_tasks):
nvr = missing_builds[index]
try:
task_id = task.result
task_id_nvr_map[task_id] = nvr
except Exception as ex:
failed_to_tag.append(nvr)
logger.error(f"Failed to tag {nvr}: {ex}")
if task_id_nvr_map:
# wait for tag task to finish
logger.info("Waiting for tag tasks to finish")
brew.wait_tasks(task_id_nvr_map.keys(), brew_session, logger=logger)
# get tagging results
stopped_tasks = list(task_id_nvr_map.keys())
with brew_session.multicall(strict=False) as m:
multicall_tasks = []
for task_id in stopped_tasks:
multicall_tasks.append(
m.getTaskResult(task_id, raise_fault=False))
for index, t in enumerate(multicall_tasks):
task_id = stopped_tasks[index]
nvr = task_id_nvr_map[task_id]
tag_res = t.result
logger.debug(
f"Tagging task {task_id} {nvr} returned result {tag_res}")
click.echo(f"{nvr} has been successfully tagged into {tag}")
if tag_res and 'faultCode' in tag_res:
if "already tagged" not in tag_res["faultString"]:
failed_to_tag.append(nvr)
logger.error(
f'Failed to tag {nvr} into {tag}: {tag_res["faultString"]}'
)
if failed_to_untag:
red_print("The following builds were failed to untag:")
for nvr in failed_to_untag:
red_print(f"\t{nvr}")
elif not dont_untag:
green_print(
f"All unspecified builds have been successfully untagged from {tag}."
)
if failed_to_tag:
red_print("The following builds were failed to tag:")
for nvr in failed_to_tag:
red_print(f"\t{nvr}")
else:
green_print(f"All builds have been successfully tagged into {tag}.")
if failed_to_untag or failed_to_tag:
raise exceptions.ElliottFatalError(
"Not all builds were successfully tagged/untagged.")
def add_bugs_with_retry(advisory, bugs, noop=False, batch_size=100):
"""
adding specified bugs into advisory, retry 2 times: first time
parse the exception message to get failed bug id list, remove from original
list then add bug to advisory again, if still has failures raise exceptions
:param advisory: advisory id
:param bugs: iterable of bzutil.bug to attach to advisory
:return:
"""
print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}')
try:
advs = Erratum(errata_id=advisory)
except GSSError:
exit_unauthenticated()
if advs is False:
raise exceptions.ElliottFatalError(
"Error: Could not locate advisory {advs}".format(advs=advisory))
existing_bugs = advs.errata_bugs
new_bugs = set(bug.id for bug in bugs) - set(existing_bugs)
print(f'Bugs already attached: {len(existing_bugs)}')
print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}')
if not new_bugs:
print('No new bugs to attach. Exiting.')
return
bugs = list(new_bugs)
batches = list(range(0, len(bugs), batch_size))
if len(bugs) % batch_size != 0:
batches.append(len(bugs))
green_prefix(
f"Adding bugs in batches of {batch_size}. Number of batches: {len(batches)-1}\n"
)
for i in range(len(batches) - 1):
start, end = batches[i], batches[i + 1]
print(f"Attaching Batch {i+1}")
if noop:
print('Dry run: Would have attached bugs')
continue
try:
advs.addBugs(bugs[start:end])
advs.commit()
except ErrataException as e:
print("ErrataException Message: {}, retry it again".format(e))
block_list = parse_exception_error_message(e)
retry_list = [x for x in bugs[start:end] if x not in block_list]
if len(retry_list) == 0:
continue
try:
advs = Erratum(errata_id=advisory)
advs.addBugs(retry_list)
advs.commit()
except ErrataException as e:
raise exceptions.ElliottFatalError(
getattr(e, 'message', repr(e)))
print("remaining bugs attached")