def add_bugs_with_retry(advisory, bug_list, retried): """ adding specified bug_list into advisory, retry 2 times: first time parse the exception message to get failed bug id list, remove from original list then add bug to advisory again, if still has failures raise exceptions :param advisory: advisory id :param bug_list: bug id list which suppose to attach to advisory :param retried: retry 2 times, first attempt fetch failed bugs sift out then attach again :return: """ try: advs = Erratum(errata_id=advisory) except GSSError: exit_unauthenticated() if advs is False: raise exceptions.ElliottFatalError( "Error: Could not locate advisory {advs}".format(advs=advisory)) green_prefix("Adding {count} bugs to advisory {retry_times} times:".format( count=len(bug_list), retry_times=1 if retried is False else 2)) print(" {advs}".format(advs=advs)) try: advs.addBugs(bug_list) advs.commit() except ErrataException as e: print("ErrataException Message: {}, retry it again".format(e)) if retried is not True: black_list = parse_exception_error_message(e) retry_list = [x for x in bug_list if x not in black_list] if len(retry_list) > 0: add_bugs_with_retry(advisory, retry_list, True) else: raise exceptions.ElliottFatalError(getattr(e, 'message', repr(e)))
def add_bugzilla_bugs_with_retry( advisory_id: int, bugids: List, noop: bool = False, batch_size: int = constants.BUG_ATTACH_CHUNK_SIZE): """ adding specified bugs into advisory, retry 2 times: first time parse the exception message to get failed bug id list, remove from original list then add bug to advisory again, if still has failures raise exceptions :param advisory_id: advisory id :param bugids: iterable of bugzilla bug ids to attach to advisory :param noop: do not modify anything :param batch_size: perform operation in batches of given size :return: """ logger.info( f'Request to attach {len(bugids)} bugs to the advisory {advisory_id}') try: advisory = Erratum(errata_id=advisory_id) except GSSError: exit_unauthenticated() if not advisory: raise exceptions.ElliottFatalError( f"Error: Could not locate advisory {advisory_id}") existing_bugs = bzutil.BugzillaBugTracker.advisory_bug_ids(advisory) new_bugs = set(bugids) - set(existing_bugs) logger.info( f'Bugs already attached: {len(existing_bugs)}. New bugs: {len(new_bugs)}' ) if not new_bugs: return for chunk_of_bugs in chunk(list(new_bugs), batch_size): if noop: logger.info('Dry run: Would have attached bugs') continue try: advisory.addBugs(chunk_of_bugs) advisory.commit() except ErrataException as e: logger.info(f"ErrataException Message: {e}\nRetrying...") block_list = parse_exception_error_message(e) retry_list = [x for x in chunk_of_bugs if x not in block_list] if len(retry_list) == 0: continue try: advisory = Erratum(errata_id=advisory_id) advisory.addBugs(retry_list) advisory.commit() except ErrataException as e: raise exceptions.ElliottFatalError( getattr(e, 'message', repr(e))) logger.info("remaining bugs attached") logger.info("All bugzilla bugs attached")
def add_bugs_with_retry(advisory, bugs, retried=False, noop=False): """ adding specified bugs into advisory, retry 2 times: first time parse the exception message to get failed bug id list, remove from original list then add bug to advisory again, if still has failures raise exceptions :param advisory: advisory id :param bugs: iterable of bzutil.bug to attach to advisory :param retried: retry 2 times, first attempt fetch failed bugs sift out then attach again :return: """ print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}') try: advs = Erratum(errata_id=advisory) except GSSError: exit_unauthenticated() if advs is False: raise exceptions.ElliottFatalError( "Error: Could not locate advisory {advs}".format(advs=advisory)) existing_bugs = advs.errata_bugs new_bugs = set(bug.id for bug in bugs) - set(existing_bugs) print(f'Bugs already attached: {len(existing_bugs)}') print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}') if noop: print('Dry run. Exiting.') return if not new_bugs: print('No new bugs to attach. Exiting.') return green_prefix("Adding {count} bugs to advisory {retry_times} times:".format( count=len(bugs), retry_times=1 if retried is False else 2)) try: advs.addBugs([bug.id for bug in bugs]) advs.commit() except ErrataException as e: print("ErrataException Message: {}, retry it again".format(e)) if retried is not True: block_list = parse_exception_error_message(e) retry_list = [x for x in bugs if x.id not in block_list] if len(retry_list) > 0: add_bugs_with_retry(advisory, retry_list, retried=True, noop=noop) else: raise exceptions.ElliottFatalError(getattr(e, 'message', repr(e)))
def get_advisory_nvrs(advisory): """ :return: dict, with keys as package names and values as strs in the form: '{version}-{release}' """ try: green_prefix("Fetching advisory builds: ") click.echo("Advisory - {}".format(advisory)) builds = get_builds(advisory) except GSSError: exit_unauthenticated() except exceptions.ErrataToolError as ex: raise exceptions.ElliottFatalError(getattr(ex, 'message', repr(ex))) all_advisory_nvrs = {} # Results come back with top level keys which are brew tags green_prefix("Looping over tags: ") click.echo("{} tags to check".format(len(builds))) for tag in builds.keys(): # Each top level has a key 'builds' which is a list of dicts green_prefix("Looping over builds in tag: ") click.echo("{} with {} builds".format(tag, len(builds[tag]['builds']))) for build in builds[tag]['builds']: # Each dict has a top level key which might be the actual # 'nvr' but I don't have enough data to know for sure # yet. Also I don't know when there might be more than one # key in the build dict. We'll loop over it to be sure. for name in build.keys(): n, v, r = name.rsplit('-', 2) version_release = "{}-{}".format(v, r) all_advisory_nvrs[n] = version_release return all_advisory_nvrs
def get_corresponding_flaw_bugs(bzapi, tracker_bugs, fields, strict=False): """ Get corresponding flaw bugs objects for the given tracker bug objects """ # fields needed for is_flaw_bug() if "product" not in fields: fields.append("product") if "component" not in fields: fields.append("component") blocking_bugs = bzapi.getbugs(unique( flatten([t.blocks for t in tracker_bugs])), include_fields=fields, permissive=False) flaw_bugs = [ flaw_bug for flaw_bug in blocking_bugs if bzutil.is_flaw_bug(flaw_bug) ] # Validate that each tracker has a corresponding flaw bug flaw_ids = set([b.id for b in flaw_bugs]) no_flaws = set() for tracker in tracker_bugs: if not set(tracker.blocks).intersection(flaw_ids): no_flaws.add(tracker.id) if no_flaws: msg = f'No flaw bugs could be found for these trackers: {no_flaws}' if strict: raise exceptions.ElliottFatalError(msg) else: logger.warn(msg) return flaw_bugs
def get_corresponding_flaw_bugs(self, tracker_bugs: List[Bug], flaw_bug_tracker=None, strict: bool = False, verbose: bool = False): """Get corresponding flaw bug objects for given list of tracker bug objects. Accepts a flaw_bug_tracker object to fetch flaw bugs from incase it's different from self :return: (tracker_flaws, flaw_id_bugs): tracker_flaws is a dict with tracker bug id as key and list of flaw bug id as value, flaw_id_bugs is a dict with flaw bug id as key and flaw bug object as value """ bug_tracker = flaw_bug_tracker if flaw_bug_tracker else self flaw_bugs = bug_tracker.get_flaw_bugs( list(set(sum([t.corresponding_flaw_bug_ids for t in tracker_bugs], []))), verbose=verbose ) flaw_id_bugs = {bug.id: bug for bug in flaw_bugs} # Validate that each tracker has a corresponding flaw bug flaw_ids = set(flaw_id_bugs.keys()) no_flaws = set() for tracker in tracker_bugs: if not set(tracker.corresponding_flaw_bug_ids).intersection(flaw_ids): no_flaws.add(tracker.id) if no_flaws: msg = f'No flaw bugs could be found for these trackers: {no_flaws}' if strict: raise exceptions.ElliottFatalError(msg) else: logger.warn(msg) tracker_flaws = { tracker.id: [b for b in tracker.corresponding_flaw_bug_ids if b in flaw_id_bugs] for tracker in tracker_bugs } return tracker_flaws, flaw_id_bugs
def add_jira_bugs_with_retry( advisory_id: int, bugids: List[str], noop: bool = False, batch_size: int = constants.BUG_ATTACH_CHUNK_SIZE): """ :param advisory_id: advisory id :param bugids: iterable of jira bug ids to attach to advisory :param noop: do not modify anything :param batch_size: perform operation in batches of given size """ logger.info( f'Request to attach {len(bugids)} bugs to the advisory {advisory_id}') try: advisory = Erratum(errata_id=advisory_id) except GSSError: exit_unauthenticated() if not advisory: raise exceptions.ElliottFatalError( f"Error: Could not locate advisory {advisory_id}") existing_bugs = bzutil.JIRABugTracker.advisory_bug_ids(advisory) new_bugs = set(bugids) - set(existing_bugs) logger.info( f'Bugs already attached: {len(existing_bugs)}. New bugs: {len(new_bugs)}' ) if not new_bugs: return for chunk_of_bugs in chunk(bugids, batch_size): if noop: logger.info('Dry run: Would have attached bugs') continue results = add_multi_jira_issues(advisory_id, chunk_of_bugs) for i, result in enumerate(results): if result.status_code != 201: rt = add_jira_issue(advisory_id, chunk_of_bugs[i]) if rt.status_code != 201: raise exceptions.ElliottFatalError( f"attach jira bug {chunk_of_bugs[i]} failed with " f"status={rt.status_code}: {rt.json()}") logger.info("All jira bugs attached")
def get_all_advisory_nvrs(advisory): """ :return: list of tuples (name, version, release) """ try: builds = get_builds(advisory) except GSSError: exit_unauthenticated() except exceptions.ErrataToolError as ex: raise exceptions.ElliottFatalError(getattr(ex, 'message', repr(ex))) all_advisory_nvrs = [] # Results come back with top level keys which are brew tags for tag in builds.keys(): # Each top level has a key 'builds' which is a list of dicts for build in builds[tag]['builds']: for name in build.keys(): n, v, r = name.rsplit('-', 2) all_advisory_nvrs.append((n, v, r)) return all_advisory_nvrs
def tag_builds_cli(runtime: Runtime, advisories: Tuple[int], default_advisory_type: str, product_version: str, builds: Tuple[str], tag: str, dont_untag: bool, dry_run: bool): """ Tag builds into Brew tag and optionally untag unspecified builds. Example 1: Tag RHEL7 RPMs that on ocp-build-data recorded advisory into rhaos-4.3-rhel-7-image-build $ elliott --group=openshift-4.3 tag-builds --use-default-advisory rpm --product-version RHEL-7-OSE-4.3 --tag rhaos-4.3-rhel-7-image-build Example 2: Tag RHEL8 RPMs that are on advisory 55016 into rhaos-4.3-rhel-8-image-build $ elliott --group=openshift-4.3 tag-builds --advisory 55016 --product-version OSE-4.4-RHEL-8 --tag rhaos-4.3-rhel-8-image-build Example 3: Tag specified builds into rhaos-4.3-rhel-8-image-build $ elliott --group=openshift-4.3 tag-builds --build buildah-1.11.6-6.rhaos4.3.el8 --build openshift-4.3.23-202005230952.g1.b596217.el8 --tag rhaos-4.3-rhel-8-image-build """ if advisories and builds: raise click.BadParameter('Use only one of --build or --advisory/-a.') if advisories and default_advisory_type: raise click.BadParameter( 'Use only one of --use-default-advisory or --advisory/-a.') if default_advisory_type and builds: raise click.BadParameter( 'Use only one of --build or --use-default-advisory.') if product_version and not advisories and not default_advisory_type: raise click.BadParameter( '--product-version should only be used with --use-default-advisory or --advisory/-a.' ) runtime.initialize() logger = runtime.logger if default_advisory_type: advisories = (find_default_advisory(runtime, default_advisory_type), ) all_builds = set() # All Brew builds that should be in the tag if advisories: errata_session = requests.session() for advisory in advisories: logger.info( f"Fetching attached Brew builds from advisory {advisory}...") errata_builds = errata.get_builds(advisory, errata_session) product_versions = list(errata_builds.keys()) logger.debug( f"Advisory {advisory} has builds for {len(product_versions)} product versions: {product_versions}" ) if product_version: # Only this product version should be concerned product_versions = [product_version] for pv in product_versions: logger.debug(f"Extract Errata builds for product version {pv}") nvrs = _extract_nvrs_from_errata_build_list(errata_builds, pv) logger.info( f"Found {len(nvrs)} builds from advisory {advisory} with product version {pv}" ) logger.debug( f"The following builds are found for product version {pv}:\n\t{list(nvrs)}" ) all_builds |= set(nvrs) brew_session = koji.ClientSession(runtime.group_config.urls.brewhub or constants.BREW_HUB) if builds: # NVRs are directly specified with --build build_objs = brew.get_build_objects(list(builds), brew_session) all_builds = {build["nvr"] for build in build_objs} click.echo( f"The following {len(all_builds)} build(s) should be in tag {tag}:") for nvr in all_builds: green_print(f"\t{nvr}") # get NVRs that have been tagged tagged_build_objs = brew_session.listTagged(tag, latest=False, inherit=False) tagged_builds = {build["nvr"] for build in tagged_build_objs} # get NVRs that should be tagged missing_builds = all_builds - tagged_builds click.echo(f"{len(missing_builds)} build(s) need to be tagged into {tag}:") for nvr in missing_builds: green_print(f"\t{nvr}") # get NVRs that should be untagged extra_builds = tagged_builds - all_builds click.echo(f"{len(extra_builds)} build(s) need to be untagged from {tag}:") for nvr in extra_builds: green_print(f"\t{nvr}") if dry_run: yellow_print("Dry run: Do nothing.") return brew_session.gssapi_login() if not dont_untag: # untag extra builds extra_builds = list(extra_builds) logger.info(f"Untagging {len(extra_builds)} build(s) from {tag}...") multicall_tasks = brew.untag_builds(tag, extra_builds, brew_session) failed_to_untag = [] for index, task in enumerate(multicall_tasks): try: task.result click.echo(f"{nvr} has been successfully untagged from {tag}") except Exception as ex: nvr = extra_builds[index] failed_to_untag.append(nvr) logger.error(f"Failed to untag {nvr}: {ex}") # tag missing builds missing_builds = list(missing_builds) task_id_nvr_map = {} logger.info(f"Tagging {len(missing_builds)} build(s) into {tag}...") multicall_tasks = brew.tag_builds(tag, missing_builds, brew_session) failed_to_tag = [] for index, task in enumerate(multicall_tasks): nvr = missing_builds[index] try: task_id = task.result task_id_nvr_map[task_id] = nvr except Exception as ex: failed_to_tag.append(nvr) logger.error(f"Failed to tag {nvr}: {ex}") if task_id_nvr_map: # wait for tag task to finish logger.info("Waiting for tag tasks to finish") brew.wait_tasks(task_id_nvr_map.keys(), brew_session, logger=logger) # get tagging results stopped_tasks = list(task_id_nvr_map.keys()) with brew_session.multicall(strict=False) as m: multicall_tasks = [] for task_id in stopped_tasks: multicall_tasks.append( m.getTaskResult(task_id, raise_fault=False)) for index, t in enumerate(multicall_tasks): task_id = stopped_tasks[index] nvr = task_id_nvr_map[task_id] tag_res = t.result logger.debug( f"Tagging task {task_id} {nvr} returned result {tag_res}") click.echo(f"{nvr} has been successfully tagged into {tag}") if tag_res and 'faultCode' in tag_res: if "already tagged" not in tag_res["faultString"]: failed_to_tag.append(nvr) logger.error( f'Failed to tag {nvr} into {tag}: {tag_res["faultString"]}' ) if failed_to_untag: red_print("The following builds were failed to untag:") for nvr in failed_to_untag: red_print(f"\t{nvr}") elif not dont_untag: green_print( f"All unspecified builds have been successfully untagged from {tag}." ) if failed_to_tag: red_print("The following builds were failed to tag:") for nvr in failed_to_tag: red_print(f"\t{nvr}") else: green_print(f"All builds have been successfully tagged into {tag}.") if failed_to_untag or failed_to_tag: raise exceptions.ElliottFatalError( "Not all builds were successfully tagged/untagged.")
def add_bugs_with_retry(advisory, bugs, noop=False, batch_size=100): """ adding specified bugs into advisory, retry 2 times: first time parse the exception message to get failed bug id list, remove from original list then add bug to advisory again, if still has failures raise exceptions :param advisory: advisory id :param bugs: iterable of bzutil.bug to attach to advisory :return: """ print(f'Request to attach {len(bugs)} bugs to the advisory {advisory}') try: advs = Erratum(errata_id=advisory) except GSSError: exit_unauthenticated() if advs is False: raise exceptions.ElliottFatalError( "Error: Could not locate advisory {advs}".format(advs=advisory)) existing_bugs = advs.errata_bugs new_bugs = set(bug.id for bug in bugs) - set(existing_bugs) print(f'Bugs already attached: {len(existing_bugs)}') print(f'New bugs ({len(new_bugs)}) : {sorted(new_bugs)}') if not new_bugs: print('No new bugs to attach. Exiting.') return bugs = list(new_bugs) batches = list(range(0, len(bugs), batch_size)) if len(bugs) % batch_size != 0: batches.append(len(bugs)) green_prefix( f"Adding bugs in batches of {batch_size}. Number of batches: {len(batches)-1}\n" ) for i in range(len(batches) - 1): start, end = batches[i], batches[i + 1] print(f"Attaching Batch {i+1}") if noop: print('Dry run: Would have attached bugs') continue try: advs.addBugs(bugs[start:end]) advs.commit() except ErrataException as e: print("ErrataException Message: {}, retry it again".format(e)) block_list = parse_exception_error_message(e) retry_list = [x for x in bugs[start:end] if x not in block_list] if len(retry_list) == 0: continue try: advs = Erratum(errata_id=advisory) advs.addBugs(retry_list) advs.commit() except ErrataException as e: raise exceptions.ElliottFatalError( getattr(e, 'message', repr(e))) print("remaining bugs attached")