def get_updates(self, number_of_updates: int = 3) -> List: """ Get specific number of latest updates in bodhi :param number_of_updates: int :return: None """ # https://github.com/fedora-infra/bodhi/issues/3058 from bodhi.client.bindings import BodhiClient b = BodhiClient() results = b.query( packages=self.dg.package_config.downstream_package_name)["updates"] logger.debug("Bodhi updates fetched.") stable_branches: Set[str] = set() all_updates = [[ result["title"], result["karma"], result["status"], result["release"]["branch"], ] for result in results] updates = [] for [update, karma, status, branch] in all_updates: # Don't return more than one stable update per branch if branch not in stable_branches or status != "stable": updates.append([update, karma, status]) if status == "stable": stable_branches.add(branch) if len(updates) == number_of_updates: break return updates
def __init__(self, dry_run=False): self._client = BodhiClient() self._client.init_username() self._dry_run = dry_run self._log_prefix = '' if dry_run: self._log_prefix = 'dry run: '
def get_aliases() -> Dict[str, List[str]]: """ Function to automatically determine fedora-all, fedora-stable, fedora-development and epel-all aliases. Current data are fetched via bodhi client, with default base url `https://bodhi.fedoraproject.org/'. :return: dictionary containing aliases """ bodhi_client = BodhiClient() releases = bodhi_client.get_releases(exclude_archived=True) aliases = defaultdict(list) for release in releases.releases: if release.id_prefix == "FEDORA" and release.name != "ELN": name = release.long_name.lower().replace(" ", "-") aliases["fedora-all"].append(name) if release.state == "current": aliases["fedora-stable"].append(name) elif release.state == "pending": aliases["fedora-development"].append(name) elif release.id_prefix == "FEDORA-EPEL": name = release.name.lower() aliases["epel-all"].append(name) aliases["fedora-all"].append("fedora-rawhide") aliases["fedora-development"].append("fedora-rawhide") return aliases
def get_aliases() -> Dict[str, List[str]]: """ Function to automatically determine fedora-all, fedora-stable, fedora-development and epel-all aliases. Current data are fetched via bodhi client, with default base url `https://bodhi.fedoraproject.org/'. :return: dictionary containing aliases """ bodhi_client = BodhiClient() releases = bodhi_client.get_releases(exclude_archived=True) aliases = defaultdict(list) for release in releases.releases: if release.id_prefix == "FEDORA" and release.name != "ELN": name = release.long_name.lower().replace(" ", "-") if release.state == "current": aliases["fedora-stable"].append(name) elif release.state == "pending": aliases["fedora-development"].append(name) elif release.id_prefix == "FEDORA-EPEL": name = release.name.lower() aliases["epel-all"].append(name) if aliases.get("fedora-development"): aliases["fedora-development"].sort(key=lambda x: int(x.rsplit("-")[-1])) # The Fedora with the highest version is "rawhide", but # Bodhi always uses release names, and has no concept of "rawhide". aliases["fedora-development"][-1] = "fedora-rawhide" aliases["fedora-all"] = aliases["fedora-development"] + aliases["fedora-stable"] return aliases
def get_builds(self, number_of_builds: int = 3) -> None: """ Get specific number of latest builds from koji :param number_of_builds: int :return: None """ logger.info("\nLatest builds:") # https://github.com/fedora-infra/bodhi/issues/3058 from bodhi.client.bindings import BodhiClient b = BodhiClient() builds_d = b.latest_builds(self.dg.package_name) branches = self.dg.local_project.git_project.get_branches() branches.remove("master") # there is no master tag in koji for branch in branches: koji_tag = f"{branch}-updates-candidate" try: koji_builds = [builds_d[koji_tag]] # take last three builds koji_builds = (koji_builds[:number_of_builds] if len(koji_builds) > number_of_builds else koji_builds) koji_builds_str = "\n".join(f" - {b}" for b in koji_builds) logger.info(f"{branch}:\n{koji_builds_str}") except KeyError: logger.info(f"{branch}: No builds.")
def _get_bodhi_history(username): """ Print the last action performed on bodhi by the given FAS user. :arg username, the fas username whose action is searched. """ from bodhi.client.bindings import BodhiClient bodhiclient = BodhiClient("https://bodhi.fedoraproject.org/") log.debug('Querying Bodhi for user: {0}'.format(username)) json_obj = bodhiclient.send_request( "updates/?user=%s" % username, verb='GET') def dategetter(field): def getter(item): return datetime.datetime.strptime(item[field], "%Y-%m-%d %H:%M:%S") return getter print('Last package update on bodhi:') if json_obj['updates']: latest = sorted(json_obj['updates'], key=dategetter("date_submitted") )[-1] print(' {0} on package {1}'.format( latest["date_submitted"], latest["title"])) else: print(' No activity found on bodhi')
def create_bodhi_update( self, dist_git_branch: str, update_type: str, update_notes: str, koji_builds: Sequence[str] = None, ): logger.debug( f"About to create a Bodhi update of type {update_type} from {dist_git_branch}" ) # https://github.com/fedora-infra/bodhi/issues/3058 from bodhi.client.bindings import BodhiClient, BodhiClientException if not self.package_name: raise PackitException("Package name is not set.") # bodhi will likely prompt for username and password if kerb ticket is not up b = BodhiClient() if not koji_builds: # alternatively we can call something like `koji latest-build rawhide sen` builds_d = b.latest_builds(self.package_name) builds_str = "\n".join(f" - {b}" for b in builds_d) logger.debug( f"Koji builds for package {self.package_name}: \n{builds_str}") koji_tag = f"{dist_git_branch}-updates-candidate" try: koji_builds = [builds_d[koji_tag]] koji_builds_str = "\n".join(f" - {b}" for b in koji_builds) logger.info( f"Koji builds for package {self.package_name} and koji tag {koji_tag}:" f"\n{koji_builds_str}") except KeyError: raise PackitException( f"There is no build for {self.package_name} in koji tag {koji_tag}" ) # I was thinking of verifying that the build is valid for a new bodhi update # but in the end it's likely a waste of resources since bodhi will tell us rendered_note = update_notes.format( version=self.specfile.get_version()) try: result = b.save(builds=koji_builds, notes=rendered_note, type=update_type) logger.debug(f"Bodhi response:\n{result}") logger.info(f"Bodhi update {result['alias']}:\n" f"- {result['url']}\n" f"- stable_karma: {result['stable_karma']}\n" f"- unstable_karma: {result['unstable_karma']}\n" f"- notes:\n{result['notes']}\n") if "caveats" in result: for cav in result["caveats"]: logger.info(f"- {cav['name']}: {cav['description']}\n") except BodhiClientException as ex: logger.error(ex) raise PackitException( f"There is a problem with creating the bodhi update:\n{ex}") return result["alias"]
def get_testing_updates(self, update_alias: Optional[str]) -> List: from bodhi.client.bindings import BodhiClient b = BodhiClient() results = b.query( alias=update_alias, packages=self.dg.package_config.downstream_package_name, status="testing", )["updates"] logger.debug("Bodhi updates with status 'testing' fetched.") return results
def test_push_updates(cwd_upstream_or_distgit, api_instance, query_response, request_response): from bodhi.client.bindings import BodhiClient u, d, api = api_instance flexmock(BodhiClient) BodhiClient.should_receive("query").and_return(query_response).once() BodhiClient.should_receive("request").with_args( update="FEDORA-2019-89c99f680c", request="stable").and_return(request_response).once() api.push_updates()
def push_bodhi_update(update_alias: str): from bodhi.client.bindings import BodhiClient, UpdateNotFound b = BodhiClient() try: response = b.request(update=update_alias, request="stable") logger.debug(f"Bodhi response:\n{response}") logger.info(f"Bodhi update {response['alias']} pushed to stable:\n" f"- {response['url']}\n" f"- stable_karma: {response['stable_karma']}\n" f"- unstable_karma: {response['unstable_karma']}\n" f"- notes:\n{response['notes']}\n") except UpdateNotFound: logger.error("Update was not found.")
def __init__(self, environ, request): super(BodhiConnector, self).__init__(environ, request) self._prod_url = config.get( 'fedoracommunity.connector.bodhi.produrl', 'https://bodhi.fedoraproject.org') self._bodhi_client = BodhiClient(self._base_url, insecure=self._insecure)
def get_updates(self, number_of_updates: int = 3) -> List: """ Get specific number of latest updates in bodhi :param number_of_updates: int :return: None """ # https://github.com/fedora-infra/bodhi/issues/3058 from bodhi.client.bindings import BodhiClient b = BodhiClient() results = b.query(packages=self.dg.package_name)["updates"] if len(results) > number_of_updates: results = results[:number_of_updates] return [ [result["title"], result["karma"], result["status"]] for result in results ]
def get_updates(self, number_of_updates: int = 3) -> None: """ Get specific number of latest updates in bodhi :param number_of_updates: int :return: None """ logger.info("\nLatest bodhi updates:") # https://github.com/fedora-infra/bodhi/issues/3058 from bodhi.client.bindings import BodhiClient b = BodhiClient() results = b.query(packages=self.dg.package_name)["updates"] if len(results) > number_of_updates: results = results[:number_of_updates] table = [[result["title"], result["karma"], result["status"]] for result in results] logger.info(tabulate(table, headers=["Update", "Karma", "status"]))
def search(self, query): """ Perform Bodhi query """ result = [] current_page = 1 original_query = query while current_page: log.debug("Bodhi query: {0}".format(query)) client = BodhiClient(self.url) data = client.send_request(query, verb='GET') objects = data['updates'] log.debug("Result: {0} fetched".format(listed( len(objects), "item"))) log.data(pretty(data)) result.extend(objects) if current_page < data['pages']: current_page = current_page + 1 query = f"{original_query}&page={current_page}" else: current_page = None return result
class UpdatePromoter(object): def __init__(self, dry_run=False): self._client = BodhiClient() self._client.init_username() self._dry_run = dry_run self._log_prefix = '' if dry_run: self._log_prefix = 'dry run: ' def promote_update(self, update, status='stable'): print('{}{} - requesting {}'.format(self._log_prefix, update.title, status)) request_params = { 'update': update.alias, 'request': status, } if not self._dry_run: self._client.request(**request_params) def promote_updates(self, updates, status='stable'): for update in updates: if status in ['stable', 'batched' ] and not update.meets_testing_requirements: print('{}skipping {} - not eligible for {}'.format( self._log_prefix, update.title, status)) continue if update.request == status: continue self.promote_update(update, status) def get_updates(self, release, package=None, status='testing'): query_params = { 'mine': True, 'releases': release, 'rows_per_page': 100, 'status': status, } if package: query_params['packages'] = package return self._client.query(**query_params).updates
def test_push_updates(upstream_n_distgit, query_response, request_response): from bodhi.client.bindings import BodhiClient u, d = upstream_n_distgit with cwd(u): c = get_test_config() pc = get_local_package_config(str(u)) pc.upstream_project_url = str(u) pc.dist_git_clone_path = str(d) up_lp = LocalProject(working_dir=u) api = PackitAPI(c, pc, up_lp) flexmock(BodhiClient) BodhiClient.should_receive("query").and_return(query_response).once() BodhiClient.should_receive("request").with_args( update="FEDORA-2019-89c99f680c", request="stable" ).and_return(request_response).once() api.push_updates()
def get_builds(self, number_of_builds: int = 3) -> Dict: """ Get specific number of latest builds from koji :param number_of_builds: int :return: None """ # https://github.com/fedora-infra/bodhi/issues/3058 from bodhi.client.bindings import BodhiClient b = BodhiClient() builds_d = b.latest_builds(self.dg.package_name) branches = self.dg.local_project.git_project.get_branches() branches.remove("master") # there is no master tag in koji builds: Dict = {} for branch in branches: koji_tag = f"{branch}-updates-candidate" try: # take last three builds builds[branch] = builds_d[koji_tag][:number_of_builds] except KeyError: logger.info(f"There are no builds for branch {branch}") return builds
def get_builds(self, ) -> Dict: """ Get latest koji builds """ # https://github.com/fedora-infra/bodhi/issues/3058 from bodhi.client.bindings import BodhiClient b = BodhiClient() # { koji-target: "latest-build-nvr"} builds_d = b.latest_builds( self.dg.package_config.downstream_package_name) branches = self.dg.local_project.git_project.get_branches() logger.debug("Latest koji builds fetched.") builds: Dict = {} for branch in branches: # there is no master tag in koji if branch == "master": continue koji_tag = f"{branch}-updates-candidate" try: builds[branch] = builds_d[koji_tag] except KeyError: logger.info(f"There are no builds for branch {branch}") return builds
def main(): module_args = dict( name=dict(type="str", required=True), long_name=dict(type="str", required=True), id_prefix=dict(type="str", required=True), version=dict(type="str", required=True), branch=dict(type="str", required=True), dist_tag=dict(type="str", required=True), stable_tag=dict(type="str", required=True), testing_tag=dict(type="str", required=True), candidate_tag=dict(type="str", required=True), pending_stable_tag=dict(type="str", required=True), pending_testing_tag=dict(type="str", required=True), pending_signing_tag=dict(type="str", required=False), override_tag=dict(type="str", required=True), state=dict(choice=["disabled", "pending", "current", "archived"], default="pending"), user=dict(type="str", required=True), mail_template=dict(type="str", required=False), composed_by_bodhi=dict(type="str", required=False), url=dict(type="str", required=False, default="https://bodhi.fedoraproject.org"), ) module = AnsibleModule(argument_spec=module_args, supports_check_mode=False) try: from bodhi.client.bindings import BodhiClient except ImportError: module.fail_json( msg="the bodhi python module not found on the target system") # Connect to bodhi client = BodhiClient( base_url=module.params["url"], username=module.params["user"], ) result = ensure_release(client, module, **module.params) module.exit_json(**result)
#!/usr/bin/python3 """ Generate %patch, %source and %prep data about most patched Fedora spec files. """ import json import pathlib import re from typing import Dict, Tuple from bodhi.client.bindings import BodhiClient # rpm-specs-latest.tar.xz unpacked in /fedora-spec-files/rpm-specs/ RPM_SPECS = pathlib.Path("./rpm-specs") # process this many packages ordered by the amount of patches PROCESS_PACKAGE_COUNT = 50 bodhi = BodhiClient() def get_patch_stats() -> Dict[str, int]: """provide an ordered dict with package names as keys and number of patches as values""" patch_re = re.compile(r"\nPatch\d*:") result: Dict[str, int] = {} for spec in RPM_SPECS.iterdir(): spec_content = spec.read_text() match = patch_re.findall(spec_content) if match: result[spec.name[:-5]] = len(match) return dict(sorted(result.items(), key=lambda item: item[1], reverse=True)) def get_update_frequency(package_name: str, distro: str = "f34") -> int: """provide number of builds a package has in koji in a given distro""" return len(bodhi.koji_session.listTagged(distro, package=package_name))
class BodhiConnector(IConnector, ICall, IQuery): _method_paths = dict() _query_paths = dict() _cache_prompts = dict() def __init__(self, environ, request): super(BodhiConnector, self).__init__(environ, request) self._prod_url = config.get( 'fedoracommunity.connector.bodhi.produrl', 'https://bodhi.fedoraproject.org') self._bodhi_client = BodhiClient(self._base_url, insecure=self._insecure) @classmethod def query_updates_cache_prompt(cls, msg): if '.bodhi.' not in msg['topic']: return msg = msg['msg'] if 'update' in msg: update = msg['update'] release = update['release']['name'] status = update['status'] nvrs = [build['nvr'] for build in update['builds']] names = ['-'.join(nvr.split('-')[:-2]) for nvr in nvrs] releases = [release, ''] statuses = [status, ''] groupings = [False] headers = ['package', 'release', 'status', 'group_updates'] combinations = product(names, releases, statuses, groupings) for values in combinations: yield dict(zip(headers, values)) @classmethod def query_active_releases_cache_prompt(cls, msg): if '.bodhi.' not in msg['topic']: return msg = msg['msg'] if 'update' in msg: nvrs = [build['nvr'] for build in msg['update']['builds']] names = ['-'.join(nvr.split('-')[:-2]) for nvr in nvrs] for name in names: yield {'package': name} # IConnector @classmethod def register(cls): cls._base_url = config.get('fedoracommunity.connector.bodhi.baseurl', 'https://bodhi.fedoraproject.org/') check_certs = asbool(config.get('fedora.clients.check_certs', True)) cls._insecure = not check_certs cls.register_query_updates() cls.register_query_active_releases() def request_data(self, path, params): return self._bodhi_client.send_request(path, auth=False, params=params) def introspect(self): # FIXME: return introspection data return None #ICall def call(self, resource_path, params): log.debug('BodhiConnector.call(%s)' % locals()) # proxy client only returns structured data so we can pass # this off to request_data but we should fix that in ProxyClient return self.request_data(resource_path, params) #IQuery @classmethod def register_query_updates(cls): path = cls.register_query( 'query_updates', cls.query_updates, cls.query_updates_cache_prompt, primary_key_col='request_id', default_sort_col='request_id', default_sort_order=-1, can_paginate=True) path.register_column('request_id', default_visible=False, can_sort=False, can_filter_wildcards=False) path.register_column('updateid', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('nvr', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('submitter', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('status', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('request', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('karma', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('nagged', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('type', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('approved', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('date_submitted', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('date_pushed', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('date_modified', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('comments', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('bugs', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('builds', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('releases', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('release', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('karma_level', default_visible=True, can_sort=False, can_filter_wildcards=False) f = ParamFilter() f.add_filter('package', ['nvr'], allow_none=False) f.add_filter('status', ['status'], allow_none=True) f.add_filter('group_updates', allow_none=True, cast=bool) f.add_filter('granularity', allow_none=True) f.add_filter('release', allow_none=False) cls._query_updates_filter = f def query_updates(self, start_row=None, rows_per_page=None, order=-1, sort_col=None, filters=None, **params): if not filters: filters = {} filters = self._query_updates_filter.filter(filters, conn=self) group_updates = filters.get('group_updates', True) params.update(filters) params['page'] = int(start_row/rows_per_page) + 1 # If we're grouping updates, ask for twice as much. This is so we can # handle the case where there are two updates for each package, one for # each release. Yes, worst case we get twice as much data as we ask # for, but this allows us to do *much* more efficient database calls on # the server. if group_updates: params['rows_per_page'] = rows_per_page * 2 else: params['rows_per_page'] = rows_per_page # Convert bodhi1 query format to bodhi2. if 'package' in params: params['packages'] = params.pop('package') if 'release' in params: params['releases'] = params.pop('release') results = self._bodhi_client.send_request('updates', auth=False, params=params) total_count = results['total'] if group_updates: updates_list = self._group_updates(results['updates'], num_packages=rows_per_page) else: updates_list = results['updates'] for up in updates_list: versions = [] releases = [] if group_updates: up['title'] = up['dist_updates'][0]['title'] for dist_update in up['dist_updates']: versions.append(dist_update['version']) releases.append(dist_update['release_name']) up['name'] = up['package_name'] up['versions'] = versions up['releases'] = releases up['status'] = up['dist_updates'][0]['status'] up['nvr'] = up['dist_updates'][0]['title'] up['request_id'] = up['package_name'] + \ dist_update['version'].replace('.', '') else: chunks = up['title'].split('-') up['name'] = '-'.join(chunks[:-2]) up['version'] = '-'.join(chunks[-2:]) up['versions'] = chunks[-2] up['releases'] = up['release']['long_name'] up['nvr'] = up['title'] up['request_id'] = up.get('updateid') or \ up['nvr'].replace('.', '').replace(',', '') up['id'] = up['nvr'].split(',')[0] # A unique id that we can use in HTML class fields. #up['request_id'] = up.get('updateid') or \ # up['nvr'].replace('.', '').replace(',', '') actions = [] up['actions'] = '' for action in actions: reqs = '' if group_updates: for u in up['dist_updates']: reqs += "update_action('%s', '%s');" % (u['title'], action[0]) title = up['dist_updates'][0]['title'] else: reqs += "update_action('%s', '%s');" % (up['title'], action[0]) title = up['title'] # FIXME: Don't embed HTML up['actions'] += """ <button id="%s_%s" onclick="%s return false;">%s</button><br/> """ % (title.replace('.', ''), action[0], reqs, action[1]) # Dates if group_updates: date_submitted = up['dist_updates'][0]['date_submitted'] date_pushed = up['dist_updates'][0]['date_pushed'] else: date_submitted = up['date_submitted'] date_pushed = up['date_pushed'] granularity = filters.get('granularity', 'day') ds = DateTimeDisplay(date_submitted) up['date_submitted_display'] = ds.age(granularity=granularity, general=True) + ' ago' if date_pushed: dp = DateTimeDisplay(date_pushed) up['date_pushed'] = dp.datetime.strftime('%d %b %Y') up['date_pushed_display'] = dp.age(granularity=granularity, general=True) + ' ago' # karma # FIXME: take into account karma from both updates if group_updates: k = up['dist_updates'][0]['karma'] else: k = up['karma'] if k: up['karma_str'] = "%+d" % k else: up['karma_str'] = " %d" % k up['karma_level'] = 'meh' if k > 0: up['karma_level'] = 'good' if k < 0: up['karma_level'] = 'bad' up['details'] = self._get_update_details(up) return (total_count, updates_list) def _get_update_details(self, update): details = '' if update['status'] == 'stable': if update.get('updateid'): details += HTML.tag('a', c=update['updateid'], href='%s/updates/%s' % ( self._prod_url, update['alias'])) if update.get('date_pushed'): details += HTML.tag('br') + update['date_pushed'] else: details += 'In process...' elif update['status'] == 'pending' and update.get('request'): details += 'Pending push to %s' % update['request'] details += HTML.tag('br') details += HTML.tag('a', c="View update details >", href="%s/updates/%s" % (self._prod_url, update['alias'])) elif update['status'] == 'obsolete': for comment in update['comments']: if comment['user']['name'] == 'bodhi': if comment['text'].startswith('This update has been ' 'obsoleted by '): details += markdown.markdown( comment['text'], safe_mode="replace") return details def _get_update_actions(self, update): actions = [] if update['request']: actions.append(('revoke', 'Cancel push')) else: if update['status'] == 'testing': actions.append(('unpush', 'Unpush')) actions.append(('stable', 'Push to stable')) if update['status'] == 'pending': actions.append(('testing', 'Push to testing')) actions.append(('stable', 'Push to stable')) return actions def _group_updates(self, updates, num_packages=None): """ Group a list of updates by release. This method allows allows you to limit the number of packages, for when we want to display 1 package per row, regardless of how many updates there are for it. """ packages = {} done = False i = 0 if not updates: return [] for update in updates: for build in update['builds']: pkg = build['nvr'].rsplit('-', 2)[0] if pkg not in packages: if num_packages and i >= num_packages: done = True break packages[pkg] = { 'package_name': pkg, 'dist_updates': list() } i += 1 else: skip = False for up in packages[pkg]['dist_updates']: if up['release_name'] == \ update['release']['long_name']: skip = True break if skip: break packages[pkg]['dist_updates'].append({ 'release_name': update['release']['long_name'], 'version': '-'.join(build['nvr'].split('-')[-2:]) }) packages[pkg]['dist_updates'][-1].update(update) if done: break result = [packages[p] for p in packages] sort_col = 'date_submitted' if result[0]['dist_updates'][0]['status'] == 'stable': sort_col = 'date_pushed' result = sorted(result, reverse=True, cmp=lambda x, y: cmp( x['dist_updates'][0][sort_col], y['dist_updates'][0][sort_col]) ) return result def add_updates_to_builds(self, builds): """Update a list of koji builds with the corresponding bodhi updates. This method makes a single query to bodhi, asking if it knows about any updates for a given list of koji builds. For builds with existing updates, the `update` will be added to it's dictionary. Currently it also adds `update_details`, which is HTML for rendering the builds update options. Ideally, this should be done client-side in the template (builds/templates/table_widget.mak). """ start = datetime.now() updates = self.call('get_updates_from_builds', { 'builds': ' '.join([b['nvr'] for b in builds])}) if updates: # FIXME: Lets stop changing the upstream APIs by putting the # session id as the first element, and the results in the second. updates = updates[1] for build in builds: if build['nvr'] in updates: build['update'] = updates[build['nvr']] status = build['update']['status'] details = '' # FIXME: ideally, we should just return the update JSON and do # this logic client-side in the template when the grid data # comes in. if status == 'stable': details = 'Pushed to updates' elif status == 'testing': details = 'Pushed to updates-testing' elif status == 'pending': details = 'Pending push to %s' % build['update']['request'] details += HTML.tag('br') details += HTML.tag('a', c="View update details >", href="%s/updates/%s" % (self._prod_url, build['update']['alias'])) else: details = HTML.tag('a', c='Push to updates >', href='%s/new?builds.text=%s' % ( self._prod_url, build['nvr'])) build['update_details'] = details log.debug( "Queried bodhi for builds in: %s" % (datetime.now() - start)) @classmethod def register_query_active_releases(cls): path = cls.register_query('query_active_releases', cls.query_active_releases, cls.query_active_releases_cache_prompt, primary_key_col='release', default_sort_col='release', default_sort_order=-1, can_paginate=True) path.register_column('release', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('stable_version', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('testing_version', default_visible=True, can_sort=False, can_filter_wildcards=False) f = ParamFilter() f.add_filter('package', ['nvr'], allow_none=False) cls._query_active_releases = f def get_all_releases(self): releases_obj = self.call('releases', {}) releases_all = None for i in range(1, releases_obj['pages'] + 1): if releases_all == None: releases_all = releases_obj['releases'] continue temp = self.call('releases?page=' + str(i), {})['releases'] releases_all.extend(temp) if releases_all == None: raise TypeError return releases_all def query_active_releases(self, filters=None, **params): releases = list() queries = list() # Mapping of tag -> release release_tag = dict() # List of testing builds to query bodhi for testing_builds = list() # nvr -> release lookup table testing_builds_row = dict() if not filters: filters = dict() filters = self._query_updates_filter.filter(filters, conn=self) package = filters.get('package') koji = get_connector('koji')._koji_client koji.multicall = True releases_all = self.get_all_releases() releases_all.append({'dist_tag': 'rawhide', 'long_name':'Rawhide', 'stable_tag': 'rawhide', 'testing_tag': 'no_testing_tag_found', 'state': 'current'}) releases_all = sorted(releases_all, key=lambda k: k['dist_tag'], reverse=True) for release in releases_all: if release['state'] not in ['current', 'pending']\ or 'Modular' in release['long_name']: continue tag = release['dist_tag'] name = release['long_name'] r = {'release': name, 'stable_version': 'None', 'testing_version': 'None'} if tag == 'rawhide': koji.listTagged( tag, package=package, latest=True, inherit=True) queries.append(tag) release_tag[tag] = r else: stable_tag = release['stable_tag'] testing_tag = release['testing_tag'] koji.listTagged(stable_tag, package=package, latest=True, inherit=True) queries.append(stable_tag) release_tag[stable_tag] = r koji.listTagged(testing_tag, package=package, latest=True) queries.append(testing_tag) release_tag[testing_tag] = r releases.append(r) results = koji.multiCall() for i, result in enumerate(results): if isinstance(result, dict): if 'faultString' in result: log.error("FAULT: %s" % result['faultString']) else: log.error("Can't find fault string in result: %s" % result) else: query = queries[i] row = release_tag[query] release = result[0] if query == 'dist-rawhide': if release: nvr = parse_build(release[0]['nvr']) row['stable_version'] = HTML.tag( 'a', c='%(version)s-%(release)s' % nvr, href=koji_build_url % nvr) else: row['stable_version'] = \ 'No builds tagged with %s' % tag row['testing_version'] = HTML.tag('i', c='Not Applicable') continue if release: release = release[0] if query.endswith('-testing'): nvr = parse_build(release['nvr']) row['testing_version'] = HTML.tag( 'a', c='%(version)s-%(release)s' % nvr, href=koji_build_url % nvr) testing_builds.append(release['nvr']) testing_builds_row[release['nvr']] = row else: # stable nvr = parse_build(release['nvr']) row['stable_version'] = HTML.tag( 'a', c='%(version)s-%(release)s' % nvr, href=koji_build_url % nvr) if release['tag_name'].endswith('-updates'): row['stable_version'] += ' (' + HTML.tag( 'a', c='update', href='%s/updates/?builds=%s' % ( self._prod_url, nvr['nvr'] ) ) + ')' # If there are updates in testing, then query bodhi with a single call if testing_builds: data = self.call('updates', { 'builds': ' '.join(testing_builds) }) updates = data['updates'] for up in updates: for build in up['builds']: if build['nvr'] in testing_builds: break else: continue build = build['nvr'] if up.karma > 1: up.karma_icon = 'good' elif up.karma < 0: up.karma_icon = 'bad' else: up.karma_icon = 'meh' karma_ico_16 = '/images/16_karma-%s.png' % up.karma_icon karma_icon_url = \ self._request.environ.get('SCRIPT_NAME', '') + \ karma_ico_16 karma = 'karma_%s' % up.karma_icon row = testing_builds_row[build] row['testing_version'] += " " + HTML.tag( 'div', c=HTML.tag( 'a', href="%s/updates/%s" % ( self._prod_url, up.alias), c=HTML.tag( 'img', src=karma_icon_url) + HTML.tag( 'span', c='%s karma' % up.karma)), **{'class': '%s' % karma}) return (len(releases), releases)
def create_bodhi_update( self, dist_git_branch: str, update_type: str, update_notes: str, koji_builds: Sequence[str] = None, bugzilla_ids: Optional[List[int]] = None, ): logger.debug( f"About to create a Bodhi update of type {update_type!r} from {dist_git_branch!r}" ) # bodhi will likely prompt for username and password if kerb ticket is not up b = BodhiClient() if not koji_builds: # alternatively we can call something like `koji latest-build rawhide sen` builds_d = b.latest_builds(self.package_config.downstream_package_name) builds_str = "\n".join(f" - {b}" for b in builds_d) logger.debug( "Koji builds for package " f"{self.package_config.downstream_package_name!r}: \n{builds_str}" ) # EPEL uses "testing-candidate" instead of "updates-candidate" prefix = "testing" if dist_git_branch.startswith("epel") else "updates" koji_tag = f"{dist_git_branch}-{prefix}-candidate" try: koji_builds = [builds_d[koji_tag]] koji_builds_str = "\n".join(f" - {b}" for b in koji_builds) logger.info( "Koji builds for package " f"{self.package_config.downstream_package_name!r} and koji tag {koji_tag}:" f"\n{koji_builds_str}" ) except KeyError: raise PackitException( f"There is no build for {self.package_config.downstream_package_name!r} " f"in koji tag {koji_tag}" ) # I was thinking of verifying that the build is valid for a new bodhi update # but in the end it's likely a waste of resources since bodhi will tell us rendered_note = update_notes.format(version=self.specfile.get_version()) try: save_kwargs = { "builds": koji_builds, "notes": rendered_note, "type": update_type, } if bugzilla_ids: save_kwargs["bugs"] = list(map(str, bugzilla_ids)) result = b.save(**save_kwargs) logger.debug(f"Bodhi response:\n{result}") logger.info( f"Bodhi update {result['alias']}:\n" f"- {result['url']}\n" f"- stable_karma: {result['stable_karma']}\n" f"- unstable_karma: {result['unstable_karma']}\n" f"- notes:\n{result['notes']}\n" ) if "caveats" in result: for cav in result["caveats"]: logger.info(f"- {cav['name']}: {cav['description']}\n") except BodhiClientException as ex: logger.error(ex) raise PackitException( f"There is a problem with creating the bodhi update:\n{ex}" ) return result["alias"]
def __init__(self, environ, request): super(BodhiConnector, self).__init__(environ, request) self._prod_url = config.get('fedoracommunity.connector.bodhi.produrl', 'https://bodhi.fedoraproject.org') self._bodhi_client = BodhiClient(self._base_url, insecure=self._insecure)
class BodhiConnector(IConnector, ICall, IQuery): _method_paths = dict() _query_paths = dict() _cache_prompts = dict() def __init__(self, environ, request): super(BodhiConnector, self).__init__(environ, request) self._prod_url = config.get('fedoracommunity.connector.bodhi.produrl', 'https://bodhi.fedoraproject.org') self._bodhi_client = BodhiClient(self._base_url, insecure=self._insecure) @classmethod def query_updates_cache_prompt(cls, msg): if '.bodhi.' not in msg['topic']: return msg = msg['msg'] if 'update' in msg: update = msg['update'] release = update['release']['name'] status = update['status'] nvrs = [build['nvr'] for build in update['builds']] names = ['-'.join(nvr.split('-')[:-2]) for nvr in nvrs] releases = [release, ''] statuses = [status, ''] groupings = [False] headers = ['package', 'release', 'status', 'group_updates'] combinations = product(names, releases, statuses, groupings) for values in combinations: yield dict(zip(headers, values)) @classmethod def query_active_releases_cache_prompt(cls, msg): if '.bodhi.' not in msg['topic']: return msg = msg['msg'] if 'update' in msg: nvrs = [build['nvr'] for build in msg['update']['builds']] names = ['-'.join(nvr.split('-')[:-2]) for nvr in nvrs] for name in names: yield {'package': name} # IConnector @classmethod def register(cls): cls._base_url = config.get('fedoracommunity.connector.bodhi.baseurl', 'https://bodhi.fedoraproject.org/') check_certs = asbool(config.get('fedora.clients.check_certs', True)) cls._insecure = not check_certs cls.register_query_updates() cls.register_query_active_releases() def request_data(self, path, params): return self._bodhi_client.send_request(path, auth=False, params=params) def introspect(self): # FIXME: return introspection data return None #ICall def call(self, resource_path, params): log.debug('BodhiConnector.call(%s)' % locals()) # proxy client only returns structured data so we can pass # this off to request_data but we should fix that in ProxyClient return self.request_data(resource_path, params) #IQuery @classmethod def register_query_updates(cls): path = cls.register_query('query_updates', cls.query_updates, cls.query_updates_cache_prompt, primary_key_col='request_id', default_sort_col='request_id', default_sort_order=-1, can_paginate=True) path.register_column('request_id', default_visible=False, can_sort=False, can_filter_wildcards=False) path.register_column('updateid', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('nvr', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('submitter', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('status', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('request', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('karma', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('nagged', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('type', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('approved', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('date_submitted', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('date_pushed', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('date_modified', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('comments', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('bugs', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('builds', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('releases', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('release', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('karma_level', default_visible=True, can_sort=False, can_filter_wildcards=False) f = ParamFilter() f.add_filter('package', ['nvr'], allow_none=False) f.add_filter('status', ['status'], allow_none=True) f.add_filter('group_updates', allow_none=True, cast=bool) f.add_filter('granularity', allow_none=True) f.add_filter('release', allow_none=False) cls._query_updates_filter = f def query_updates(self, start_row=None, rows_per_page=None, order=-1, sort_col=None, filters=None, **params): if not filters: filters = {} filters = self._query_updates_filter.filter(filters, conn=self) group_updates = filters.get('group_updates', True) params.update(filters) params['page'] = int(start_row / rows_per_page) + 1 # If we're grouping updates, ask for twice as much. This is so we can # handle the case where there are two updates for each package, one for # each release. Yes, worst case we get twice as much data as we ask # for, but this allows us to do *much* more efficient database calls on # the server. if group_updates: params['rows_per_page'] = rows_per_page * 2 else: params['rows_per_page'] = rows_per_page # Convert bodhi1 query format to bodhi2. if 'package' in params: params['packages'] = params.pop('package') if 'release' in params: params['releases'] = params.pop('release') results = self._bodhi_client.send_request('updates', auth=False, params=params) total_count = results['total'] if group_updates: updates_list = self._group_updates(results['updates'], num_packages=rows_per_page) else: updates_list = results['updates'] for up in updates_list: versions = [] releases = [] if group_updates: up['title'] = up['dist_updates'][0]['title'] for dist_update in up['dist_updates']: versions.append(dist_update['version']) releases.append(dist_update['release_name']) up['name'] = up['package_name'] up['versions'] = versions up['releases'] = releases up['status'] = up['dist_updates'][0]['status'] up['nvr'] = up['dist_updates'][0]['title'] up['request_id'] = up['package_name'] + \ dist_update['version'].replace('.', '') else: chunks = up['title'].split('-') up['name'] = '-'.join(chunks[:-2]) up['version'] = '-'.join(chunks[-2:]) up['versions'] = chunks[-2] up['releases'] = up['release']['long_name'] up['nvr'] = up['title'] up['request_id'] = up.get('updateid') or \ up['nvr'].replace('.', '').replace(',', '') up['id'] = up['nvr'].split(',')[0] # A unique id that we can use in HTML class fields. #up['request_id'] = up.get('updateid') or \ # up['nvr'].replace('.', '').replace(',', '') actions = [] up['actions'] = '' for action in actions: reqs = '' if group_updates: for u in up['dist_updates']: reqs += "update_action('%s', '%s');" % (u['title'], action[0]) title = up['dist_updates'][0]['title'] else: reqs += "update_action('%s', '%s');" % (up['title'], action[0]) title = up['title'] # FIXME: Don't embed HTML up['actions'] += """ <button id="%s_%s" onclick="%s return false;">%s</button><br/> """ % (title.replace('.', ''), action[0], reqs, action[1]) # Dates if group_updates: date_submitted = up['dist_updates'][0]['date_submitted'] date_pushed = up['dist_updates'][0]['date_pushed'] else: date_submitted = up['date_submitted'] date_pushed = up['date_pushed'] granularity = filters.get('granularity', 'day') ds = DateTimeDisplay(date_submitted) up['date_submitted_display'] = ds.age(granularity=granularity, general=True) + ' ago' if date_pushed: dp = DateTimeDisplay(date_pushed) up['date_pushed'] = dp.datetime.strftime('%d %b %Y') up['date_pushed_display'] = dp.age(granularity=granularity, general=True) + ' ago' # karma # FIXME: take into account karma from both updates if group_updates: k = up['dist_updates'][0]['karma'] else: k = up['karma'] if k: up['karma_str'] = "%+d" % k else: up['karma_str'] = " %d" % k up['karma_level'] = 'meh' if k > 0: up['karma_level'] = 'good' if k < 0: up['karma_level'] = 'bad' up['details'] = self._get_update_details(up) return (total_count, updates_list) def _get_update_details(self, update): details = '' if update['status'] == 'stable': if update.get('updateid'): details += HTML.tag('a', c=update['updateid'], href='%s/updates/%s' % (self._prod_url, update['alias'])) if update.get('date_pushed'): details += HTML.tag('br') + update['date_pushed'] else: details += 'In process...' elif update['status'] == 'pending' and update.get('request'): details += 'Pending push to %s' % update['request'] details += HTML.tag('br') details += HTML.tag('a', c="View update details >", href="%s/updates/%s" % (self._prod_url, update['alias'])) elif update['status'] == 'obsolete': for comment in update['comments']: if comment['user']['name'] == 'bodhi': if comment['text'].startswith('This update has been ' 'obsoleted by '): details += markdown.markdown(comment['text'], safe_mode="replace") return details def _get_update_actions(self, update): actions = [] if update['request']: actions.append(('revoke', 'Cancel push')) else: if update['status'] == 'testing': actions.append(('unpush', 'Unpush')) actions.append(('stable', 'Push to stable')) if update['status'] == 'pending': actions.append(('testing', 'Push to testing')) actions.append(('stable', 'Push to stable')) return actions def _group_updates(self, updates, num_packages=None): """ Group a list of updates by release. This method allows allows you to limit the number of packages, for when we want to display 1 package per row, regardless of how many updates there are for it. """ packages = {} done = False i = 0 if not updates: return [] for update in updates: for build in update['builds']: pkg = build['nvr'].rsplit('-', 2)[0] if pkg not in packages: if num_packages and i >= num_packages: done = True break packages[pkg] = { 'package_name': pkg, 'dist_updates': list() } i += 1 else: skip = False for up in packages[pkg]['dist_updates']: if up['release_name'] == \ update['release']['long_name']: skip = True break if skip: break packages[pkg]['dist_updates'].append({ 'release_name': update['release']['long_name'], 'version': '-'.join(build['nvr'].split('-')[-2:]) }) packages[pkg]['dist_updates'][-1].update(update) if done: break result = [packages[p] for p in packages] sort_col = 'date_submitted' if result[0]['dist_updates'][0]['status'] == 'stable': sort_col = 'date_pushed' result = sorted(result, reverse=True, cmp=lambda x, y: cmp(x['dist_updates'][0][sort_col], y[ 'dist_updates'][0][sort_col])) return result def add_updates_to_builds(self, builds): """Update a list of koji builds with the corresponding bodhi updates. This method makes a single query to bodhi, asking if it knows about any updates for a given list of koji builds. For builds with existing updates, the `update` will be added to it's dictionary. Currently it also adds `update_details`, which is HTML for rendering the builds update options. Ideally, this should be done client-side in the template (builds/templates/table_widget.mak). """ start = datetime.now() updates = self.call('get_updates_from_builds', {'builds': ' '.join([b['nvr'] for b in builds])}) if updates: # FIXME: Lets stop changing the upstream APIs by putting the # session id as the first element, and the results in the second. updates = updates[1] for build in builds: if build['nvr'] in updates: build['update'] = updates[build['nvr']] status = build['update']['status'] details = '' # FIXME: ideally, we should just return the update JSON and do # this logic client-side in the template when the grid data # comes in. if status == 'stable': details = 'Pushed to updates' elif status == 'testing': details = 'Pushed to updates-testing' elif status == 'pending': details = 'Pending push to %s' % build['update']['request'] details += HTML.tag('br') details += HTML.tag('a', c="View update details >", href="%s/updates/%s" % (self._prod_url, build['update']['alias'])) else: details = HTML.tag('a', c='Push to updates >', href='%s/new?builds.text=%s' % (self._prod_url, build['nvr'])) build['update_details'] = details log.debug("Queried bodhi for builds in: %s" % (datetime.now() - start)) @classmethod def register_query_active_releases(cls): path = cls.register_query('query_active_releases', cls.query_active_releases, cls.query_active_releases_cache_prompt, primary_key_col='release', default_sort_col='release', default_sort_order=-1, can_paginate=True) path.register_column('release', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('stable_version', default_visible=True, can_sort=False, can_filter_wildcards=False) path.register_column('testing_version', default_visible=True, can_sort=False, can_filter_wildcards=False) f = ParamFilter() f.add_filter('package', ['nvr'], allow_none=False) cls._query_active_releases = f def get_all_releases(self): releases_obj = self.call('releases', {}) releases_all = None for i in range(1, releases_obj['pages'] + 1): if releases_all == None: releases_all = releases_obj['releases'] continue temp = self.call('releases?page=' + str(i), {})['releases'] releases_all.extend(temp) if releases_all == None: raise TypeError return releases_all def query_active_releases(self, filters=None, **params): releases = list() queries = list() # Mapping of tag -> release release_tag = dict() # List of testing builds to query bodhi for testing_builds = list() # nvr -> release lookup table testing_builds_row = dict() if not filters: filters = dict() filters = self._query_updates_filter.filter(filters, conn=self) package = filters.get('package') koji = get_connector('koji')._koji_client koji.multicall = True releases_all = self.get_all_releases() releases_all.append({ 'dist_tag': 'rawhide', 'long_name': 'Rawhide', 'stable_tag': 'rawhide', 'testing_tag': 'no_testing_tag_found', 'state': 'current' }) releases_all = sorted(releases_all, key=lambda k: k['dist_tag'], reverse=True) for release in releases_all: if release['state'] not in ['current', 'pending']\ or 'Modular' in release['long_name']: continue tag = release['dist_tag'] name = release['long_name'] r = { 'release': name, 'stable_version': 'None', 'testing_version': 'None' } if tag == 'rawhide': koji.listTagged(tag, package=package, latest=True, inherit=True) queries.append(tag) release_tag[tag] = r else: stable_tag = release['stable_tag'] testing_tag = release['testing_tag'] koji.listTagged(stable_tag, package=package, latest=True, inherit=True) queries.append(stable_tag) release_tag[stable_tag] = r koji.listTagged(testing_tag, package=package, latest=True) queries.append(testing_tag) release_tag[testing_tag] = r releases.append(r) results = koji.multiCall() for i, result in enumerate(results): if isinstance(result, dict): if 'faultString' in result: log.error("FAULT: %s" % result['faultString']) else: log.error("Can't find fault string in result: %s" % result) else: query = queries[i] row = release_tag[query] release = result[0] if query == 'dist-rawhide': if release: nvr = parse_build(release[0]['nvr']) row['stable_version'] = HTML.tag( 'a', c='%(version)s-%(release)s' % nvr, href=koji_build_url % nvr) else: row['stable_version'] = \ 'No builds tagged with %s' % tag row['testing_version'] = HTML.tag('i', c='Not Applicable') continue if release: release = release[0] if query.endswith('-testing'): nvr = parse_build(release['nvr']) row['testing_version'] = HTML.tag( 'a', c='%(version)s-%(release)s' % nvr, href=koji_build_url % nvr) testing_builds.append(release['nvr']) testing_builds_row[release['nvr']] = row else: # stable nvr = parse_build(release['nvr']) row['stable_version'] = HTML.tag( 'a', c='%(version)s-%(release)s' % nvr, href=koji_build_url % nvr) if release['tag_name'].endswith('-updates'): row['stable_version'] += ' (' + HTML.tag( 'a', c='update', href='%s/updates/?builds=%s' % (self._prod_url, nvr['nvr'])) + ')' # If there are updates in testing, then query bodhi with a single call if testing_builds: data = self.call('updates', {'builds': ' '.join(testing_builds)}) updates = data['updates'] for up in updates: for build in up['builds']: if build['nvr'] in testing_builds: break else: continue build = build['nvr'] if up.karma > 1: up.karma_icon = 'good' elif up.karma < 0: up.karma_icon = 'bad' else: up.karma_icon = 'meh' karma_ico_16 = '/images/16_karma-%s.png' % up.karma_icon karma_icon_url = \ self._request.environ.get('SCRIPT_NAME', '') + \ karma_ico_16 karma = 'karma_%s' % up.karma_icon row = testing_builds_row[build] row['testing_version'] += " " + HTML.tag( 'div', c=HTML.tag('a', href="%s/updates/%s" % (self._prod_url, up.alias), c=HTML.tag('img', src=karma_icon_url) + HTML.tag('span', c='%s karma' % up.karma)), **{'class': '%s' % karma}) return (len(releases), releases)