def __init__(self): self.bz = bugzilla.RHBugzilla( url=self.BUGZILLA_API_URL, cookiefile=None, tokenfile=None, use_creds=False, )
def connect(url, user, password): '''connect to bugzilla, return the bugzilla connection''' ret = bugzilla.RHBugzilla(url=url, user=user, password=password) if not ret: raise RuntimeError("Couldn't connect to bugzilla: %s %s %s" % (url, username, password)) logger.debug('connected to bugzilla: %s %s %s %s', url, user, password, ret) return ret
def _get_bugzilla_bug(bug_id): """Fetch bug ``bug_id``. :param int bug_id: The ID of a bug in the Bugzilla database. :return: A FRIGGIN UNDOCUMENTED python-bugzilla THING. :raises BugFetchError: If an error occurs while fetching the bug. For example, a network timeout occurs or the bug does not exist. """ # Is bug ``bug_id`` in the cache? if bug_id in _bugzilla: LOGGER.debug('Bugzilla bug {0} found in cache.'.format(bug_id)) else: LOGGER.info('Bugzilla bug {0} not in cache. Fetching.'.format(bug_id)) # Make a network connection to the Bugzilla server. try: bz_conn = bugzilla.RHBugzilla() bz_conn.connect(BUGZILLA_URL) except (TypeError, ValueError): raise BugFetchError( 'Could not connect to {0}'.format(BUGZILLA_URL)) # Fetch the bug and place it in the cache. try: _bugzilla[bug_id] = bz_conn.getbugsimple(bug_id) except Fault as err: raise BugFetchError('Could not fetch bug. Error: {0}'.format( err.faultString)) except ExpatError as err: raise BugFetchError('Could not interpret bug. Error: {0}'.format( ErrorString(err.code))) return _bugzilla[bug_id]
def is_bzbug(bz_id): """Method to test if the bugzilla's bug is open""" try: mybz = bugzilla.RHBugzilla() mybz.connect(BUGZILLA_URL) except (TypeError, ValueError): logging.warning("Invalid Bugzilla ID {0}".format(bz_id)) return False attempts = 0 mybug = None while attempts < 3 and mybug is None: try: mybug = mybz.getbugsimple(bz_id) except ExpatError: attempts += 1 except Fault: return True if mybug is None: return True if (mybug.status == 'NEW') or (mybug.status == 'ASSIGNED'): logging.debug(mybug) return True else: return False
def _from_bugzilla(self, bug_id): if bugzilla is None: return '' rhbz = bugzilla.RHBugzilla(url="https://bugzilla.redhat.com/xmlrpc.cgi") query = rhbz.build_query(bug_id=bug_id) bug = rhbz.query(query)[0] return bug.summary
def testPostTranslation(self): def _testPostCompare(bz, indict, outexpect): outdict = indict.copy() bz.post_translation({}, outdict) self.assertTrue(outdict == outexpect) # Make sure multiple calls don't change anything bz.post_translation({}, outdict) self.assertTrue(outdict == outexpect) bug3 = bugzilla.Bugzilla3(url=None, cookiefile=None) rhbz = bugzilla.RHBugzilla(url=None, cookiefile=None) test1 = { "component": ["comp1"], "version": ["ver1", "ver2"], 'flags': [{ 'is_active': 1, 'name': 'qe_test_coverage', 'setter': '*****@*****.**', 'status': '?', }, { 'is_active': 1, 'name': 'rhel-6.4.0', 'setter': '*****@*****.**', 'status': '+', }], 'alias': ["FOO", "BAR"], 'blocks': [782183, 840699, 923128], 'keywords': ['Security'], 'groups': ['redhat'], } out_simple = test1.copy() out_simple["components"] = out_simple["component"] out_simple["component"] = out_simple["components"][0] out_simple["versions"] = out_simple["version"] out_simple["version"] = out_simple["versions"][0] out_complex = out_simple.copy() out_complex["keywords"] = ",".join(out_complex["keywords"]) out_complex["blocks"] = ",".join([str(b) for b in out_complex["blocks"]]) out_complex["alias"] = ",".join(out_complex["alias"]) out_complex["groups"] = [{'description': 'redhat', 'ison': 1, 'name': 'redhat'}] out_complex["flags"] = "qe_test_coverage?,rhel-6.4.0+" _testPostCompare(bug3, test1, test1) _testPostCompare(rhbz, test1, out_simple) rhbz.rhbz_back_compat = True _testPostCompare(rhbz, test1, out_complex)
def _get_bugzilla_bug(bug_id): """Fetch bug ``bug_id``. :param int bug_id: The ID of a bug in the Bugzilla database. :return: A FRIGGIN UNDOCUMENTED python-bugzilla THING. :raises BugFetchError: If an error occurs while fetching the bug. For example, a network timeout occurs or the bug does not exist. """ # Is bug ``bug_id`` in the cache? if bug_id in _bugzilla: LOGGER.debug('Bugzilla bug {0} found in cache.'.format(bug_id)) else: LOGGER.info('Bugzilla bug {0} not in cache. Fetching.'.format(bug_id)) # Make a network connection to the Bugzilla server. bz_credentials = {} if setting_is_set('bugzilla'): bz_credentials = settings.bugzilla.get_credentials() try: bz_conn = bugzilla.RHBugzilla(url=BUGZILLA_URL, **bz_credentials) except (TypeError, ValueError): # pragma: no cover raise BugFetchError( 'Could not connect to {0}'.format(BUGZILLA_URL) ) # Fetch the bug and place it in the cache. try: _bugzilla[bug_id] = bz_conn.getbug( bug_id, include_fields=['id', 'status', 'whiteboard', 'flags'] ) if not bz_credentials: raise BZUnauthenticatedCall( _bugzilla[bug_id], 'Unauthenticated call made to BZ API, no flags data will ' 'be available' ) except Fault as err: raise BugFetchError( 'Could not fetch bug. Error: {0}'.format(err.faultString) ) except ExpatError as err: raise BugFetchError( 'Could not interpret bug. Error: {0}' .format(ErrorString(err.code)) ) return _bugzilla[bug_id]
def __init__(self, bugzilla_url, product, release, status, comment=False, dry_run=False): self.bugzilla_url = bugzilla_url self.product = product self.release = release self.status = status self.dry_run = dry_run self.comment = comment self.bzapi = bugzilla.RHBugzilla(self.bugzilla_url) if not self.bzapi.logged_in: logging.error('The application requires bugzilla credentials.') sys.exit(1)
def get_rfes_from_bugzilla(bugs): # Open connection into bugzilla user_params = parse_config() username = user_params.get("user") + "@redhat.com" password = user_params.get("password") #used for local testing #if bugs == "": #bugs = "1517272" #,1011755,1003044" #1517272"#, #connect to Bugzilla bz_connection = bugzilla.RHBugzilla(url=BUGZILLA_SERVER) bz_connection.login(username,password) print "Bugzilla connection: " + str(bz_connection.logged_in) # Build RFE query query = bz_connection.build_query( bug_id = bugs ) bz_rfes = bz_connection.query(query) return bz_rfes, bz_connection
def get_rfes_from_bugzilla(): # Open connection into bugzilla user_params = parse_config() username = user_params.get("user") + "@redhat.com" password = user_params.get("password") # rhbugzilla = bugzilla.RHBugzilla() bz_connection = bugzilla.RHBugzilla(url=BUGZILLA_SERVER) bz_connection.login(username,password) # query print "Bugzilla connection: " + str(bz_connection.logged_in) query = bz_connection.build_query( savedsearch="QE_Test_Coverage_RHOS_7" ) bz_rfes = bz_connection.query(query) return bz_rfes, bz_connection
def bzbug(bz_id): """Decorator that skips the test if the bugzilla's bug is open""" if bz_id not in _bugzilla: try: mybz = bugzilla.RHBugzilla() mybz.connect(BUGZILLA_URL) except (TypeError, ValueError): logging.warning("Invalid Bugzilla ID {0}".format(bz_id)) return lambda func: func attempts = 0 mybug = None while attempts < 3 and mybug is None: try: mybug = mybz.getbugsimple(bz_id) _bugzilla[bz_id] = mybug except ExpatError: attempts += 1 except Fault as error: return unittest.skip( "Test skipped: %s" % error.faultString ) if mybug is None: return unittest.skip( "Test skipped due to not being able to fetch bug #%s info" % bz_id) else: mybug = _bugzilla[bz_id] if (mybug.status == 'NEW') or (mybug.status == 'ASSIGNED'): logging.debug(mybug) return unittest.skip("Test skipped due to %s" % mybug) else: return lambda func: func
def get_rfes_from_bugzilla(): # Open connection into bugzilla user_params = parse_config() username = user_params.get("user") + "@redhat.com" password = user_params.get("password") # rhbugzilla = bugzilla.RHBugzilla() bz_connection = bugzilla.RHBugzilla(url=BUGZILLA_SERVER) bz_connection.login(username, password) # Build RFE query #https: // bugzilla.redhat.com / buglist.cgi?action = wrap & bug_status = NEW & bug_status = ASSIGNED & bug_status = POST & bug_status = MODIFIED & bug_status = ON_DEV & bug_status = ON_QA & bug_status = VERIFIED & classification = Red % 20 #Hat & f1 = component & f2 = cf_devel_whiteboard & f3 = cf_devel_whiteboard & keywords = FutureFeature % 2 #C % 20 & keywords_type = allwords & list_id = 8251986 & o1 = notsubstring & o2 = substring & o3 = notsubstring & product = Red % 20 #Hat % 20 OpenStack & v1 = doc & v2 = osp13add & v3 = osp13rem # https: // bugzilla.redhat.com / buglist.cgi?action = wrap & bug_status = POST & bug_status = MODIFIED & bug_status = ON_QA & bug_status = VERIFIED & classification = Red % 20 Hat & # f2 = flagtypes.name & # f3 = target_milestone & # keywords = FutureFeature % 2C % 20 Triaged % 2C % 20 & # keywords_type = allwords & # list_id = 9078558 & # o2 = regexp & # o3 = notequals & # product = Red % 20Hat % 20 OpenStack & # v2 = rhos. % 2A - 14.0 % 5C % 2B & # v3 = --- print "Bugzilla connection: " + str(bz_connection.logged_in) query = bz_connection.build_query(quicksearch="1241596") # exclude: 1475556, 1288035 , 1375207, 1467591, 1512686, 1524393, 1524402 bz_rfes = bz_connection.query(query) return bz_rfes, bz_connection
def testUserAgent(self): b3 = bugzilla.Bugzilla3(url=None, cookiefile=None) rhbz = bugzilla.RHBugzilla(url=None, cookiefile=None) self.assertTrue(b3.user_agent.endswith("Bugzilla3/0.1")) self.assertTrue(rhbz.user_agent.endswith("RHBugzilla/0.1"))
def _get_connection(self): if not self._connection: bz_conn = bugzilla.RHBugzilla(url=BUGZILLA_URL, **self.credentials) self._connection = bz_conn return self._connection
def main(): """ Main """ argparser = argparse.ArgumentParser(description='Report validation result to bugzilla') argparser.add_argument('--bugzilla-component', default="images", help='use specified bugzilla component') argparser.add_argument('--bugzilla-product', default="Cloud Image Validation", help='use specified bugzilla product') argparser.add_argument('--bugzilla-url', default="https://bugzilla.redhat.com/xmlrpc.cgi", help='use specified bugzilla xmlrpc url') argparser.add_argument('--config', default="/etc/validation.yaml", help='use supplied yaml config file') argparser.add_argument('--debug', action='store_const', const=True, default=False, help='debug mode') argparser.add_argument('--result', help='yaml file with validation result', required=True) argparser.add_argument('--test', action='store_const', const=True, default=False, help='report to stdout instead of bugzilla') argparser.add_argument('-y', '--yaml-summary', help='provide info in yaml', action='store_const', default=False, const=True) argparser.add_argument('-a', '--all-commands', help='show all commands in bugzillas not just failed', action='store_true') args = argparser.parse_args() resultd = open(args.result, 'r') try: from yaml import CLoader as Loader except ImportError: from yaml import Loader result = yaml.load(resultd, Loader=Loader) resultd.close() summary = Summary(yaml_summary=args.yaml_summary) if not args.test: confd = open(args.config, 'r') yamlconfig = yaml.load(confd) confd.close() bugzilla_user = yamlconfig["bugzilla"]["user"] bugzilla_password = yamlconfig["bugzilla"]["password"] bzid = bugzilla.RHBugzilla(url=args.bugzilla_url, user=bugzilla_user, password=bugzilla_password) if not bzid: print "Failed to connect to bugzilla!" sys.exit(1) for ami in result: ami_fd = tempfile.NamedTemporaryFile() ami_fd.write(yaml.safe_dump(ami)) ami_fd.seek(0) overall_result, bug_summary, info, bug_description = valid_result.get_overall_result(ami, verbose=args.all_commands) bugnr = None if not args.test: bzobject = bzid.createbug(product=args.bugzilla_product, component=args.bugzilla_component, version="RHEL" + ami["version"], rep_platform=ami["arch"], summary=bug_summary, op_sys="Linux", keywords=["TestOnly"]) if not bzobject: print "Failed to create bug in bugzilla!" sys.exit(1) bugid = str(bzobject.bug_id) attach_name = ami["ami"] + ".yaml" bzid.attachfile(bugid, ami_fd, attach_name, filename=attach_name, contenttype="text/yaml", ispatch=False) # FIXME: check previous call result bug = None for ntry in xrange(10): try: bug = bzid.getbug(bugid) break except: # bug not found, retry time.sleep(10) if bug: bugnr = bug.id for comment in bug_description: bug.addcomment(comment) bug.setstatus("VERIFIED" if overall_result == "succeeded" else "ON_QA") else: print info print '\n'.join(bug_description) summary.add(ami['ami'], bug=bugnr, status='pass' if overall_result == "succeeded" else 'fail') ami_fd.close() print summary
def run(self): reponames = self.opts.py3query_repo if not reponames: reponames = ['rawhide'] self.base_query = self.base.sack.query() self.pkg_query = self.base_query.filter(reponame=list(reponames)) source_reponames = [n + '-source' for n in reponames] self.src_query = self.base_query.filter(reponame=source_reponames).filter(arch=['src']) # python_versions: {package: set of Python versions} python_versions = collections.defaultdict(set) # rpm_pydeps: {package: set of dep names} rpm_pydeps = collections.defaultdict(set) # dep_versions: {dep name: Python version} dep_versions = collections.defaultdict(set) for n, seeds in SEED_PACKAGES.items(): provides = sorted(self.all_provides(reponames, seeds), key=str) # This effectively includes packages that still need # Python 3.4 while Rawhide only provides Python 3.5 provides += sorted(seeds) for dep in progressbar(provides, 'Getting py{} requires'.format(n)): dep_versions[str(dep)] = n for pkg in self.whatrequires(dep, self.pkg_query): python_versions[pkg].add(n) rpm_pydeps[pkg].add(str(dep)) # Add packages with 'python?' as a component of their name, if they # haven't been added as dependencies for name, version in { 'python': 0, 'python2': 2, 'python3': 3, }.items(): for pattern in '{}-*', '*-{}', '*-{}-*': name_glob = pattern.format(name) query = self.pkg_query.filter(name__glob=name_glob) message = 'Getting {} packages'.format(name_glob) for pkg in progressbar(query, message): if pkg.sourcerpm.startswith('mingw-'): # Ignore mingw packages continue if pkg not in python_versions: python_versions[pkg].add(version) # srpm_names: {package: srpm name} # by_srpm_name: {srpm name: set of packages} srpm_names = {} by_srpm_name = collections.defaultdict(set) # repo_srpms: {repo name: set of srpm names} repo_srpms = {} for pkg in progressbar(python_versions.keys(), 'Getting SRPMs'): srpm_name = get_srpm_name(pkg) srpm_names[pkg] = srpm_name by_srpm_name[srpm_name].add(pkg) repo_srpms.setdefault(pkg.reponame, set()).add(srpm_name) # deps_of_pkg: {package: set of packages} deps_of_pkg = collections.defaultdict(set) # build_deps_of_srpm: {srpm: set of packages} build_deps_of_srpm = collections.defaultdict(set) # requirers_of_pkg: {package: set of srpm} requirers_of_pkg = collections.defaultdict(set) # build_requirers_of_pkg: {pkg: set of srpm} build_requirers_of_pkg = collections.defaultdict(set) # all_provides: {provide_name: package} all_provides = {str(r).split()[0]: p for p in python_versions for r in p.provides if not str(r).startswith(PROVIDES_BLACKLIST)} for pkg in progressbar(sorted(python_versions.keys()), 'Getting requirements'): reqs = set() build_reqs = set() for provide in pkg.provides: reqs.update(self.whatrequires(provide, self.pkg_query)) build_reqs.update(self.whatrequires(provide, self.src_query)) for req in reqs: if req in python_versions.keys(): deps_of_pkg[req].add(pkg) # Both Python and non-Python packages here. requirers_of_pkg[pkg].add(req) for req in build_reqs: if req.name in by_srpm_name.keys(): build_deps_of_srpm[req.name].add(pkg) # Both Python and non-Python packages here. build_requirers_of_pkg[pkg].add(req) # unversioned_requirers: {srpm_name: set of srpm_names} unversioned_requirers = collections.defaultdict(set) for pkg in progressbar(set.union(*requirers_of_pkg.values(), *build_requirers_of_pkg.values()), 'Processing packages with ambiguous dependencies'): # Ignore packages that are: if (python_versions.get(pkg) == {3} or # Python 3 only pkg.name.endswith('-doc')): # Documentation continue for require in (pkg.requires + pkg.requires_pre + pkg.recommends + pkg.suggests + pkg.supplements + pkg.enhances): require = str(require).split()[0] requirement = all_provides.get(require) if (is_unversioned(require) and requirement and not (require.endswith('-doc') or python_versions.get(requirement) == {3}) and require != 'python-unversioned-command'): requirement_srpm_name = get_srpm_name(requirement) requirer_srpm_name = get_srpm_name(pkg) unversioned_requirers[requirement_srpm_name].add(requirer_srpm_name) # deps_of_pkg: {srpm name: info} json_output = dict() for name in progressbar(by_srpm_name, 'Generating output'): pkgs = sorted(by_srpm_name[name]) r = json_output[name] = {} r['rpms'] = { format_rpm_name(p): { 'py_deps': {str(d): dep_versions[d] for d in rpm_pydeps[p]}, 'non_python_requirers': { 'build_time': sorted(get_srpm_names(build_requirers_of_pkg[p]) - by_srpm_name.keys()), 'run_time': sorted(get_srpm_names(requirers_of_pkg[p]) - by_srpm_name.keys()), }, 'almost_leaf': ( # is Python 2 and is not required by anything EXCEPT # sibling subpackages 2 in python_versions[p] and not get_srpm_names(build_requirers_of_pkg[p] | requirers_of_pkg[p]) - {name} ), 'legacy_leaf': ( # is Python 2 and is not required by anything 2 in python_versions[p] and not get_srpm_names(build_requirers_of_pkg[p] | requirers_of_pkg[p]) ), } for p in pkgs} set_status(r, pkgs, python_versions) r['deps'] = sorted(set(srpm_names[d] for p in pkgs for d in deps_of_pkg.get(p, '') if srpm_names[d] != name)) r['build_deps'] = sorted(set(srpm_names[d] for d in build_deps_of_srpm.get(name, '') if srpm_names[d] != name)) if unversioned_requirers.get(name): r['unversioned_requirers'] = sorted(unversioned_requirers[name]) # add Bugzilla links if self.opts.fetch_bugzilla: bar = iter(progressbar(['connecting', 'tracker', 'individual'], 'Getting bugs')) next(bar) bz = bugzilla.RHBugzilla(BUGZILLA_URL) next(bar) include_fields = ['id', 'depends_on', 'blocks', 'component', 'status', 'resolution', 'last_change_time'] trackers = bz.getbugs(TRACKER_BUG_IDS, include_fields=include_fields) all_ids = [b for t in trackers for b in t.depends_on] next(bar) bugs = bz.getbugs(all_ids, include_fields=include_fields) bar.close() def bug_namegetter(bug): return '{bug.id} {bug.status} {bug.component}'.format(bug=bug) rank = ['NEW', 'ASSIGNED', 'POST', 'MODIFIED', 'ON_QA', 'VERIFIED', 'RELEASE_PENDING', 'CLOSED'] def key(bug): return rank.index(bug.status), bug.last_change_time bugs = sorted(bugs, key=key) for bug in progressbar(bugs, 'Merging bugs', namegetter=bug_namegetter): r = json_output.get(bug.component, {}) if 'bug' in r.get('links', {}): continue status = bug.status if bug.resolution: status += ' ' + bug.resolution # Let's get the datetime of the last comment and convert to string last_change_datetime = time.strftime( '%Y-%m-%d %H:%M:%S', bug.last_change_time.timetuple()) r.setdefault('links', {})['bug'] = [ bug.weburl, status, last_change_datetime] for tb in bug.blocks: if tb in ADDITIONAL_TRACKER_BUGS: r.setdefault('tracking_bugs', []).append( BUGZILLA_BUG_URL.format(tb)) if (any(tb in bug.blocks for tb in MISPACKAGED_TRACKER_BUG_IDS) and r.get('status') == 'idle' and status not in NOTABUG_STATUSES): r['status'] = "mispackaged" r['note'] = ('There is a problem in Fedora packaging, ' 'not necessarily with the software itself. ' 'See the linked Fedora bug.') # Print out output if self.opts.output: with open(self.opts.output, 'w') as f: json.dump(json_output, f, indent=2, sort_keys=True) else: json.dump(json_output, sys.stdout, indent=2, sort_keys=True) sys.stdout.flush() # Write out a groups.json if self.opts.repo_groups_file: output = {repo_name: {'name': repo_name, 'packages': sorted(srpm_names)} for repo_name, srpm_names in repo_srpms.items()} with open(self.opts.repo_groups_file, 'w') as f: json.dump(output, f, indent=2, sort_keys=True)
data['months'][-1]['month'] += (" (%d)" % curcount) write_html(loader, 'bymonth.html', data, tmpdir, 'NEW.html') return data['count'] if __name__ == '__main__': options = parse_commandline() verbose = options.verbose config = parse_config(options.configfile) if config['maxpackages']: maxpackages = int(config['maxpackages']) dbprint("Limiting to {0} packages".format(maxpackages)) bz = bugzilla.RHBugzilla(url=config['url'], cookiefile=None, user=config['username'], password=config['password']) t = time.time() (bugs, bugdata, usermap) = run_query(bz) querytime = time.time() - t # Don't bother running this stuff until the query completes, since it fails # so often. loader = TemplateLoader(options.templdir) tmpdir = tempfile.mkdtemp(dir=options.dirname) # The initial set of substitutions that's shared between the report functions subs = { 'update': datetime.datetime.now().strftime('%Y-%m-%d %H:%M'), 'querytime': querytime, 'version': VERSION,
#!/usr/bin/env python3 from __future__ import print_function import bugzilla import os import sys api_key = None try: api_key = open(os.path.expanduser('~/.bugzilla_apikey')).read().strip() except FileNotFoundError: # api key in ~/.config/python-bugzilla/bugzillarc will be used pass bz = bugzilla.RHBugzilla(url=None, api_key=api_key) bz.connect('https://bugzilla.redhat.com') PRODUCT_NAME = "OpenShift Container Platform" try: TARGET_RELEASE = sys.argv[1] # e.g. "4.5.0" except IndexError: sys.exit("Target Release required, e.g. '4.5.0'") query = bz.build_query( product=PRODUCT_NAME, status=["MODIFIED", "ON_QA", "VERIFIED"], target_release=TARGET_RELEASE, keywords=["Security", "SecurityTracking"], )
import pytest import bugzilla from bugzilla import Bugzilla import tests RHURL = tests.CLICONFIG.REDHAT_URL or "partner-bugzilla.redhat.com" def _split_int(s): return [int(i) for i in s.split(",")] if not bugzilla.RHBugzilla(url=RHURL).logged_in: print("R/W tests require cached login credentials for url=%s" % RHURL) sys.exit(1) class RHPartnerTest(unittest.TestCase): # Despite its name, this instance is simply for bugzilla testing, # doesn't send out emails and is blown away occasionally. The front # page has some info. url = RHURL bzclass = bugzilla.RHBugzilla def _check_have_admin(self, bz, funcname): # groupnames is empty for any user if our logged in user does not # have admin privs. # Check a known account that likely won't ever go away
def run(self, args): if self.opts.help_cmd: print(self.parser.format_help()) return reponame = self.opts.py3query_repo self.base_query = self.base.sack.query() self.pkg_query = self.base_query.filter(reponame=reponame) self.src_query = self.base_query.filter(reponame=reponame + '-source').filter(arch=['src']) # python_versions: {package: set of Python versions} python_versions = collections.defaultdict(set) # rpm_pydeps: {package: set of dep names} rpm_pydeps = collections.defaultdict(set) # dep_versions: {dep name: Python version} dep_versions = collections.defaultdict(set) for n, seeds in SEED_PACKAGES.items(): provides = sorted(self.all_provides(reponame, seeds), key=str) # This effectively includes packages that still need # Python 3.4 while Rawhide only provides Python 3.5 provides += sorted(seeds) for dep in progressbar(provides, 'Getting py{} requires'.format(n)): dep_versions[str(dep)] = n for pkg in self.whatrequires(dep): python_versions[pkg].add(n) rpm_pydeps[pkg].add(str(dep)) # srpm_names: {package: srpm name} # by_srpm_name: {srpm name: set of packages} srpm_names = {} by_srpm_name = collections.defaultdict(set) for pkg in progressbar(python_versions.keys(), 'Getting SRPMs'): srpm_name = hawkey.split_nevra(pkg.sourcerpm).name srpm_names[pkg] = srpm_name by_srpm_name[srpm_name].add(pkg) # deps_of_pkg: {package: set of packages} deps_of_pkg = collections.defaultdict(set) all_provides = { str(r): r for p in python_versions for r in p.provides if not str(r).startswith(PROVIDES_BLACKLIST) } for pkg in progressbar(sorted(python_versions.keys()), 'Getting requirements'): reqs = set() for provide in pkg.provides: reqs.update(self.whatrequires(provide)) for req in reqs: if req in python_versions.keys(): deps_of_pkg[req].add(pkg) # deps_of_pkg: {srpm name: info} json_output = dict() for name in progressbar(by_srpm_name, 'Generating output'): pkgs = sorted(by_srpm_name[name]) r = json_output[name] = {} set_status(r, pkgs, python_versions) r['rpms'] = { format_rpm_name(p): {str(d): dep_versions[d] for d in rpm_pydeps[p]} for p in pkgs } r['deps'] = sorted( set(srpm_names[d] for p in pkgs for d in deps_of_pkg.get(p, '') if srpm_names[d] != name)) # add Bugzilla links if self.opts.fetch_bugzilla: bar = iter( progressbar(['connecting', 'tracker', 'individual'], 'Getting bugs')) next(bar) bz = bugzilla.RHBugzilla(BUGZILLA_URL) next(bar) include_fields = [ 'id', 'depends_on', 'blocks', 'component', 'status', 'resolution' ] trackers = bz.getbugs(TRACKER_BUG_IDS, include_fields=include_fields) all_ids = [b for t in trackers for b in t.depends_on] next(bar) bugs = bz.getbugs(all_ids, include_fields=include_fields) bar.close() def bug_namegetter(bug): return '{bug.id} {bug.status} {bug.component}'.format(bug=bug) for bug in progressbar(bugs, 'Merging bugs', namegetter=bug_namegetter): r = json_output.get(bug.component, {}) url = '{bug.weburl}#{bug.status}'.format(bug=bug) status = bug.status if bug.resolution: status += ' ' + bug.resolution r.setdefault('links', {})['bug'] = [bug.weburl, status] inprogress_statuses = ('ASSIGNED', 'POST', 'MODIFIED', 'ON_QA') inprogress_resolutions = ('CURRENTRELEASE', 'RAWHIDE', 'ERRATA', 'NEXTRELEASE') if r.get('status') == 'idle' and bug.status != 'NEW': r['status'] = 'in-progress' elif r.get('status') == 'idle' and bug.status == 'NEW' and \ any(tb in bug.blocks for tb in MISPACKAGED_TRACKER_BUG_IDS): r['status'] = "mispackaged" r['note'] = ('There is a problem in Fedora packaging, ' + 'not necessarily with the software itself. ' + 'See the linked Fedora bug.') # Print out output if self.opts.output: with open(self.opts.output, 'w') as f: json.dump(json_output, f, indent=2, sort_keys=True) else: json.dump(json_output, sys.stdout, indent=2, sort_keys=True) sys.stdout.flush()
def run(self): self.base_query = self.base.sack.query() self.pkg_query = self.base_query.filter(arch__neq=['src']) self.src_query = self.base_query.filter(arch=['src']) # python_versions: {package: set of Python versions} python_versions = collections.defaultdict(set) # rpm_pydeps: {package: set of dep names} rpm_pydeps = collections.defaultdict(set) # dep_versions: {dep name: Python version} dep_versions = collections.defaultdict(set) for n, seeds in SEED_PACKAGES.items(): provides = sorted(self.all_provides(seeds), key=str) # This effectively includes packages that still need # Python 3.4 while Rawhide only provides Python 3.5 provides += sorted(seeds) for dep in progressbar(provides, 'Getting py{} requires'.format(n)): dep_versions[str(dep)] = n for pkg in self.whatrequires(dep, self.base_query): python_versions[pkg].add(n) rpm_pydeps[pkg].add(str(dep)) # Add packages with 'python?' as a component of their name, if they # haven't been added as dependencies for name, version in { 'python': 0, 'python2': 2, 'python3': 3, }.items(): for pattern in '{}-*', '*-{}', '*-{}-*': name_glob = pattern.format(name) query = self.base_query.filter(name__glob=name_glob) message = 'Getting {} packages'.format(name_glob) for pkg in progressbar(query, message): if pkg.sourcerpm and pkg.sourcerpm.startswith('mingw-'): # Ignore mingw packages continue if pkg not in python_versions: python_versions[pkg].add(version) # add python27 package manually, it doesn't require Python 2, but it is query = self.pkg_query.filter(name='python27') for pkg in query: python_versions[pkg].add(2) # srpm_names: {package: srpm name} # by_srpm_name: {srpm name: set of packages} srpm_names = {} by_srpm_name = collections.defaultdict(set) # repo_srpms: {repo name: set of srpm names} repo_srpms = {} for pkg in progressbar(python_versions.keys(), 'Getting SRPMs'): srpm_name = get_srpm_name(pkg) srpm_names[pkg] = srpm_name by_srpm_name[srpm_name].add(pkg) repo_srpms.setdefault(pkg.reponame, set()).add(srpm_name) old_misnamed = {} old_misnamed_flat = {} if self.opts.py3query_misnamed: with open(self.opts.py3query_misnamed) as f: old_misnamed = yaml.safe_load(f) old_misnamed_flat = { pk: pr for pkg in old_misnamed for pr, pk in old_misnamed[pkg].items() } # deps_of_pkg: {package: set of packages} deps_of_pkg = collections.defaultdict(set) # build_deps_of_srpm: {srpm: set of packages} build_deps_of_srpm = collections.defaultdict(set) # requirers_of_pkg: {package: set of srpm} requirers_of_pkg = collections.defaultdict(set) # build_requirers_of_pkg: {pkg: set of srpm} build_requirers_of_pkg = collections.defaultdict(set) # all_provides: {provide_name: package} all_provides = { str(r).split()[0]: p for p in python_versions for r in p.provides if not str(r).startswith(PROVIDES_BLACKLIST) } for pkg in progressbar(sorted(python_versions.keys()), 'Getting requirements'): if python_versions[pkg] == {3}: continue if pkg.name in NAME_NOTS: # "NAME_NOTS" are Python-version-agnostic packages, # such as wheels, RPM macros and documentation. # Don't track those as python2 dependencies. continue reqs = set() build_reqs = set() provides = set(pkg.provides) for provide in pkg.provides: str_provide = str(provide).split(' ')[0] if str_provide in old_misnamed_flat: provides.add(old_misnamed_flat[str_provide]) for provide in provides: reqs.update(self.whatrequires(provide, self.pkg_query)) build_reqs.update(self.whatrequires(provide, self.src_query)) for req in reqs: if req in python_versions.keys(): deps_of_pkg[req].add(pkg) # Both Python and non-Python packages here. requirers_of_pkg[pkg].add(req) for req in build_reqs: if req.name in by_srpm_name.keys(): build_deps_of_srpm[req.name].add(pkg) # Both Python and non-Python packages here. build_requirers_of_pkg[pkg].add(req) # unversioned_requirers: {srpm_name: set of srpm_names} unversioned_requirers = collections.defaultdict(set) for pkg in progressbar( set.union(*requirers_of_pkg.values(), *build_requirers_of_pkg.values()), 'Processing packages with ambiguous dependencies'): # Ignore packages that are: if (python_versions.get(pkg) == {3} or # Python 3 only pkg.name.endswith('-doc')): # Documentation continue for require in (pkg.requires + pkg.requires_pre + pkg.recommends + pkg.suggests + pkg.supplements + pkg.enhances): require = str(require).split()[0] real_require = require try: require = old_misnamed[pkg.name][real_require] except KeyError: pass requirement = all_provides.get(require) if (is_unversioned(real_require) and requirement and not (real_require.endswith('-doc') or python_versions.get(requirement) == {3}) and real_require not in NAME_NOTS and real_require != 'python-unversioned-command'): requirement_srpm_name = get_srpm_name(requirement) requirer_srpm_name = get_srpm_name(pkg) unversioned_requirers[requirement_srpm_name].add( requirer_srpm_name) # json_output: {srpm name: info} json_output = dict() for name in progressbar(by_srpm_name, 'Generating output'): pkgs = sorted(by_srpm_name[name]) r = json_output[name] = {} r['rpms'] = { format_rpm_name(p): { 'py_deps': {str(d): dep_versions[d] for d in rpm_pydeps[p]}, 'non_python_requirers': { 'build_time': sorted( get_srpm_names(build_requirers_of_pkg[p]) - by_srpm_name.keys()), 'run_time': sorted( get_srpm_names(requirers_of_pkg[p]) - by_srpm_name.keys()), }, 'almost_leaf': ( # not SRPM and is Python 2 and is not required by anything EXCEPT # sibling subpackages p.arch != 'src' and 2 in python_versions[p] and not get_srpm_names(build_requirers_of_pkg[p] | requirers_of_pkg[p]) - {name}), 'legacy_leaf': ( # not SRPM and is Python 2 and is not required by anything p.arch != 'src' and 2 in python_versions[p] and not get_srpm_names(build_requirers_of_pkg[p] | requirers_of_pkg[p])), 'arch': p.arch, } for p in pkgs } set_status(r, pkgs, python_versions) r['deps'] = sorted( set(srpm_names[d] for p in pkgs for d in deps_of_pkg.get(p, '') if srpm_names[d] != name)) r['build_deps'] = sorted( set(srpm_names[d] for d in build_deps_of_srpm.get(name, '') if srpm_names[d] != name)) if unversioned_requirers.get(name): r['unversioned_requirers'] = sorted( unversioned_requirers[name]) # add Bugzilla links if self.opts.fetch_bugzilla: bar = iter( progressbar(['connecting', 'tracker', 'individual'], 'Getting bugs')) next(bar) bz = bugzilla.RHBugzilla(BUGZILLA_URL) next(bar) include_fields = [ 'id', 'depends_on', 'blocks', 'component', 'status', 'resolution', 'last_change_time', 'short_desc' ] trackers = bz.getbugs(TRACKER_BUGS, include_fields=include_fields) all_ids = set(b for t in trackers for b in t.depends_on) next(bar) bugs = bz.getbugs(all_ids, include_fields=include_fields) bar.close() def bug_namegetter(bug): return '{bug.id} {bug.status} {bug.component}'.format(bug=bug) rank = [ 'NEW', 'ASSIGNED', 'POST', 'MODIFIED', 'ON_QA', 'VERIFIED', 'RELEASE_PENDING', 'CLOSED' ] def key(bug): return rank.index(bug.status), bug.last_change_time bugs = sorted(bugs, key=key) for bug in progressbar(bugs, 'Merging bugs', namegetter=bug_namegetter): r = json_output.get(bug.component, {}) bugs = r.setdefault('bugs', {}) entry = bugs.get(bug.id) if not entry: entry = { 'url': bug.weburl, 'short_desc': bug.short_desc, 'status': bug.status, 'resolution': bug.resolution, 'last_change': time.strftime('%Y-%m-%d %H:%M:%S', bug.last_change_time.timetuple()), 'trackers': [], } for tb in bug.blocks: alias = TRACKER_BUGS.get(tb) if alias: entry['trackers'].append(alias) bugs[bug.id] = entry # Print out output if self.opts.output: with open(self.opts.output, 'w') as f: json.dump(json_output, f, indent=2, sort_keys=True) else: json.dump(json_output, sys.stdout, indent=2, sort_keys=True) sys.stdout.flush() # Write out a groups.json if self.opts.repo_groups_file: output = { repo_name: { 'name': repo_name, 'packages': sorted(srpm_names) } for repo_name, srpm_names in repo_srpms.items() } with open(self.opts.repo_groups_file, 'w') as f: json.dump(output, f, indent=2, sort_keys=True)