def testing_statistics(): """ Calculate and display various testing statistics """ from datetime import timedelta from bodhi.model import PackageUpdate deltas = [] occurrences = {} accumulative = timedelta() for update in PackageUpdate.select(): for comment in update.comments: if comment.text == 'This update has been pushed to testing': for othercomment in update.comments: if othercomment.text == 'This update has been pushed to stable': delta = othercomment.timestamp - comment.timestamp deltas.append(delta) occurrences[delta.days] = occurrences.setdefault( delta.days, 0) + 1 accumulative += deltas[-1] break break deltas.sort() all = PackageUpdate.select().count() percentage = int(float(len(deltas)) / float(all) * 100) mode = sorted(occurrences.items(), cmp=lambda x, y: cmp(x[1], y[1]))[-1][0] print "%d out of %d updates went through testing (%d%%)" % ( len(deltas), all, percentage) print "mean = %d days" % (accumulative.days / len(deltas)) print "median = %d days" % deltas[len(deltas) / 2].days print "mode = %d days" % mode
def testing_statistics(): """ Calculate and display various testing statistics """ from datetime import timedelta from bodhi.model import PackageUpdate deltas = [] occurrences = {} accumulative = timedelta() for update in PackageUpdate.select(): for comment in update.comments: if comment.text == 'This update has been pushed to testing': for othercomment in update.comments: if othercomment.text == 'This update has been pushed to stable': delta = othercomment.timestamp - comment.timestamp deltas.append(delta) occurrences[delta.days] = occurrences.setdefault(delta.days, 0) + 1 accumulative += deltas[-1] break break deltas.sort() all = PackageUpdate.select().count() percentage = int(float(len(deltas)) / float(all) * 100) mode = sorted(occurrences.items(), cmp=lambda x, y: cmp(x[1], y[1]))[-1][0] print "%d out of %d updates went through testing (%d%%)" % (len(deltas), all, percentage) print "mean = %d days" % (accumulative.days / len(deltas)) print "median = %d days" % deltas[len(deltas) / 2].days print "mode = %d days" % mode
def nagmail(): """ Nag the submitters of updates based on a list of queries """ log.info("Starting nagmail job!") queries = [ ('old_testing', PackageUpdate.select( AND(PackageUpdate.q.status == 'testing', PackageUpdate.q.request == None)), lambda update: update.days_in_testing), ('old_pending', PackageUpdate.select( AND(PackageUpdate.q.status == 'pending', PackageUpdate.q.request == None)), lambda update: get_age_in_days(update.date_submitted)), ] oldname = None mail_admin = False #mail_proventesters = False for name, query, date in queries: for update in query: if date(update) > 14: if update.nagged: if update.nagged.has_key(name) and update.nagged[name]: if (datetime.utcnow() - update.nagged[name]).days < 7: continue # Only nag once a week at most nagged = update.nagged else: nagged = {} if update.critpath: if update.critpath_approved: continue else: oldname = name name = 'old_testing_critpath' mail_admin = True #mail_proventesters = True log.info("[%s] Nagging %s about %s" % (name, update.submitter, update.title)) mail.send(update.submitter, name, update) if mail_admin: mail.send_admin(name, update) mail_admin = False #if mail_proventesters: # mail.send(config.get('proventesters_email'), name, update) # mail_proventesters = False nagged[name] = datetime.utcnow() update.nagged = nagged if oldname: name = oldname oldname = None log.info("nagmail complete!")
def main(): load_config() print "Calculating F11 0day update metrics..." updates = { 'bugfix': [], 'security': [], 'enhancement': [], 'newpackage': [] } date = datetime(*time.strptime('06-09-2009', '%m-%d-%Y')[:-2]) f11 = Release.byName('F11') for update in PackageUpdate.select(PackageUpdate.q.releaseID == f11.id): for comment in update.comments: if comment.author == 'bodhi' and comment.timestamp < date and \ comment.text.startswith('This update has been pushed to stable'): updates[update.type].append(update.title) break pprint(updates) print '=' * 80 print 'F11 0day stats' print ' * %d security' % len(updates['security']) print ' * %d bugfixes' % len(updates['bugfix']) print ' * %d enhancements' % len(updates['enhancement']) print ' * %d newpackage' % len(updates['newpackage'])
def save_db(): ## Save each release and it's metrics releases = [] for release in Release.select(): rel = {} for attr in ('name', 'long_name', 'id_prefix', 'dist_tag', 'locked', 'metrics'): rel[attr] = getattr(release, attr) releases.append(rel) updates = [] all_updates = PackageUpdate.select() progress = ProgressBar(maxValue=all_updates.count()) for update in all_updates: data = {} data['title'] = update.title data['builds'] = [(build.package.name, build.nvr) for build in update.builds] data['date_submitted'] = update.date_submitted data['date_pushed'] = update.date_pushed data['date_modified'] = update.date_modified data['release'] = [update.release.name, update.release.long_name, update.release.id_prefix, update.release.dist_tag] data['submitter'] = update.submitter data['update_id'] = hasattr(update, 'update_id') and update.update_id or update.updateid data['type'] = update.type data['karma'] = update.karma data['cves'] = [cve.cve_id for cve in update.cves] data['bugs'] = [] for bug in update.bugs: data['bugs'].append([bug.bz_id, bug.title, bug.security]) if hasattr(bug, 'parent'): data['bugs'][-1].append(bug.parent) else: data['bugs'][-1].append(False) data['status'] = update.status data['pushed'] = update.pushed data['notes'] = update.notes data['request'] = update.request data['comments'] = [(c.timestamp, c.author, c.text, c.karma, c.anonymous) for c in update.comments] if hasattr(update, 'approved'): data['approved'] = update.approved else: data['approved'] = None updates.append(data) progress() # Save all buildroot overrides overrides = [] for override in BuildRootOverride.select(): try: overrides.append(override.__json__()) except: print("Removing stray override: %s" % override) override.destroySelf() dump = file('bodhi-pickledb-%s' % time.strftime("%y%m%d.%H%M"), 'w') pickle.dump({'updates': updates, 'releases': releases, 'overrides': overrides}, dump) dump.close()
def refresh(self): """ Refresh all of the metrics for all releases. For each release, initialize our metrics objects, and feed them every update for that release. Do the necessary calculations, and then save our metrics to the database in the Release.metrics PickleCol. """ log.info("Doing a hard refresh of our metrics data") metrics = {} updates = {} # {release: [updates,]} all_updates = list(PackageUpdate.select()) releases = list(Release.select()) for release in releases: updates[release.name] = [] for update in all_updates: updates[update.release.name].append(update) for release in releases: log.debug("Calculating metrics for %s" % release.name) self.init_metrics(release) for update in updates[release.name]: for metric in self.metrics: metric.update(update) for metric in self.metrics: metric.done() metrics[metric.__class__.__name__] = metric.get_data() release.metrics = metrics hub.commit() del all_updates del releases log.info("Metrics generation complete!")
def main(): load_config() __connection__ = hub = PackageHub("bodhi") if len(sys.argv) != 2: print "Usage: %s <release>" % sys.argv[0] sys.exit(1) try: release = Release.byName(sys.argv[1].upper()) except SQLObjectNotFound: print "Cannot find Release '%s'" % sys.argv[1] sys.exit(1) updates = PackageUpdate.select(PackageUpdate.q.releaseID == release.id) progress = ProgressBar(maxValue=updates.count()) print "Destroying all updates, comments, and bugs associated with %s" % release.name for update in updates: for comment in update.comments: comment.destroySelf() for build in update.builds: build.destroySelf() for bug in update.bugs: if len(bug.updates) == 1: bug.destroySelf() update.destroySelf() progress() release.destroySelf() hub.commit() print
def reset_date_pushed(status='testing'): """ Reset the date_pushed on all testing updates with the most recent bodhi comment that relates to it's current status. This needed to happen when a few batches of updates were pushed without a date_pushed field, so we had to recreate it based on bodhi's comments. """ from bodhi.model import PackageUpdate from sqlobject import AND for update in PackageUpdate.select( AND(PackageUpdate.q.date_pushed == None, PackageUpdate.q.status == status)): date = None for comment in update.comments: if comment.author == 'bodhi': if comment.text == 'This update has been pushed to %s' % update.status: if date and comment.timestamp < date: print "Skipping older push %s for %s" % ( comment.timestamp, update.title) else: date = comment.timestamp print "Setting %s to %s" % (update.title, comment.timestamp) update.date_pushed = date
def default(self, search, *args, **kw): results = set() search = search.strip() # Search name-version-release map(results.add, PackageUpdate.select( LIKE(PackageUpdate.q.title, '%%%s%%' % search), orderBy=PackageUpdate.q.date_submitted)) # Search bug numbers try: map(lambda bug: map(results.add, bug.updates), Bugzilla.select(Bugzilla.q.bz_id==int(search))) except ValueError: # can't convert search search to integer pass # Search CVEs if search.startswith('CVE') or search.startswith('CAN'): # Search bug titles for CVE, since that is how we track them now map(lambda bug: map(results.add, bug.updates), Bugzilla.select(LIKE(Bugzilla.q.title, '%%%s%%' % search))) # We still have some CVE objects lying around, so search them too map(lambda cve: map(results.add, cve.updates), CVE.select(CVE.q.cve_id==search)) # If there is only 1 result, then jump right to it num_items = len(results) if len(results) == 1: raise redirect(results.pop().get_url()) return dict(updates=list(results), num_items=num_items, title="%d Results Found" % num_items)
def approve_testing_updates(): """ Scan all testing updates and approve ones that have met the per-release testing requirements. https://fedoraproject.org/wiki/Package_update_acceptance_criteria """ log.info('Running approve_testing_updates job...') for update in PackageUpdate.select( AND(PackageUpdate.q.status == 'testing', PackageUpdate.q.request == None)): # If this release does not have any testing requirements, skip it if not update.release.mandatory_days_in_testing: continue # If this has already met testing requirements, skip it if update.met_testing_requirements: continue # If this is a critpath update, skip it, since they have their own # testing requirements, aside from spending time in testing. if update.critpath: continue if update.meets_testing_requirements: log.info('%s now meets testing requirements' % update.title) update.comment( config.get('testing_approval_msg') % update.days_in_testing, author='bodhi') log.info('approve_testing_updates job complete.')
def approve_testing_updates(): """ Scan all testing updates and approve ones that have met the per-release testing requirements. https://fedoraproject.org/wiki/Package_update_acceptance_criteria """ log.info('Running approve_testing_updates job...') for update in PackageUpdate.select( AND(PackageUpdate.q.status == 'testing', PackageUpdate.q.request == None)): # If this release does not have any testing requirements, skip it if not update.release.mandatory_days_in_testing: continue # If this has already met testing requirements, skip it if update.met_testing_requirements: continue # If this is a critpath update, skip it, since they have their own # testing requirements, aside from spending time in testing. if update.critpath: continue if update.meets_testing_requirements: log.info('%s now meets testing requirements' % update.title) update.comment(config.get('testing_approval_msg') % update.days_in_testing, author='bodhi') log.info('approve_testing_updates job complete.')
def get_security_updates(self, release): release = Release.select(Release.q.long_name==release)[0] return PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.type == 'security', PackageUpdate.q.status == 'testing', PackageUpdate.q.request == None))
def get_security_updates(self, release): release = Release.select(Release.q.long_name == release)[0] updates = PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.type == 'security', PackageUpdate.q.status == 'testing', PackageUpdate.q.request == None)) updates = self.sort_by_days_in_testing(updates) return updates
def save_db(): ## Save each release and it's metrics releases = [] for release in Release.select(): rel = {} for attr in ('name', 'long_name', 'id_prefix', 'dist_tag', 'locked', 'metrics'): rel[attr] = getattr(release, attr) releases.append(rel) updates = [] all_updates = PackageUpdate.select() progress = ProgressBar(maxValue=all_updates.count()) for update in all_updates: data = {} data['title'] = update.title data['builds'] = [(build.package.name, build.nvr) for build in update.builds] data['date_submitted'] = update.date_submitted data['date_pushed'] = update.date_pushed data['date_modified'] = update.date_modified data['release'] = [ update.release.name, update.release.long_name, update.release.id_prefix, update.release.dist_tag ] data['submitter'] = update.submitter data['update_id'] = hasattr( update, 'update_id') and update.update_id or update.updateid data['type'] = update.type data['karma'] = update.karma data['cves'] = [cve.cve_id for cve in update.cves] data['bugs'] = [] for bug in update.bugs: data['bugs'].append([bug.bz_id, bug.title, bug.security]) if hasattr(bug, 'parent'): data['bugs'][-1].append(bug.parent) else: data['bugs'][-1].append(False) data['status'] = update.status data['pushed'] = update.pushed data['notes'] = update.notes data['request'] = update.request data['comments'] = [(c.timestamp, c.author, c.text, c.karma, c.anonymous) for c in update.comments] if hasattr(update, 'approved'): data['approved'] = update.approved else: data['approved'] = None updates.append(data) progress() dump = file('bodhi-pickledb-%s' % time.strftime("%y%m%d.%H%M"), 'w') pickle.dump({'updates': updates, 'releases': releases}, dump) dump.close()
def save_db(): ## Save each release and it's metrics releases = [] for release in Release.select(): rel = {} for attr in ("name", "long_name", "id_prefix", "dist_tag", "locked", "metrics"): rel[attr] = getattr(release, attr) releases.append(rel) updates = [] all_updates = PackageUpdate.select() progress = ProgressBar(maxValue=all_updates.count()) for update in all_updates: data = {} data["title"] = update.title data["builds"] = [(build.package.name, build.nvr) for build in update.builds] data["date_submitted"] = update.date_submitted data["date_pushed"] = update.date_pushed data["date_modified"] = update.date_modified data["release"] = [ update.release.name, update.release.long_name, update.release.id_prefix, update.release.dist_tag, ] data["submitter"] = update.submitter data["update_id"] = hasattr(update, "update_id") and update.update_id or update.updateid data["type"] = update.type data["karma"] = update.karma data["cves"] = [cve.cve_id for cve in update.cves] data["bugs"] = [] for bug in update.bugs: data["bugs"].append([bug.bz_id, bug.title, bug.security]) if hasattr(bug, "parent"): data["bugs"][-1].append(bug.parent) else: data["bugs"][-1].append(False) data["status"] = update.status data["pushed"] = update.pushed data["notes"] = update.notes data["request"] = update.request data["comments"] = [(c.timestamp, c.author, c.text, c.karma, c.anonymous) for c in update.comments] if hasattr(update, "approved"): data["approved"] = update.approved else: data["approved"] = None updates.append(data) progress() dump = file("bodhi-pickledb-%s" % time.strftime("%y%m%d.%H%M"), "w") pickle.dump({"updates": updates, "releases": releases}, dump) dump.close()
def get_unapproved_critpath_updates(self, release): release = Release.select(Release.q.long_name==release)[0] updates = [] for update in PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.status != 'stable', PackageUpdate.q.status != 'obsolete', PackageUpdate.q.request == None), orderBy=PackageUpdate.q.date_submitted).reversed(): if update.critpath and not update.critpath_approved: updates.append(update) return updates
def get_unapproved_critpath_updates(self, release): release = Release.select(Release.q.long_name == release)[0] updates = [] for update in PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.status == 'testing', PackageUpdate.q.request == None), orderBy=PackageUpdate.q.date_submitted).reversed(): if update.critpath and not update.critpath_approved: updates.append(update) updates = self.sort_by_days_in_testing(updates) return updates
def get_critpath_updates(self, release=None, unapproved=None): i = 0 entries = [] base = config.get('base_address') title = 'Latest Critical Path Updates' query = [PackageUpdate.q.status != 'obsolete'] if release: try: release = Release.byName(release) except SQLObjectNotFound: return dict(title = '%s release not found' % release, entries=[]) releases = [release] title = title + ' for %s' % release.long_name else: releases = Release.select() if unapproved: query.append(PackageUpdate.q.status != 'stable') for update in PackageUpdate.select( AND(OR(*[PackageUpdate.q.releaseID == release.id for release in releases]), *query), orderBy=PackageUpdate.q.date_submitted).reversed(): delta = datetime.utcnow() - update.date_submitted if delta and delta.days > config.get('feeds.num_days_to_show'): if len(entries) >= config.get('feeds.max_entries'): break if update.critpath: if unapproved: if update.critpath_approved: continue entries.append({ 'id' : base + url(update.get_url()), 'summary' : update.notes, 'link' : base + url(update.get_url()), 'published' : update.date_submitted, 'updated' : update.date_submitted, 'title' : update.title, }) i += 1 return dict( title = title, subtitle = "", link = config.get('base_address') + url('/'), entries = entries )
def clean_testing_builds(untag=False): koji = get_session() for release in Release.select(): stable_builds = koji.listTagged(release.stable_tag, latest=True) stable_nvrs = [build["nvr"] for build in stable_builds] print "Fetched %d builds tagged with %s" % (len(stable_builds), release.stable_tag) testing_builds = koji.listTagged(release.testing_tag, latest=True) print "Fetched %d builds tagged with %s" % (len(testing_builds), release.testing_tag) testing_nvrs = [build["nvr"] for build in testing_builds] for testing_build in testing_builds: for build in testing_builds: compare_builds(testing_build, build, untag, release.testing_tag) for build in stable_builds: compare_builds(testing_build, build, untag, release.testing_tag) # Find testing updates that aren't in the list of latest builds for update in PackageUpdate.select( AND( PackageUpdate.q.releaseID == release.id, PackageUpdate.q.status == "testing", PackageUpdate.q.request == None, ) ): for build in update.builds: if build.nvr not in testing_nvrs: latest_testing = None latest_stable = None for testing in testing_nvrs: if testing.startswith(build.package.name + "-"): latest_testing = testing break for stable in stable_nvrs: if stable.startswith(build.package.name + "-"): latest_stable = stable break if latest_testing: koji_build = koji.getBuild(build.nvr) latest_build = koji.getBuild(latest_testing) if rpm.labelCompare(build_evr(koji_build), build_evr(latest_build)) < 0: print "%s in testing, latest_testing = %s, latest_stable = %s" % ( update.title, latest_testing, latest_stable, ) if untag: print "Obsoleting %s" % update.title update.obsolete(newer=latest_testing)
def get_critpath_updates(self, release=None, unapproved=None): i = 0 entries = [] base = config.get('base_address') title = 'Latest Critical Path Updates' query = [PackageUpdate.q.status != 'obsolete'] if release: try: release = Release.byName(release) except SQLObjectNotFound: return dict(title='%s release not found' % release, entries=[]) releases = [release] title = title + ' for %s' % release.long_name else: releases = Release.select() if unapproved: query.append(PackageUpdate.q.status != 'stable') for update in PackageUpdate.select( AND( OR(*[ PackageUpdate.q.releaseID == release.id for release in releases ]), *query), orderBy=PackageUpdate.q.date_submitted).reversed(): delta = datetime.utcnow() - update.date_submitted if delta and delta.days > config.get('feeds.num_days_to_show'): if len(entries) >= config.get('feeds.max_entries'): break if update.critpath: if unapproved: if update.critpath_approved: continue entries.append({ 'id': base + url(update.get_url()), 'summary': update.notes, 'link': base + url(update.get_url()), 'published': update.date_submitted, 'updated': update.date_submitted, 'title': update.title, }) i += 1 return dict(title=title, subtitle="", link=config.get('base_address') + url('/'), entries=entries)
def main(): unstable = subprocess.Popen( 'grep "\[Fedora Update\] \[unstable\]" bodhi.logs', stdout=subprocess.PIPE, shell=True) out, err = unstable.communicate() (unstable_updates, unstable_critpath, unstable_deltas, unstable_accum, unstable_occur) = parse_output(out) stable = subprocess.Popen( 'grep "\[Fedora Update\] \[stablekarma\]" bodhi.logs', stdout=subprocess.PIPE, shell=True) out, err = stable.communicate() (stable_updates, stable_critpath, stable_deltas, stable_accum, stable_occur) = parse_output(out) for release in Release.select(): print '\n' + header(release.long_name) num_updates = PackageUpdate.select( PackageUpdate.q.releaseID == release.id).count() num_stable = len(stable_updates[release.name]) num_unstable = len(unstable_updates[release.name]) num_testing = len(unstable_deltas) + len(stable_deltas) print " * %d updates automatically unpushed due to karma (%0.2f%%)" % ( num_unstable, float(num_unstable) / num_updates * 100) print " * %d of which were critical path updates" % ( unstable_critpath[release.name]) print " * %d updates automatically pushed due to karma (%0.2f%%)" % ( num_stable, float(num_stable) / num_updates * 100) print " * %d of which were critical path updates" % ( stable_critpath[release.name]) print " * Time spent in testing of updates that were pushed by karma:" print " * mean = %d days" % (stable_accum.days / len(stable_deltas)) print " * median = %d days" % stable_deltas[len(stable_deltas) / 2].days print " * mode = %d days" % sorted(stable_occur.items(), key=itemgetter(1))[-1][0] print " * Time spent in testing of updates that were unpushed by karma:" print " * mean = %d days" % (unstable_accum.days / len(unstable_deltas)) print " * median = %d days" % unstable_deltas[len(unstable_deltas) / 2].days print " * mode = %d days" % sorted(unstable_occur.items(), key=itemgetter(1))[-1][0]
def clean_testing_builds(untag=False): koji = get_session() for release in Release.select(): stable_builds = koji.listTagged(release.stable_tag, latest=True) stable_nvrs = [build['nvr'] for build in stable_builds] print "Fetched %d builds tagged with %s" % (len(stable_builds), release.stable_tag) testing_builds = koji.listTagged(release.testing_tag, latest=True) print "Fetched %d builds tagged with %s" % (len(testing_builds), release.testing_tag) testing_nvrs = [build['nvr'] for build in testing_builds] for testing_build in testing_builds: for build in testing_builds: compare_builds(testing_build, build, untag, release.testing_tag) for build in stable_builds: compare_builds(testing_build, build, untag, release.testing_tag) # Find testing updates that aren't in the list of latest builds for update in PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.status == 'testing', PackageUpdate.q.request == None)): for build in update.builds: if build.nvr not in testing_nvrs: latest_testing = None latest_stable = None for testing in testing_nvrs: if testing.startswith(build.package.name + '-'): latest_testing = testing break for stable in stable_nvrs: if stable.startswith(build.package.name + '-'): latest_stable = stable break if latest_testing: koji_build = koji.getBuild(build.nvr) latest_build = koji.getBuild(latest_testing) if rpm.labelCompare(build_evr(koji_build), build_evr(latest_build)) < 0: print "%s in testing, latest_testing = %s, latest_stable = %s" % ( update.title, latest_testing, latest_stable) if untag: print "Obsoleting %s" % update.title update.obsolete(newer=latest_testing)
def main(): load_config() print "Calculating F11 0day update metrics..." updates = {'bugfix': [], 'security': [], 'enhancement': [], 'newpackage': []} date = datetime(*time.strptime('06-09-2009', '%m-%d-%Y')[:-2]) f11 = Release.byName('F11') for update in PackageUpdate.select(PackageUpdate.q.releaseID==f11.id): for comment in update.comments: if comment.author == 'bodhi' and comment.timestamp < date and \ comment.text.startswith('This update has been pushed to stable'): updates[update.type].append(update.title) break pprint(updates) print '=' * 80 print 'F11 0day stats' print ' * %d security' % len(updates['security']) print ' * %d bugfixes' % len(updates['bugfix']) print ' * %d enhancements' % len(updates['enhancement']) print ' * %d newpackage' % len(updates['newpackage'])
def main(): unstable = subprocess.Popen('grep "\[Fedora Update\] \[unstable\]" bodhi.logs', stdout=subprocess.PIPE, shell=True) out, err = unstable.communicate() (unstable_updates, unstable_critpath, unstable_deltas, unstable_accum, unstable_occur) = parse_output(out) stable = subprocess.Popen('grep "\[Fedora Update\] \[stablekarma\]" bodhi.logs', stdout=subprocess.PIPE, shell=True) out, err = stable.communicate() (stable_updates, stable_critpath, stable_deltas, stable_accum, stable_occur) = parse_output(out) for release in Release.select(): print '\n' + header(release.long_name) num_updates = PackageUpdate.select( PackageUpdate.q.releaseID==release.id).count() num_stable = len(stable_updates[release.name]) num_unstable = len(unstable_updates[release.name]) num_testing = len(unstable_deltas) + len(stable_deltas) print " * %d updates automatically unpushed due to karma (%0.2f%%)" % ( num_unstable, float(num_unstable) / num_updates * 100) print " * %d of which were critical path updates" % ( unstable_critpath[release.name]) print " * %d updates automatically pushed due to karma (%0.2f%%)" % ( num_stable, float(num_stable) / num_updates * 100) print " * %d of which were critical path updates" % ( stable_critpath[release.name]) print " * Time spent in testing of updates that were pushed by karma:" print " * mean = %d days" % (stable_accum.days / len(stable_deltas)) print " * median = %d days" % stable_deltas[len(stable_deltas)/2].days print " * mode = %d days" % sorted(stable_occur.items(), key=itemgetter(1))[-1][0] print " * Time spent in testing of updates that were unpushed by karma:" print " * mean = %d days" % (unstable_accum.days / len(unstable_deltas)) print " * median = %d days" % unstable_deltas[len(unstable_deltas)/2].days print " * mode = %d days" % sorted(unstable_occur.items(), key=itemgetter(1))[-1][0]
def reset_date_pushed(status='testing'): """ Reset the date_pushed on all testing updates with the most recent bodhi comment that relates to it's current status. This needed to happen when a few batches of updates were pushed without a date_pushed field, so we had to recreate it based on bodhi's comments. """ from bodhi.model import PackageUpdate from sqlobject import AND for update in PackageUpdate.select(AND(PackageUpdate.q.date_pushed==None, PackageUpdate.q.status==status)): date = None for comment in update.comments: if comment.author == 'bodhi': if comment.text == 'This update has been pushed to %s' % update.status: if date and comment.timestamp < date: print "Skipping older push %s for %s" % (comment.timestamp, update.title) else: date = comment.timestamp print "Setting %s to %s" % (update.title, comment.timestamp) update.date_pushed = date
def push(self): """ List updates tagged with a push/unpush/move request """ updates = [] resume = False mash = self._current_mash() if not mash: flash_log("A masher exception has occured.") return dict(updates=[], resume=False) if mash['mashing']: flash_log('The masher is currently pushing updates') else: for update in mash.get('updates', []): try: updates.append(PackageUpdate.byTitle(update)) except SQLObjectNotFound: log.warning("Cannot find update %s in push queue" % update) if updates: flash_log('There is an updates push ready to be resumed') resume = True else: # Get a list of all updates with a request that aren't # unapproved security updates, or for a locked release requests = PackageUpdate.select( PackageUpdate.q.request != None) # Come F13+, bodhi will not have locked releases. It will # implement the 'No Frozen Rawhide' proposal, and treat 'locked' # releases as pending. #requests = filter(lambda update: not update.release.locked, # PackageUpdate.select( # PackageUpdate.q.request != None)) for update in requests: # Disable security approval requirement #if update.type == 'security' and not update.approved: # continue updates.append(update) return dict(updates=updates, resume=resume)
def push(self): """ List updates tagged with a push/unpush/move request """ updates = [] resume = False mash = self._current_mash() if not mash: flash_log("A masher exception has occured.") return dict(updates=[], resume=False) if mash['mashing']: flash_log('The masher is currently pushing updates') else: for update in mash.get('updates', []): try: updates.append(PackageUpdate.byTitle(update)) except SQLObjectNotFound: log.warning("Cannot find update %s in push queue" % update) if updates: flash_log('There is an updates push ready to be resumed') resume = True else: # Get a list of all updates with a request that aren't # unapproved security updates, or for a locked release requests = PackageUpdate.select(PackageUpdate.q.request != None) # Come F13+, bodhi will not have locked releases. It will # implement the 'No Frozen Rawhide' proposal, and treat 'locked' # releases as pending. #requests = filter(lambda update: not update.release.locked, # PackageUpdate.select( # PackageUpdate.q.request != None)) for update in requests: # Disable security approval requirement #if update.type == 'security' and not update.approved: # continue updates.append(update) return dict(updates=updates, resume=resume)
def default(self, search, *args, **kw): results = set() search = search.strip() # Search name-version-release map( results.add, PackageUpdate.select(LIKE(PackageUpdate.q.title, '%%%s%%' % search), orderBy=PackageUpdate.q.date_submitted)) # Search bug numbers try: map(lambda bug: map(results.add, bug.updates), Bugzilla.select(Bugzilla.q.bz_id == int(search))) except ValueError: # can't convert search search to integer pass # Search CVEs if search.startswith('CVE') or search.startswith('CAN'): # Search bug titles for CVE, since that is how we track them now map(lambda bug: map(results.add, bug.updates), Bugzilla.select(LIKE(Bugzilla.q.title, '%%%s%%' % search))) # We still have some CVE objects lying around, so search them too map(lambda cve: map(results.add, cve.updates), CVE.select(CVE.q.cve_id == search)) # If there is only 1 result, then jump right to it num_items = len(results) if len(results) == 1: raise redirect(results.pop().get_url()) return dict(updates=list(results), num_items=num_items, title="%d Results Found" % num_items)
def main(): load_config() hub.begin() print "Finding updates with duplicate IDs..." if os.path.exists('dupes.pickle'): out = file('dupes.pickle') dupes = cPickle.load(out) out.close() highest_fedora = int(file('highest_fedora').read()) highest_epel = int(file('highest_epel').read()) else: dupes = set() highest_fedora = 0 highest_epel = 0 for update in PackageUpdate.select(PackageUpdate.q.updateid!=None): if '-2010-' in update.updateid: if update.release.id_prefix == 'FEDORA': if update.updateid_int > highest_fedora: highest_fedora = update.updateid_int else: if update.updateid_int > highest_epel: highest_epel = update.updateid_int updates = PackageUpdate.select( AND(PackageUpdate.q.updateid == update.updateid, PackageUpdate.q.title != update.title)) if updates.count(): # Maybe TODO?: ensure these dupes have a date_pushed less tahn update?! # this way, the new ID is based on the oldest update for u in updates: dupes.add(u.title) out = file('dupes.pickle', 'w') cPickle.dump(dupes, out) out.close() print "Wrote dupes.pickle" file('highest_fedora', 'w').write(str(highest_fedora)) file('highest_epel', 'w').write(str(highest_epel)) # verify what we really found the highest IDs assert PackageUpdate.select(PackageUpdate.q.updateid=='FEDORA-2010-%d' % (highest_fedora + 1)).count() == 0 assert PackageUpdate.select(PackageUpdate.q.updateid=='FEDORA-EPEL-2010-%d' % (highest_epel + 1)).count() == 0 # Should be 740? print "%d dupes" % len(dupes) print "Highest FEDORA ID:", highest_fedora print "Highest FEDORA-EPEL ID:", highest_epel # Reassign the update IDs on all of our dupes for dupe in dupes: up = PackageUpdate.byTitle(dupe) #print "%s *was* %s" % (up.title, up.updateid) up.updateid = None # TODO: save & restore this value after new id assignment?! #up.date_pushed = None # Tweak the date_pushed to on the updates with the highest IDs PackageUpdate.select(PackageUpdate.q.updateid=='FEDORA-2010-%d' % highest_fedora).date_pushed = datetime.now() PackageUpdate.select(PackageUpdate.q.updateid=='FEDORA-EPEL-2010-%d' % highest_epel).date_pushed = datetime.now() #hub.commit() for dupe in dupes: up = PackageUpdate.byTitle(dupe) up.assign_id() ups = PackageUpdate.select(PackageUpdate.q.updateid == up.updateid) if ups.count() == 1: print "Still a dupe!!" for update in ups: if update.title == up.title: continue else: if update.title in dupes: print "%s in dupes, yet shares an updateid %s" % ( update.title, update.updateid) else: print "%s is not in dupes, but dupes %s" % ( update.title, updateid) print "Checking to ensure we have no more dupes..." dupes = set() for update in PackageUpdate.select(PackageUpdate.q.updateid != None): updates = PackageUpdate.select( AND(PackageUpdate.q.updateid == update.updateid, PackageUpdate.q.title != update.title)) if updates.count(): dupes.add(update.title) print "%d dupes (should be 0)" % len(dupes)
def main(): load_config() stats = {} # {release: {'stat': ...}} feedback = 0 # total number of updates that received feedback karma = defaultdict(int) # {username: # of karma submissions} num_updates = PackageUpdate.select().count() proventesters = set() for release in Release.select(): print header(release.long_name) updates = PackageUpdate.select(PackageUpdate.q.releaseID==release.id) stats[release.name] = { 'num_updates': updates.count(), 'num_tested': 0, 'num_tested_without_karma': 0, 'num_feedback': 0, 'num_anon_feedback': 0, 'num_critpath': 0, 'num_critpath_approved': 0, 'num_critpath_unapproved': 0, 'num_stablekarma': 0, 'num_testingtime': 0, 'critpath_without_karma': set(), 'conflicted_proventesters': [], 'critpath_positive_karma_including_proventesters': [], 'critpath_positive_karma_negative_proventesters': [], 'stable_with_negative_karma': PackageUpdate.select( AND(PackageUpdate.q.releaseID==release.id, PackageUpdate.q.status=='stable', PackageUpdate.q.karma < 0)).count(), 'bugs': set(), 'karma': defaultdict(int), 'deltas': [], 'occurrences': {}, 'accumulative': timedelta(), 'packages': defaultdict(int), 'proventesters': set(), 'proventesters_1': 0, 'proventesters_0': 0, 'proventesters_-1': 0, # for tracking number of types of karma '1': 0, '0': 0, '-1': 0, } data = stats[release.name] for status in statuses: data['num_%s' % status] = PackageUpdate.select(AND( PackageUpdate.q.releaseID==release.id, PackageUpdate.q.status==status)).count() for type in types: data['num_%s' % type] = PackageUpdate.select(AND( PackageUpdate.q.releaseID==release.id, PackageUpdate.q.type==type)).count() for update in release.updates: for build in update.builds: data['packages'][build.package] += 1 for bug in update.bugs: data['bugs'].add(bug.bz_id) feedback_done = False testingtime_done = False for comment in update.comments: if comment.author_name in ('autoqa', 'taskotron'): continue # Track the # of +1's, -1's, and +0's. if comment.author_name != 'bodhi': data[str(comment.karma)] += 1 if comment.author_group == 'proventesters': data['proventesters'].add(comment.author_name) data['proventesters_%d' % comment.karma] += 1 if comment.text == 'This update has reached the stable karma threshold and will be pushed to the stable updates repository': data['num_stablekarma'] += 1 elif comment.text and comment.text.endswith('days in testing and can be pushed to stable now if the maintainer wishes'): data['num_testingtime'] += 1 # For figuring out if an update has received feedback or not if not feedback_done: if (not comment.author.startswith('bodhi') and comment.karma != 0 and not comment.anonymous): data['num_feedback'] += 1 # per-release tracking of feedback feedback += 1 # total number of updates that have received feedback feedback_done = True # so we don't run this for each comment # Tracking per-author karma & anonymous feedback if not comment.author.startswith('bodhi'): if comment.anonymous: # @@: should we track anon +0 comments as "feedback"? if comment.karma != 0: data['num_anon_feedback'] += 1 else: author = comment.author_name data['karma'][author] += 1 karma[author] += 1 if (not testingtime_done and comment.text == 'This update has been pushed to testing'): for othercomment in update.comments: if othercomment.text == 'This update has been pushed to stable': delta = othercomment.timestamp - comment.timestamp data['deltas'].append(delta) data['occurrences'][delta.days] = \ data['occurrences'].setdefault( delta.days, 0) + 1 data['accumulative'] += delta testingtime_done = True break if update.critpath: if update.critpath_approved or update.status == 'stable': data['num_critpath_approved'] += 1 else: if status in ('testing', 'pending'): data['num_critpath_unapproved'] += 1 data['num_critpath'] += 1 #if not feedback_done: if update.status == 'stable' and update.karma == 0: data['critpath_without_karma'].add(update) # Proventester metrics proventester_karma = defaultdict(int) # {username: karma} positive_proventesters = 0 negative_proventesters = 0 for comment in update.comments: if comment.author_group == 'proventesters': proventester_karma[comment.author_name] += comment.karma for _karma in proventester_karma.values(): if _karma > 0: positive_proventesters += 1 elif _karma < 0: negative_proventesters += 1 # Conflicting proventesters if positive_proventesters and negative_proventesters: data['conflicted_proventesters'] += [short_url(update)] # Track updates with overall positive karma, including positive # karma from a proventester if update.karma > 0 and positive_proventesters: data['critpath_positive_karma_including_proventesters'] += [short_url(update)] # Track updates with overall positive karma, including negative # karma from a proventester if update.karma > 0 and negative_proventesters: data['critpath_positive_karma_negative_proventesters'] += [short_url(update)] if testingtime_done: data['num_tested'] += 1 if not feedback_done: data['num_tested_without_karma'] += 1 data['deltas'].sort() print " * %d updates" % data['num_updates'] print " * %d packages updated" % (len(data['packages'])) for status in statuses: print " * %d %s updates" % (data['num_%s' % status], status) for type in types: print " * %d %s updates (%0.2f%%)" % (data['num_%s' % type], type, float(data['num_%s' % type]) / data['num_updates'] * 100) print " * %d bugs resolved" % len(data['bugs']) print " * %d critical path updates (%0.2f%%)" % (data['num_critpath'], float(data['num_critpath']) / data['num_updates'] * 100) print " * %d approved critical path updates" % ( data['num_critpath_approved']) print " * %d unapproved critical path updates" % ( data['num_critpath_unapproved']) print " * %d updates received feedback (%0.2f%%)" % ( data['num_feedback'], (float(data['num_feedback']) / data['num_updates'] * 100)) print " * %d +0 comments" % data['0'] print " * %d +1 comments" % data['1'] print " * %d -1 comments" % data['-1'] print " * %d unique authenticated karma submitters" % ( len(data['karma'])) print " * %d proventesters" % len(data['proventesters']) print " * %d +1's from proventesters" % data['proventesters_1'] print " * %d -1's from proventesters" % data['proventesters_-1'] if data['num_critpath']: print " * %d critpath updates with conflicting proventesters (%0.2f%% of critpath)" % (len(data['conflicted_proventesters']), float(len(data['conflicted_proventesters'])) / data['num_critpath'] * 100) for u in data['conflicted_proventesters']: print " * " + u print " * %d critpath updates with positive karma and negative proventester feedback (%0.2f%% of critpath)" % (len(data['critpath_positive_karma_negative_proventesters']), float(len(data['critpath_positive_karma_negative_proventesters'])) / data['num_critpath'] * 100) for u in data['critpath_positive_karma_negative_proventesters']: print " * " + u print " * %d critpath updates with positive karma and positive proventester feedback (%0.2f%% of critpath)" % (len(data['critpath_positive_karma_including_proventesters']), float(len(data['critpath_positive_karma_including_proventesters'])) / data['num_critpath'] * 100) print " * %d anonymous users gave feedback (%0.2f%%)" % ( data['num_anon_feedback'], float(data['num_anon_feedback']) / (data['num_anon_feedback'] + sum(data['karma'].values())) * 100) # This does not take into account updates that reach stablekarma before being pushed to testing! # print " * %d out of %d stable updates went through testing (%0.2f%%)" %( # data['num_tested'], data['num_stable'], # float(data['num_tested']) / data['num_stable'] * 100) print " * %d updates reached the stable karma threshold (%0.2f%%)" % ( data['num_stablekarma'], float(data['num_stablekarma']) / data['num_stable'] * 100) print " * %d updates reached the minimum time in testing threshold (%0.2f%%)" % ( data['num_testingtime'], float(data['num_testingtime']) / data['num_stable'] * 100) print " * %d went from testing to stable *without* karma (%0.2f%%)" % ( data['num_tested_without_karma'], float(data['num_tested_without_karma']) / data['num_tested'] * 100) print " * %d updates were pushed to stable with negative karma (%0.2f%%)" % ( data['stable_with_negative_karma'], float(data['stable_with_negative_karma']) / data['num_stable'] * 100) print " * %d critical path updates pushed to stable *without* karma" % ( len(data['critpath_without_karma'])) #for update in data['critpath_without_karma']: # print " * %s submitted by %s" % (update.title, update.submitter) print " * Time spent in testing:" print " * mean = %d days" % (data['accumulative'].days / len(data['deltas'])) print " * median = %d days" % ( data['deltas'][len(data['deltas']) / 2].days) print " * mode = %d days" % ( sorted(data['occurrences'].items(), key=itemgetter(1))[-1][0]) #for package in sorted(data['packages'].items(), key=itemgetter(1), reverse=True): # print " * %s: %d" % (package[0].name, package[1]) print print print "Out of %d total updates, %d received feedback (%0.2f%%)" % ( num_updates, feedback, (float(feedback) / num_updates * 100)) print "Out of %d total unique commenters, the top 50 are:" % ( len(karma)) for submitter in sorted(karma.iteritems(), key=itemgetter(1), reverse=True)[:50]: print " * %s (%d)" % (submitter[0], submitter[1])
def main(): load_config() hub.begin() print "Finding updates with duplicate IDs..." if os.path.exists('dupes.pickle'): out = file('dupes.pickle') dupes = cPickle.load(out) out.close() highest_fedora = int(file('highest_fedora').read()) highest_epel = int(file('highest_epel').read()) else: dupes = set() highest_fedora = 0 highest_epel = 0 for update in PackageUpdate.select(PackageUpdate.q.updateid != None): if '-2010-' in update.updateid: if update.release.id_prefix == 'FEDORA': if update.updateid_int > highest_fedora: highest_fedora = update.updateid_int else: if update.updateid_int > highest_epel: highest_epel = update.updateid_int updates = PackageUpdate.select( AND(PackageUpdate.q.updateid == update.updateid, PackageUpdate.q.title != update.title)) if updates.count(): # Maybe TODO?: ensure these dupes have a date_pushed less tahn update?! # this way, the new ID is based on the oldest update for u in updates: dupes.add(u.title) out = file('dupes.pickle', 'w') cPickle.dump(dupes, out) out.close() print "Wrote dupes.pickle" file('highest_fedora', 'w').write(str(highest_fedora)) file('highest_epel', 'w').write(str(highest_epel)) # verify what we really found the highest IDs assert PackageUpdate.select(PackageUpdate.q.updateid == 'FEDORA-2010-%d' % (highest_fedora + 1)).count() == 0 assert PackageUpdate.select( PackageUpdate.q.updateid == 'FEDORA-EPEL-2010-%d' % (highest_epel + 1)).count() == 0 # Should be 740? print "%d dupes" % len(dupes) print "Highest FEDORA ID:", highest_fedora print "Highest FEDORA-EPEL ID:", highest_epel # Reassign the update IDs on all of our dupes for dupe in dupes: up = PackageUpdate.byTitle(dupe) #print "%s *was* %s" % (up.title, up.updateid) up.updateid = None # TODO: save & restore this value after new id assignment?! #up.date_pushed = None # Tweak the date_pushed to on the updates with the highest IDs PackageUpdate.select(PackageUpdate.q.updateid == 'FEDORA-2010-%d' % highest_fedora).date_pushed = datetime.now() PackageUpdate.select(PackageUpdate.q.updateid == 'FEDORA-EPEL-2010-%d' % highest_epel).date_pushed = datetime.now() #hub.commit() for dupe in dupes: up = PackageUpdate.byTitle(dupe) up.assign_id() ups = PackageUpdate.select(PackageUpdate.q.updateid == up.updateid) if ups.count() == 1: print "Still a dupe!!" for update in ups: if update.title == up.title: continue else: if update.title in dupes: print "%s in dupes, yet shares an updateid %s" % ( update.title, update.updateid) else: print "%s is not in dupes, but dupes %s" % ( update.title, updateid) print "Checking to ensure we have no more dupes..." dupes = set() for update in PackageUpdate.select(PackageUpdate.q.updateid != None): updates = PackageUpdate.select( AND(PackageUpdate.q.updateid == update.updateid, PackageUpdate.q.title != update.title)) if updates.count(): dupes.add(update.title) print "%d dupes (should be 0)" % len(dupes)
""" conffile = tempfile.mktemp() fd = open(conffile, 'w') fd.write(confheader) repo = repo_config % { 'testrepo': self.testrepo_dir, 'rel': release.repodir, 'arch': arch.name, 'testing': testing and '1' or '0', 'final': self.final and '1' or '0' } fd.write(repo) fd.close() return conffile # # Main method used for testing purposes # if __name__ == '__foo__': from turbogears.database import PackageHub hub = PackageHub("bodhi") __connection__ = hub log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) closure = TestRepoClosure(PackageUpdate.select()) closure.run()
this release supports """ conffile = tempfile.mktemp() fd = open(conffile, 'w') fd.write(confheader) repo = repo_config % { 'testrepo' : self.testrepo_dir, 'rel' : release.repodir, 'arch' : arch.name, 'testing' : testing and '1' or '0', 'final' : self.final and '1' or '0' } fd.write(repo) fd.close() return conffile # # Main method used for testing purposes # if __name__ == '__foo__': from turbogears.database import PackageHub hub = PackageHub("bodhi") __connection__ = hub log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) closure = TestRepoClosure(PackageUpdate.select()) closure.run()
def get_feed_data(self, release=None, type=None, status=None, comments=False, submitter=None, builds=None, user=None, package=None, critpath=False, unapproved=None, *args, **kw): query = [] entries = [] date = lambda update: update.date_pushed order = PackageUpdate.q.date_pushed title = [] critpath = critpath in (True, 'True', 'true') unapproved = unapproved in (True, 'True', 'true') if critpath: return self.get_critpath_updates(release=release, unapproved=unapproved) if comments: return self.get_latest_comments(user=user) if package: return self.get_package_updates(package, release) if release: try: rel = Release.byName(release.upper()) except SQLObjectNotFound: return dict(title = '%s not found' % release, entries=[]) query.append(PackageUpdate.q.releaseID == rel.id) title.append(rel.long_name) if type: query.append(PackageUpdate.q.type == type) title.append(type.title()) if status: query.append(PackageUpdate.q.status == status) if status == 'pending': date = lambda update: update.date_submitted order = PackageUpdate.q.date_submitted else: # Let's only show pushed testing/stable updates query.append(PackageUpdate.q.pushed == True) title.append(status.title()) else: query.append(PackageUpdate.q.pushed == True) if submitter: query.append(PackageUpdate.q.submitter == submitter) title.append("submitted by %s" % submitter) if builds: query.append(PackageUpdate.q.builds == builds) title.append("for %s" % builds) updates = PackageUpdate.select(AND(*query), orderBy=order).reversed() for update in updates: delta = datetime.utcnow() - update.date_submitted if delta and delta.days > config.get('feeds.num_days_to_show'): if len(entries) >= config.get('feeds.max_entries'): break entries.append({ 'id' : config.get('base_address') + url(update.get_url()), 'summary' : update.notes, 'published' : date(update), 'link' : config.get('base_address') + url(update.get_url()), 'title' : "%s %sUpdate: %s" % (update.release.long_name, update.type == 'security' and 'Security ' or '', update.title) }) if len(update.bugs): bugs = "<b>Resolved Bugs</b><br/>" for bug in update.bugs: bugs += "<a href=%s>%d</a> - %s<br/>" % (bug.get_url(), bug.bz_id, bug.title) entries[-1]['summary'] = "%s<br/>%s" % (bugs[:-2], entries[-1]['summary']) title.append('Updates') return dict( title = ' '.join(title), subtitle = "", link = config.get('base_address') + url('/'), entries = entries )
def main(): load_config() __connection__ = hub = PackageHub("bodhi") koji = get_session() tasks = [] broke = set() # Clean up any stray pending tags for release in Release.select(): print "Finding all pending-testing builds..." if release.name.startswith('EL'): continue tag = release.pending_testing_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable', 'obsolete'): print "%s %s" % (nvr, update.status) if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: print "Can't find build for %s" % nvr if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) tag = release.pending_stable_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('pending', 'obsolete', 'stable'): print "%s %s" % (nvr, update.status) if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: print "Can't find build for %s" % nvr if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) # Check for testing updates that aren't tagged properly for update in PackageUpdate.select(PackageUpdate.q.status == 'testing'): dest_tag = update.release.testing_tag for build in update.builds: tags = [tag['name'] for tag in koji.listTags(build=build.nvr)] if dest_tag not in tags: print "%s marked as testing, but tagged with %s" % (build.nvr, tags) if '--fix' in sys.argv: broke.add((tags[0], dest_tag, build.nvr)) # Check all candidate updates to see if they are in a different bodhi state for release in Release.select(): tag = release.candidate_tag tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable'): print "%s %s but tagged as %s" % (nvr, update.status, tag) if '--fix' in sys.argv: dest = release.testing_tag if update.status == 'stable': dest = release.stable_tag elif update.status == 'obsolete': dest = release.candidate_tag broke.add((tag, dest, nvr)) except SQLObjectNotFound: pass # Make sure that all builds in koji tagged as an update exist # in bodhi, and are in the expect state. for release in Release.select(): for tag in (release.testing_tag, release.stable_tag): tagged = [ build['nvr'] for build in koji.listTagged(tag, latest=True) ] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) except SQLObjectNotFound: print "PackageUpdate(%s) not found!" % nvr continue if not len(build.updates): print "PackageBuild(%s) has no updates" % (build.nvr) status = 'testing' in tag and 'testing' or 'stable' for update in build.updates: if update.status != status: print "%s is %s in bodhi but tagged as %s in koji" % ( update.title, update.status, tag) if '--fix' in sys.argv: dest = release.testing_tag if update.status == 'stable': dest = release.stable_tag elif update.status == 'obsolete': dest = release.candidate_tag for b in update.builds: broke.add((tag, dest, b.nvr)) if broke: print " ** Fixing broken tags! **" koji.multicall = True for tag, dest, build in broke: print "Moving %s from %s to %s" % (build, tag, dest) koji.moveBuild(tag, dest, build, force=True) print "Running koji.multiCall()" results = koji.multiCall() success = False print "Waiting for tasks" bad_tasks = wait_for_tasks([task[0] for task in results]) if bad_tasks == 0: success = True if success: print "Tags successfully moved!" else: print "Error moving tags!" print "bad_tasks = %r" % bad_tasks
def main(): load_config() __connection__ = hub = PackageHub("bodhi") koji = get_session() tasks = [] broke = set() # Clean up any stray pending tags for release in Release.select(): print "Finding all pending-testing builds..." if release.name.startswith('EL'): continue tag = release.pending_testing_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable', 'obsolete'): print "%s %s" % (nvr, update.status) if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: print "Can't find build for %s" % nvr if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) tag = release.pending_stable_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('pending', 'obsolete'): print "%s %s" % (nvr, update.status) if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: print "Can't find build for %s" % nvr if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) # Check for testing updates that aren't tagged properly for update in PackageUpdate.select(PackageUpdate.q.status=='testing'): dest_tag = update.release.testing_tag for build in update.builds: tags = [tag['name'] for tag in koji.listTags(build=build.nvr)] if dest_tag not in tags: print "%s marked as testing, but tagged with %s" % (build.nvr, tags) if '--fix' in sys.argv: broke.add((tags[0], dest_tag, build.nvr)) # Check all candidate updates to see if they are in a different bodhi state for release in Release.select(): tag = release.candidate_tag tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable'): print "%s %s but tagged as %s" % (nvr, update.status, tag) if '--fix' in sys.argv: dest = release.testing_tag if update.status == 'stable': dest = release.stable_tag elif update.status == 'obsolete': dest = release.candidate_tag broke.add((tag, dest, nvr)) except SQLObjectNotFound: pass # Make sure that all builds in koji tagged as an update exist # in bodhi, and are in the expect state. for release in Release.select(): for tag in (release.testing_tag, release.stable_tag): tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) except SQLObjectNotFound: print "PackageUpdate(%s) not found!" % nvr continue if not len(build.updates): print "PackageBuild(%s) has no updates" % (build.nvr) status = 'testing' in tag and 'testing' or 'stable' for update in build.updates: if update.status != status: print "%s is %s in bodhi but tagged as %s in koji" % ( update.title, update.status, tag) if '--fix' in sys.argv: dest = release.testing_tag if update.status == 'stable': dest = release.stable_tag elif update.status == 'obsolete': dest = release.candidate_tag for b in update.builds: broke.add((tag, dest, b.nvr)) if broke: print " ** Fixing broken tags! **" koji.multicall = True for tag, dest, build in broke: print "Moving %s from %s to %s" % (build, tag, dest) koji.moveBuild(tag, dest, build, force=True) print "Running koji.multiCall()" results = koji.multiCall() success = False print "Waiting for tasks" bad_tasks = wait_for_tasks([task[0] for task in results]) if bad_tasks == 0: success = True if success: print "Tags successfully moved!" else: print "Error moving tags!" print "bad_tasks = %r" % bad_tasks
def main(): load_config() stats = {} # {release: {'stat': ...}} feedback = 0 # total number of updates that received feedback karma = defaultdict(int) # {username: # of karma submissions} num_updates = PackageUpdate.select().count() proventesters = set() for release in Release.select(): print header(release.long_name) updates = PackageUpdate.select(PackageUpdate.q.releaseID == release.id) stats[release.name] = { 'num_updates': updates.count(), 'num_tested': 0, 'num_tested_without_karma': 0, 'num_feedback': 0, 'num_anon_feedback': 0, 'num_critpath': 0, 'num_critpath_approved': 0, 'num_critpath_unapproved': 0, 'num_stablekarma': 0, 'num_testingtime': 0, 'critpath_without_karma': set(), 'conflicted_proventesters': [], 'critpath_positive_karma_including_proventesters': [], 'critpath_positive_karma_negative_proventesters': [], 'stable_with_negative_karma': PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.status == 'stable', PackageUpdate.q.karma < 0)).count(), 'bugs': set(), 'karma': defaultdict(int), 'deltas': [], 'occurrences': {}, 'accumulative': timedelta(), 'packages': defaultdict(int), 'proventesters': set(), 'proventesters_1': 0, 'proventesters_0': 0, 'proventesters_-1': 0, # for tracking number of types of karma '1': 0, '0': 0, '-1': 0, } data = stats[release.name] for status in statuses: data['num_%s' % status] = PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.status == status)).count() for type in types: data['num_%s' % type] = PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.type == type)).count() for update in release.updates: for build in update.builds: data['packages'][build.package] += 1 for bug in update.bugs: data['bugs'].add(bug.bz_id) feedback_done = False testingtime_done = False for comment in update.comments: if comment.author_name in ('autoqa', 'taskotron'): continue # Track the # of +1's, -1's, and +0's. if comment.author_name != 'bodhi': data[str(comment.karma)] += 1 if comment.author_group == 'proventesters': data['proventesters'].add(comment.author_name) data['proventesters_%d' % comment.karma] += 1 if comment.text == 'This update has reached the stable karma threshold and will be pushed to the stable updates repository': data['num_stablekarma'] += 1 elif comment.text and comment.text.endswith( 'days in testing and can be pushed to stable now if the maintainer wishes' ): data['num_testingtime'] += 1 # For figuring out if an update has received feedback or not if not feedback_done: if (not comment.author.startswith('bodhi') and comment.karma != 0 and not comment.anonymous): data[ 'num_feedback'] += 1 # per-release tracking of feedback feedback += 1 # total number of updates that have received feedback feedback_done = True # so we don't run this for each comment # Tracking per-author karma & anonymous feedback if not comment.author.startswith('bodhi'): if comment.anonymous: # @@: should we track anon +0 comments as "feedback"? if comment.karma != 0: data['num_anon_feedback'] += 1 else: author = comment.author_name data['karma'][author] += 1 karma[author] += 1 if (not testingtime_done and comment.text == 'This update has been pushed to testing'): for othercomment in update.comments: if othercomment.text == 'This update has been pushed to stable': delta = othercomment.timestamp - comment.timestamp data['deltas'].append(delta) data['occurrences'][delta.days] = \ data['occurrences'].setdefault( delta.days, 0) + 1 data['accumulative'] += delta testingtime_done = True break if update.critpath: if update.critpath_approved or update.status == 'stable': data['num_critpath_approved'] += 1 else: if status in ('testing', 'pending'): data['num_critpath_unapproved'] += 1 data['num_critpath'] += 1 #if not feedback_done: if update.status == 'stable' and update.karma == 0: data['critpath_without_karma'].add(update) # Proventester metrics proventester_karma = defaultdict(int) # {username: karma} positive_proventesters = 0 negative_proventesters = 0 for comment in update.comments: if comment.author_group == 'proventesters': proventester_karma[ comment.author_name] += comment.karma for _karma in proventester_karma.values(): if _karma > 0: positive_proventesters += 1 elif _karma < 0: negative_proventesters += 1 # Conflicting proventesters if positive_proventesters and negative_proventesters: data['conflicted_proventesters'] += [short_url(update)] # Track updates with overall positive karma, including positive # karma from a proventester if update.karma > 0 and positive_proventesters: data[ 'critpath_positive_karma_including_proventesters'] += [ short_url(update) ] # Track updates with overall positive karma, including negative # karma from a proventester if update.karma > 0 and negative_proventesters: data['critpath_positive_karma_negative_proventesters'] += [ short_url(update) ] if testingtime_done: data['num_tested'] += 1 if not feedback_done: data['num_tested_without_karma'] += 1 data['deltas'].sort() print " * %d updates" % data['num_updates'] print " * %d packages updated" % (len(data['packages'])) for status in statuses: print " * %d %s updates" % (data['num_%s' % status], status) for type in types: print " * %d %s updates (%0.2f%%)" % ( data['num_%s' % type], type, float(data['num_%s' % type]) / data['num_updates'] * 100) print " * %d bugs resolved" % len(data['bugs']) print " * %d critical path updates (%0.2f%%)" % ( data['num_critpath'], float(data['num_critpath']) / data['num_updates'] * 100) print " * %d approved critical path updates" % ( data['num_critpath_approved']) print " * %d unapproved critical path updates" % ( data['num_critpath_unapproved']) print " * %d updates received feedback (%0.2f%%)" % ( data['num_feedback'], (float(data['num_feedback']) / data['num_updates'] * 100)) print " * %d +0 comments" % data['0'] print " * %d +1 comments" % data['1'] print " * %d -1 comments" % data['-1'] print " * %d unique authenticated karma submitters" % (len( data['karma'])) print " * %d proventesters" % len(data['proventesters']) print " * %d +1's from proventesters" % data['proventesters_1'] print " * %d -1's from proventesters" % data['proventesters_-1'] if data['num_critpath']: print " * %d critpath updates with conflicting proventesters (%0.2f%% of critpath)" % ( len(data['conflicted_proventesters']), float(len(data['conflicted_proventesters'])) / data['num_critpath'] * 100) for u in data['conflicted_proventesters']: print " * " + u print " * %d critpath updates with positive karma and negative proventester feedback (%0.2f%% of critpath)" % ( len(data['critpath_positive_karma_negative_proventesters']), float( len(data['critpath_positive_karma_negative_proventesters']) ) / data['num_critpath'] * 100) for u in data['critpath_positive_karma_negative_proventesters']: print " * " + u print " * %d critpath updates with positive karma and positive proventester feedback (%0.2f%% of critpath)" % ( len(data['critpath_positive_karma_including_proventesters']), float( len(data['critpath_positive_karma_including_proventesters'] )) / data['num_critpath'] * 100) print " * %d anonymous users gave feedback (%0.2f%%)" % ( data['num_anon_feedback'], float(data['num_anon_feedback']) / (data['num_anon_feedback'] + sum(data['karma'].values())) * 100) # This does not take into account updates that reach stablekarma before being pushed to testing! # print " * %d out of %d stable updates went through testing (%0.2f%%)" %( # data['num_tested'], data['num_stable'], # float(data['num_tested']) / data['num_stable'] * 100) print " * %d updates reached the stable karma threshold (%0.2f%%)" % ( data['num_stablekarma'], float(data['num_stablekarma']) / data['num_stable'] * 100) print " * %d updates reached the minimum time in testing threshold (%0.2f%%)" % ( data['num_testingtime'], float(data['num_testingtime']) / data['num_stable'] * 100) print " * %d went from testing to stable *without* karma (%0.2f%%)" % ( data['num_tested_without_karma'], float(data['num_tested_without_karma']) / data['num_tested'] * 100) print " * %d updates were pushed to stable with negative karma (%0.2f%%)" % ( data['stable_with_negative_karma'], float(data['stable_with_negative_karma']) / data['num_stable'] * 100) print " * %d critical path updates pushed to stable *without* karma" % ( len(data['critpath_without_karma'])) #for update in data['critpath_without_karma']: # print " * %s submitted by %s" % (update.title, update.submitter) print " * Time spent in testing:" print " * mean = %d days" % (data['accumulative'].days / len(data['deltas'])) print " * median = %d days" % (data['deltas'][len(data['deltas']) / 2].days) print " * mode = %d days" % (sorted(data['occurrences'].items(), key=itemgetter(1))[-1][0]) #for package in sorted(data['packages'].items(), key=itemgetter(1), reverse=True): # print " * %s: %d" % (package[0].name, package[1]) print print print "Out of %d total updates, %d received feedback (%0.2f%%)" % ( num_updates, feedback, (float(feedback) / num_updates * 100)) print "Out of %d total unique commenters, the top 50 are:" % (len(karma)) for submitter in sorted(karma.iteritems(), key=itemgetter(1), reverse=True)[:50]: print " * %s (%d)" % (submitter[0], submitter[1])
def get_feed_data(self, release=None, type=None, status=None, comments=False, submitter=None, builds=None, user=None, package=None, critpath=False, unapproved=None, *args, **kw): query = [] entries = [] date = lambda update: update.date_pushed order = PackageUpdate.q.date_pushed title = [] critpath = critpath in (True, 'True', 'true') unapproved = unapproved in (True, 'True', 'true') if critpath: return self.get_critpath_updates(release=release, unapproved=unapproved) if comments: return self.get_latest_comments(user=user) if package: return self.get_package_updates(package, release) if release: try: rel = Release.byName(release.upper()) except SQLObjectNotFound: return dict(title='%s not found' % release, entries=[]) query.append(PackageUpdate.q.releaseID == rel.id) title.append(rel.long_name) if type: query.append(PackageUpdate.q.type == type) title.append(type.title()) if status: query.append(PackageUpdate.q.status == status) if status == 'pending': date = lambda update: update.date_submitted order = PackageUpdate.q.date_submitted else: # Let's only show pushed testing/stable updates query.append(PackageUpdate.q.pushed == True) title.append(status.title()) else: query.append(PackageUpdate.q.pushed == True) if submitter: query.append(PackageUpdate.q.submitter == submitter) title.append("submitted by %s" % submitter) if builds: query.append(PackageUpdate.q.builds == builds) title.append("for %s" % builds) updates = PackageUpdate.select(AND(*query), orderBy=order).reversed() for update in updates: delta = datetime.utcnow() - update.date_submitted if delta and delta.days > config.get('feeds.num_days_to_show'): if len(entries) >= config.get('feeds.max_entries'): break entries.append({ 'id': config.get('base_address') + url(update.get_url()), 'summary': update.notes, 'published': date(update), 'link': config.get('base_address') + url(update.get_url()), 'title': "%s %sUpdate: %s" % (update.release.long_name, update.type == 'security' and 'Security ' or '', update.title) }) if len(update.bugs): bugs = "<b>Resolved Bugs</b><br/>" for bug in update.bugs: bugs += "<a href=%s>%d</a> - %s<br/>" % ( bug.get_url(), bug.bz_id, bug.title) entries[-1]['summary'] = "%s<br/>%s" % (bugs[:-2], entries[-1]['summary']) title.append('Updates') return dict(title=' '.join(title), subtitle="", link=config.get('base_address') + url('/'), entries=entries)