def main(argv=sys.argv):
"""
Remove the pending and testing tags from branched updates.
Args:
argv (list): The arguments passed to the script. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging(config_uri)
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
koji = buildsys.get_session()
one_day = timedelta(days=1)
now = datetime.utcnow()
try:
for release in db.query(Release).filter_by(
state=ReleaseState.pending).all():
log.info(release.name)
for update in db.query(Update).filter_by(
release=release, status=UpdateStatus.stable).all():
if now - update.date_stable > one_day:
for build in update.builds:
tags = build.get_tags()
stable_tag = release.dist_tag
testing_tag = release.testing_tag
pending_signing_tag = release.pending_signing_tag
pending_testing_tag = release.pending_testing_tag
if stable_tag not in tags:
log.error('%s not tagged as stable %s' %
(build.nvr, tags))
continue
if testing_tag in tags:
log.info('Removing %s from %s' %
(testing_tag, build.nvr))
koji.untagBuild(testing_tag, build.nvr)
if pending_signing_tag in tags:
log.info('Removing %s from %s' %
(pending_signing_tag, build.nvr))
koji.untagBuild(pending_signing_tag, build.nvr)
if pending_testing_tag in tags:
log.info('Removing %s from %s' %
(pending_testing_tag, build.nvr))
koji.untagBuild(pending_testing_tag, build.nvr)
db.commit()
except Exception as e:
log.error(e)
db.rollback()
Session.remove()
sys.exit(1)
def dequeue_stable():
"""Convert all batched requests to stable requests."""
initialize_db(config.config)
buildsys.setup_buildsystem(config.config)
db = Session()
try:
batched = db.query(models.Update).filter_by(
request=models.UpdateRequest.batched).all()
for update in batched:
try:
update.set_request(db, models.UpdateRequest.stable, u'bodhi')
db.commit()
except Exception as e:
print('Unable to stabilize {}: {}'.format(
update.alias, str(e)))
db.rollback()
msg = u"Bodhi is unable to request this update for stabilization: {}"
update.comment(db, msg.format(str(e)), author=u'bodhi')
db.commit()
except Exception as e:
print(str(e))
sys.exit(1)
finally:
Session.remove()
def main(argv=sys.argv):
"""
Search for overrides that are past their expiration date and mark them expired.
Args:
argv (list): The command line arguments. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging()
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
setup_buildsystem(settings)
try:
now = datetime.utcnow()
overrides = db.query(BuildrootOverride)
overrides = overrides.filter(BuildrootOverride.expired_date.is_(None))
overrides = overrides.filter(BuildrootOverride.expiration_date < now)
count = overrides.count()
if not count:
log.info("No active buildroot override to expire")
return
log.info("Expiring %d buildroot overrides...", count)
for override in overrides:
log.debug(
f"Expiring BRO for {override.build.nvr} because it's due to expire."
)
override.expire()
db.add(override)
log.info("Expired %s" % override.build.nvr)
db.commit()
except Exception as e:
log.error(e)
db.rollback()
Session.remove()
sys.exit(1)
def expire_overrides(db: Session):
"""Search for overrides that are past their expiration date and mark them expired."""
now = datetime.utcnow()
overrides = db.query(BuildrootOverride)
overrides = overrides.filter(BuildrootOverride.expired_date.is_(None))
overrides = overrides.filter(BuildrootOverride.expiration_date < now)
count = overrides.count()
if not count:
log.info("No active buildroot override to expire")
return
log.info("Expiring %d buildroot overrides...", count)
for override in overrides:
log.debug(f"Expiring BRO for {override.build.nvr} because it's due to expire.")
override.expire()
db.add(override)
log.info("Expired %s" % override.build.nvr)
def main():
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, and run approve_update on
them.
"""
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
approve_update(update, db)
db.commit()
except Exception:
log.exception("There was an error approving testing updates.")
db.rollback()
Session.remove()
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
buildsys.setup_buildsystem(config)
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(
f"{update.release.name} doesn't have mandatory days in testing"
)
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
# Only send email notification about the update reaching
# testing approval on releases composed by bodhi
update.comment(
db,
str(config.get('testing_approval_msg')),
author='bodhi',
email_notification=update.release.composed_by_bodhi)
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
# For now only rawhide update can be created using side tag
# Do not add the release.pending_stable_tag if the update
# was created from a side tag.
if update.release.composed_by_bodhi:
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
# For updates that are not included in composes run by bodhi itself,
# mark them as stable
else:
# Single and Multi build update
conflicting_builds = update.find_conflicting_builds()
if conflicting_builds:
builds_str = str.join(", ", conflicting_builds)
update.comment(
db, "This update cannot be pushed to stable. "
f"These builds {builds_str} have a more recent "
f"build in koji's {update.release.stable_tag} tag.",
author="bodhi")
update.request = None
if update.from_tag is not None:
update.status = UpdateStatus.pending
update.remove_tag(
update.release.get_testing_side_tag(
update.from_tag))
else:
update.status = UpdateStatus.obsolete
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
continue
update.add_tag(update.release.stable_tag)
update.status = UpdateStatus.stable
update.request = None
update.pushed = True
update.date_stable = update.date_pushed = func.current_timestamp(
)
update.comment(
db,
"This update has been submitted for stable by bodhi",
author=u'bodhi')
# Multi build update
if update.from_tag:
# Merging the side tag should happen here
pending_signing_tag = update.release.get_pending_signing_side_tag(
update.from_tag)
testing_tag = update.release.get_testing_side_tag(
update.from_tag)
update.remove_tag(pending_signing_tag)
update.remove_tag(testing_tag)
update.remove_tag(update.from_tag)
koji = buildsys.get_session()
koji.deleteTag(pending_signing_tag)
koji.deleteTag(testing_tag)
# Removes the tag and the build target from koji.
koji.removeSideTag(update.from_tag)
else:
# Single build update
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(
update.release.pending_stable_tag)
update.remove_tag(
update.release.pending_signing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
# If this release does not have any testing requirements, skip it
if not update.release.mandatory_days_in_testing:
print('%s doesn\'t have mandatory days in testing' %
update.release.name)
continue
# If this has already met testing requirements, skip it
if update.met_testing_requirements:
continue
# Approval message when testing based on karma threshold
if update.stable_karma not in (0, None) and update.karma >= update.stable_karma \
and not update.autokarma and update.meets_testing_requirements:
print('%s now reaches stable karma threshold' % update.title)
text = config.get('testing_approval_msg_based_on_karma')
update.comment(db, text, author=u'bodhi')
continue
# If autokarma updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print('%s now meets testing requirements' % update.title)
text = six.text_type(
config.get('testing_approval_msg') %
update.mandatory_days_in_testing)
update.comment(db, text, author=u'bodhi')
notifications.publish(topic='update.requirements_met.stable',
msg=dict(update=update))
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def main(releases=None):
initialize_db(bodhi.server.config)
db = Session()
stats = {} # {release: {'stat': ...}}
feedback = 0 # total number of updates that received feedback
karma = defaultdict(int) # {username: # of karma submissions}
num_updates = db.query(Update).count()
for release in db.query(Release).all():
if releases and release.name not in releases:
continue
updates = db.query(Update).filter_by(release=release)
critpath_pkgs = get_critpath_components(release.name.lower())
total = updates.count()
if not total:
continue
print(header(release.long_name))
stats[release.name] = {
'num_updates':
total,
'num_tested':
0,
'num_tested_without_karma':
0,
'num_feedback':
0,
'num_anon_feedback':
0,
'critpath_pkgs':
defaultdict(int),
'num_critpath':
0,
'num_critpath_approved':
0,
'num_critpath_unapproved':
0,
'num_stablekarma':
0,
'num_testingtime':
0,
'critpath_without_karma':
set(),
'conflicted_proventesters': [],
'critpath_positive_karma_including_proventesters': [],
'critpath_positive_karma_negative_proventesters': [],
'stable_with_negative_karma':
updates.filter(
and_(Update.status == UpdateStatus.stable,
Update.karma < 0)).count(),
'bugs':
set(),
'karma':
defaultdict(int),
'deltas': [],
'occurrences': {},
'accumulative':
timedelta(),
'packages':
defaultdict(int),
'proventesters':
set(),
'proventesters_1':
0,
'proventesters_0':
0,
'proventesters_-1':
0,
'submitters':
defaultdict(int),
# for tracking number of types of karma
'1':
0,
'0':
0,
'-1':
0,
}
data = stats[release.name]
for status in statuses:
data['num_%s' % status] = updates.filter(
and_(Update.status == UpdateStatus.from_string(
status))).count()
for type in types:
data['num_%s' % type] = updates.filter(
Update.type == UpdateType.from_string(type)).count()
for update in updates.all():
assert update.user, update.title
data['submitters'][update.user.name] += 1
for build in update.builds:
data['packages'][build.package.name] += 1
if build.package.name in critpath_pkgs:
data['critpath_pkgs'][build.package.name] += 1
for bug in update.bugs:
data['bugs'].add(bug.bug_id)
feedback_done = False
testingtime_done = False
stablekarma_done = False
for comment in update.comments:
if not comment.user:
print('Error: None comment for %s' % update.title)
if comment.user.name == 'autoqa':
continue
# Track the # of +1's, -1's, and +0's.
if comment.user.name != 'bodhi':
data[str(comment.karma)] += 1
if 'proventesters' in [g.name for g in comment.user.groups]:
data['proventesters'].add(comment.user.name)
data['proventesters_%d' % comment.karma] += 1
if update.status == UpdateStatus.stable:
if not stablekarma_done:
if comment.text == (
'This update has reached the stable karma threshold and will be '
'pushed to the stable updates repository'):
data['num_stablekarma'] += 1
stablekarma_done = True
elif comment.text and comment.text.endswith(
'days in testing and can be pushed to stable now if the maintainer '
'wishes'):
data['num_testingtime'] += 1
stablekarma_done = True
# For figuring out if an update has received feedback or not
if not feedback_done:
if (not comment.user.name == 'bodhi' and comment.karma != 0
and not comment.anonymous):
data[
'num_feedback'] += 1 # per-release tracking of feedback
feedback += 1 # total number of updates that have received feedback
feedback_done = True # so we don't run this for each comment
# Tracking per-author karma & anonymous feedback
if not comment.user.name == 'bodhi':
if comment.anonymous:
# @@: should we track anon +0 comments as "feedback"?
if comment.karma != 0:
data['num_anon_feedback'] += 1
else:
author = comment.user.name
data['karma'][author] += 1
karma[author] += 1
if (not testingtime_done and comment.text
== 'This update has been pushed to testing'):
for othercomment in update.comments:
if othercomment.text == 'This update has been pushed to stable':
delta = othercomment.timestamp - comment.timestamp
data['deltas'].append(delta)
data['occurrences'][delta.days] = \
data['occurrences'].setdefault(
delta.days, 0) + 1
data['accumulative'] += delta
testingtime_done = True
break
if update.critpath:
if update.critpath_approved or update.status == UpdateStatus.stable:
data['num_critpath_approved'] += 1
else:
if update.status in (UpdateStatus.testing,
UpdateStatus.pending):
data['num_critpath_unapproved'] += 1
data['num_critpath'] += 1
if update.status == UpdateStatus.stable and update.karma == 0:
data['critpath_without_karma'].add(update)
# Proventester metrics
proventester_karma = defaultdict(int) # {username: karma}
positive_proventesters = 0
negative_proventesters = 0
for comment in update.comments:
if 'proventesters' in [
g.name for g in comment.user.groups
]:
proventester_karma[comment.user.name] += comment.karma
for _karma in proventester_karma.values():
if _karma > 0:
positive_proventesters += 1
elif _karma < 0:
negative_proventesters += 1
# Conflicting proventesters
if positive_proventesters and negative_proventesters:
data['conflicted_proventesters'] += [short_url(update)]
# Track updates with overall positive karma, including positive
# karma from a proventester
if update.karma > 0 and positive_proventesters:
data[
'critpath_positive_karma_including_proventesters'] += [
short_url(update)
]
# Track updates with overall positive karma, including negative
# karma from a proventester
if update.karma > 0 and negative_proventesters:
data['critpath_positive_karma_negative_proventesters'] += [
short_url(update)
]
if testingtime_done:
data['num_tested'] += 1
if not feedback_done:
data['num_tested_without_karma'] += 1
data['deltas'].sort()
print(" * %d updates" % data['num_updates'])
print(" * %d packages updated" % (len(data['packages'])))
for status in statuses:
print(" * %d %s updates" % (data['num_%s' % status], status))
for type in types:
print(" * %d %s updates (%0.2f%%)" %
(data['num_%s' % type], type,
float(data['num_%s' % type]) / data['num_updates'] * 100))
print(" * %d bugs resolved" % len(data['bugs']))
print(" * %d critical path updates (%0.2f%%)" %
(data['num_critpath'],
float(data['num_critpath']) / data['num_updates'] * 100))
print(" * %d approved critical path updates" %
(data['num_critpath_approved']))
print(" * %d unapproved critical path updates" %
(data['num_critpath_unapproved']))
print(" * %d updates received feedback (%0.2f%%)" %
(data['num_feedback'],
(float(data['num_feedback']) / data['num_updates'] * 100)))
print(" * %d +0 comments" % data['0'])
print(" * %d +1 comments" % data['1'])
print(" * %d -1 comments" % data['-1'])
print(" * %d unique authenticated karma submitters" %
(len(data['karma'])))
print(" * %d proventesters" % len(data['proventesters']))
print(" * %d +1's from proventesters" % data['proventesters_1'])
print(" * %d -1's from proventesters" % data['proventesters_-1'])
if data['num_critpath']:
print(
" * %d critpath updates with conflicting proventesters (%0.2f%% of critpath)"
% (len(data['conflicted_proventesters']),
float(len(data['conflicted_proventesters'])) /
data['num_critpath'] * 100))
for u in sorted(data['conflicted_proventesters']):
print(' <li><a href="%s">%s</a></li>' %
(u, u.split('/')[-1]))
msg = (
" * %d critpath updates with positive karma and negative proventester feedback "
"(%0.2f%% of critpath)")
print(
msg %
(len(data['critpath_positive_karma_negative_proventesters']),
float(
len(data['critpath_positive_karma_negative_proventesters']
)) / data['num_critpath'] * 100))
for u in sorted(
data['critpath_positive_karma_negative_proventesters']):
print(' <li><a href="%s">%s</a></li>' %
(u, u.split('/')[-1]))
msg = (
" * %d critpath updates with positive karma and positive proventester feedback "
"(%0.2f%% of critpath)")
print(msg % (
len(data['critpath_positive_karma_including_proventesters']),
float(
len(data['critpath_positive_karma_including_proventesters']
)) / data['num_critpath'] * 100))
print(
" * %d anonymous users gave feedback (%0.2f%%)" %
(data['num_anon_feedback'], float(data['num_anon_feedback']) /
(data['num_anon_feedback'] + sum(data['karma'].values())) * 100))
print(
" * %d stable updates reached the stable karma threshold (%0.2f%%)"
% (data['num_stablekarma'],
float(data['num_stablekarma']) / data['num_stable'] * 100))
print(
" * %d stable updates reached the minimum time in testing threshold (%0.2f%%)"
% (data['num_testingtime'],
float(data['num_testingtime']) / data['num_stable'] * 100))
print(" * %d went from testing to stable *without* karma (%0.2f%%)" %
(data['num_tested_without_karma'],
float(data['num_tested_without_karma']) / data['num_tested'] *
100))
print(
" * %d updates were pushed to stable with negative karma (%0.2f%%)"
% (data['stable_with_negative_karma'],
float(data['stable_with_negative_karma']) / data['num_stable'] *
100))
print(" * %d critical path updates pushed to stable *without* karma" %
(len(data['critpath_without_karma'])))
print(" * Time spent in testing:")
print(" * mean = %d days" %
(data['accumulative'].days / len(data['deltas'])))
print(" * median = %d days" %
(data['deltas'][len(data['deltas']) / 2].days))
print(" * mode = %d days" % (sorted(
list(data['occurrences'].items()), key=itemgetter(1))[-1][0]))
print("Out of %d packages updated, the top 50 were:" %
(len(data['packages'])))
for package in sorted(six.iteritems(data['packages']),
key=itemgetter(1),
reverse=True)[:50]:
print(" * %s (%d)" % (package[0], package[1]))
print("Out of %d update submitters, the top 50 were:" %
(len(data['submitters'])))
for submitter in sorted(six.iteritems(data['submitters']),
key=itemgetter(1),
reverse=True)[:50]:
print(" * %s (%d)" % (submitter[0], submitter[1]))
print("Out of %d critical path updates, the top 50 updated were:" %
(len(data['critpath_pkgs'])))
for x in sorted(six.iteritems(data['critpath_pkgs']),
key=itemgetter(1),
reverse=True)[:50]:
print(" * %s (%d)" % (x[0], x[1]))
critpath_not_updated = set()
for pkg in critpath_pkgs:
if pkg not in data['critpath_pkgs']:
critpath_not_updated.add(pkg)
print("Out of %d critical path packages, %d were never updated:" %
(len(critpath_pkgs), len(critpath_not_updated)))
for pkg in sorted(critpath_not_updated):
print(' * %s' % pkg)
print()
print()
print("Out of %d total updates, %d received feedback (%0.2f%%)" %
(num_updates, feedback, (float(feedback) / num_updates * 100)))
print("Out of %d total unique commenters, the top 50 were:" % (len(karma)))
for submitter in sorted(six.iteritems(karma),
key=itemgetter(1),
reverse=True)[:50]:
print(" * %s (%d)" % (submitter[0], submitter[1]))
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(
f"{update.release.name} doesn't have mandatory days in testing"
)
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
update.comment(db,
str(config.get('testing_approval_msg')),
author='bodhi')
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
