def __init__(self, hub, *args, **kwargs):
"""
Initialize the UpdatesHandler, subscribing it to the appropriate topics.
Args:
hub (moksha.hub.hub.CentralMokshaHub): The hub this handler is consuming messages from.
It is used to look up the hub config.
"""
initialize_db(config)
self.db_factory = util.transactional_session_maker()
prefix = hub.config.get('topic_prefix')
env = hub.config.get('environment')
self.topic = [
prefix + '.' + env + '.bodhi.update.request.testing',
prefix + '.' + env + '.bodhi.update.edit',
]
self.handle_bugs = bool(config.get('bodhi_email'))
if not self.handle_bugs:
log.warning("No bodhi_email defined; not fetching bug details")
else:
bug_module.set_bugtracker()
super(UpdatesHandler, self).__init__(hub, *args, **kwargs)
log.info('Bodhi updates handler listening on:\n'
'%s' % pprint.pformat(self.topic))
def dequeue_stable():
"""Convert all batched requests to stable requests."""
initialize_db(config.config)
buildsys.setup_buildsystem(config.config)
db = Session()
try:
batched = db.query(models.Update).filter_by(
request=models.UpdateRequest.batched).all()
for update in batched:
try:
update.set_request(db, models.UpdateRequest.stable, u'bodhi')
db.commit()
except Exception as e:
print('Unable to stabilize {}: {}'.format(
update.alias, str(e)))
db.rollback()
msg = u"Bodhi is unable to request this update for stabilization: {}"
update.comment(db, msg.format(str(e)), author=u'bodhi')
db.commit()
except Exception as e:
print(str(e))
sys.exit(1)
finally:
Session.remove()
def check():
"""Check the enforced policies by Greenwave for each open update."""
initialize_db(config.config)
session = Session()
updates = models.Update.query.filter(
models.Update.status.in_(
[models.UpdateStatus.pending, models.UpdateStatus.testing])
).filter(
models.Update.release_id == models.Release.id
).filter(
models.Release.state.in_(
[models.ReleaseState.current, models.ReleaseState.pending])
).order_by(
# Check the older updates first so there is more time for the newer to
# get their test results
models.Update.id.asc()
)
for update in updates:
try:
update.update_test_gating_status()
session.commit()
except Exception as e:
# If there is a problem talking to Greenwave server, print the error.
click.echo(str(e))
session.rollback()
def main(argv=sys.argv):
"""
Remove the pending and testing tags from branched updates.
Args:
argv (list): The arguments passed to the script. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging(config_uri)
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
koji = buildsys.get_session()
one_day = timedelta(days=1)
now = datetime.utcnow()
try:
for release in db.query(Release).filter_by(
state=ReleaseState.pending).all():
log.info(release.name)
for update in db.query(Update).filter_by(
release=release, status=UpdateStatus.stable).all():
if now - update.date_stable > one_day:
for build in update.builds:
tags = build.get_tags()
stable_tag = release.dist_tag
testing_tag = release.testing_tag
pending_signing_tag = release.pending_signing_tag
pending_testing_tag = release.pending_testing_tag
if stable_tag not in tags:
log.error('%s not tagged as stable %s' %
(build.nvr, tags))
continue
if testing_tag in tags:
log.info('Removing %s from %s' %
(testing_tag, build.nvr))
koji.untagBuild(testing_tag, build.nvr)
if pending_signing_tag in tags:
log.info('Removing %s from %s' %
(pending_signing_tag, build.nvr))
koji.untagBuild(pending_signing_tag, build.nvr)
if pending_testing_tag in tags:
log.info('Removing %s from %s' %
(pending_testing_tag, build.nvr))
koji.untagBuild(pending_testing_tag, build.nvr)
db.commit()
except Exception as e:
log.error(e)
db.rollback()
Session.remove()
sys.exit(1)
def __init__(self, *args, **kwargs):
"""Initialize the UpdatesHandler."""
initialize_db(config)
self.db_factory = util.transactional_session_maker()
self.handle_bugs = bool(config.get('bodhi_email'))
if not self.handle_bugs:
log.warning("No bodhi_email defined; not fetching bug details")
else:
bug_module.set_bugtracker()
def __init__(self):
"""Set up the database, build system, bug tracker, and handlers."""
log.info('Initializing Bodhi')
initialize_db(config)
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
self.handler_infos = [
HandlerInfo('.buildsys.tag', "Signed", SignedHandler()),
HandlerInfo('.buildsys.tag', 'Automatic Update', AutomaticUpdateHandler()),
HandlerInfo('.greenwave.decision.update', 'Greenwave', GreenwaveHandler()),
HandlerInfo('.ci.koji-build.test.running', 'CI', CIHandler())
]
def __init__(self):
"""Set up the database, build system, bug tracker, and handlers."""
log.info('Initializing Bodhi')
initialize_db(config)
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
self.handler_infos = [
HandlerInfo('.buildsys.tag', "Signed", SignedHandler()),
HandlerInfo('.buildsys.tag', 'Automatic Update', AutomaticUpdateHandler()),
HandlerInfo('.ci.koji-build.test.running', 'CI', CIHandler()),
HandlerInfo('.waiverdb.waiver.new', 'WaiverDB', WaiverdbHandler()),
HandlerInfo('.resultsdb.result.new', 'ResultsDB', ResultsdbHandler()),
]
def main(argv=sys.argv):
"""
Search for overrides that are past their expiration date and mark them expired.
Args:
argv (list): The command line arguments. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging()
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
setup_buildsystem(settings)
try:
now = datetime.utcnow()
overrides = db.query(BuildrootOverride)
overrides = overrides.filter(BuildrootOverride.expired_date.is_(None))
overrides = overrides.filter(BuildrootOverride.expiration_date < now)
count = overrides.count()
if not count:
log.info("No active buildroot override to expire")
return
log.info("Expiring %d buildroot overrides...", count)
for override in overrides:
log.debug(
f"Expiring BRO for {override.build.nvr} because it's due to expire."
)
override.expire()
db.add(override)
log.info("Expired %s" % override.build.nvr)
db.commit()
except Exception as e:
log.error(e)
db.rollback()
Session.remove()
sys.exit(1)
def check():
"""Check the enforced policies by Greenwave for each open update."""
initialize_db(config.config)
session = Session()
updates = models.Update.query.filter(models.Update.status.in_(
[models.UpdateStatus.pending, models.UpdateStatus.testing]))
for update in updates:
try:
update.update_test_gating_status()
session.commit()
except Exception as e:
# If there is a problem talking to Greenwave server, print the error.
click.echo(str(e))
session.rollback()
def __init__(self):
"""Set up the database, build system, bug tracker, and handlers."""
log.info('Initializing Bodhi')
initialize_db(config)
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
if ComposerHandler:
self.composer_handler = ComposerHandler()
else:
log.info(
'The composer is not installed - Bodhi will ignore composer.start messages.'
)
self.composer_handler = None
self.signed_handler = SignedHandler()
self.updates_handler = UpdatesHandler()
def monitor():
"""Print a simple human-readable report about existing Compose objects."""
initialize_db(config.config)
buildsys.setup_buildsystem(config.config)
click.echo('Locked updates: %s\n' %
models.Update.query.filter_by(locked=True).count())
for c in sorted([c for c in models.Compose.query.all()]):
click.echo(c)
for attr in ('state', 'state_date', 'security', 'error_message'):
if getattr(c, attr) is not None:
click.echo('\t%s: %s' % (attr, getattr(c, attr)))
click.echo('\tcheckpoints: %s' %
', '.join(json.loads(c.checkpoints).keys()))
click.echo('\tlen(updates): %s\n' % len(c.updates))
def __init__(self, hub, *args, **kwargs):
"""
Initialize the SignedHandler, configuring its topic and database.
Args:
hub (moksha.hub.hub.CentralMokshaHub): The hub this handler is consuming messages from.
It is used to look up the hub config.
"""
initialize_db(config)
self.db_factory = transactional_session_maker()
prefix = hub.config.get('topic_prefix')
env = hub.config.get('environment')
self.topic = [prefix + '.' + env + '.buildsys.tag']
super(SignedHandler, self).__init__(hub, *args, **kwargs)
log.info('Bodhi signed handler listening on:\n'
'%s' % pprint.pformat(self.topic))
def check():
"""Check the enforced policies by Greenwave for each open update."""
initialize_db(config.config)
session = Session()
updates = models.Update.query.filter(models.Update.pushed == false())\
.filter(models.Update.status.in_(
[models.UpdateStatus.pending, models.UpdateStatus.testing]))
for update in updates:
# We retrieve updates going to testing (status=pending) and updates
# (status=testing) going to stable.
# If the update is pending, we want to know if it can go to testing
decision_context = u'bodhi_update_push_testing'
if update.status == models.UpdateStatus.testing:
# Update is already in testing, let's ask if it can go to stable
decision_context = u'bodhi_update_push_stable'
data = {
'product_version': update.product_version,
'decision_context': decision_context,
'subject': update.greenwave_subject
}
api_url = '{}/decision'.format(
config.config.get('greenwave_api_url').rstrip('/'))
try:
decision = greenwave_api_post(api_url, data)
if decision['policies_satisfied']:
# If an unrestricted policy is applied and no tests are required
# on this update, let's set the test gating as ignored in Bodhi.
if decision['summary'] == 'no tests are required':
update.test_gating_status = models.TestGatingStatus.ignored
else:
update.test_gating_status = models.TestGatingStatus.passed
else:
update.test_gating_status = models.TestGatingStatus.failed
update.greenwave_summary_string = decision['summary']
session.commit()
except Exception as e:
# If there is a problem talking to Greenwave server, print the error.
click.echo(str(e))
session.rollback()
def get_user_data(username, human_readable):
"""Get username SAR data."""
initialize_db(config.config)
sar_data = {}
try:
user = models.User.query.filter_by(name=username).one()
except sqlalchemy.orm.exc.NoResultFound:
if human_readable:
click.echo("User not found.", err=True)
sys.exit(0)
sar_data[user.name] = {}
sar_data[user.name]['comments'] = [
{'karma': c.karma, 'karma_critpath': c.karma_critpath, 'text': c.text,
'timestamp': c.timestamp.strftime('%Y-%m-%d %H:%M:%S'),
'update_alias': c.update.alias, 'username': c.user.name}
for c in user.comments]
sar_data[user.name]['email'] = user.email
sar_data[user.name]['groups'] = [g.name for g in user.groups]
sar_data[user.name]['name'] = user.name
sar_data[user.name]['updates'] = [
{'autokarma': u.autokarma, 'stable_karma': u.stable_karma,
'unstable_karma': u.unstable_karma, 'requirements': u.requirements,
'require_bugs': u.require_bugs, 'require_testcases': u.require_testcases,
'notes': u.notes, 'type': str(u.type), 'severity': str(u.severity),
'suggest': str(u.suggest), 'close_bugs': u.close_bugs, 'alias': u.alias,
'builds': [b.nvr for b in u.builds], 'release_name': u.release.name,
'bugs': [b.bug_id for b in u.bugs], 'user': u.user.name,
'date_submitted': u.date_submitted.strftime('%Y-%m-%d %H:%M:%S')}
for u in user.updates]
if human_readable:
print_human_readable_format(sar_data)
else:
click.echo(json.dumps(sar_data, sort_keys=True))
def _configure_test_db(db_uri=DEFAULT_DB):
"""
Create and configure a test database for Bodhi.
.. note::
For some reason, this fails on the in-memory version of SQLite with an error
about nested transactions.
Args:
db_uri (str): The URI to use when creating the database engine. Defaults to an
in-memory SQLite database.
Returns:
sqlalchemy.engine: The database engine.
"""
if db_uri.startswith('sqlite:////'):
# Clean out any old file
db_path = db_uri.split('sqlite:///')[1]
if os.path.isfile(db_path):
os.unlink(db_path)
global engine
engine = initialize_db({'sqlalchemy.url': db_uri})
if db_uri.startswith('sqlite://'):
# Necessary to get nested transactions working with SQLite. See:
# http://docs.sqlalchemy.org/en/latest/dialects/sqlite.html\
# #serializable-isolation-savepoints-transactional-ddl
@event.listens_for(engine, "connect")
def connect_event(dbapi_connection, connection_record):
"""Stop pysqlite from emitting 'BEGIN'."""
# disable pysqlite's emitting of the BEGIN statement entirely.
# also stops it from emitting COMMIT before any DDL.
dbapi_connection.isolation_level = None
@event.listens_for(engine, "begin")
def begin_event(conn):
"""Emit our own 'BEGIN' instead of letting pysqlite do it."""
conn.execute('BEGIN')
@event.listens_for(Session, 'after_transaction_end')
def restart_savepoint(session, transaction):
"""Allow tests to call rollback on the session."""
if transaction.nested and not transaction._parent.nested:
session.expire_all()
session.begin_nested()
return engine
def main(argv=sys.argv):
"""
Initialize the Bodhi database on a new installation.
This function is the entry point used by the initialize_bodhi_db console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging()
settings = get_appsettings(config_uri)
engine = initialize_db(settings)
Base.metadata.bind = engine
Base.metadata.create_all(engine)
def _do_init():
config.load_config()
initialize_db(config)
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
def push(username, yes, **kwargs):
"""Push builds out to the repositories."""
resume = kwargs.pop('resume')
resume_all = False
initialize_db(config)
db_factory = transactional_session_maker()
composes = []
with db_factory() as session:
if not resume and session.query(Compose).count():
if yes:
click.echo('Existing composes detected: {}. Resuming all.'.format(
', '.join([str(c) for c in session.query(Compose).all()])))
else:
click.confirm(
'Existing composes detected: {}. Do you wish to resume them all?'.format(
', '.join([str(c) for c in session.query(Compose).all()])),
abort=True)
resume = True
resume_all = True
# If we're resuming a push
if resume:
for compose in session.query(Compose).all():
if len(compose.updates) == 0:
# Compose objects can end up with 0 updates in them if the composer ejects all
# the updates in a compose for some reason. Composes with no updates cannot be
# serialized because their content_type property uses the content_type of the
# first update in the Compose. Additionally, it doesn't really make sense to go
# forward with running an empty Compose. It makes the most sense to delete them.
click.echo("{} has no updates. It is being removed.".format(compose))
session.delete(compose)
continue
if not resume_all:
if yes:
click.echo('Resuming {}.'.format(compose))
elif not click.confirm('Resume {}?'.format(compose)):
continue
# Reset the Compose's state and error message.
compose.state = ComposeState.requested
compose.error_message = ''
composes.append(compose)
else:
updates = []
# Accept both comma and space separated request list
requests = kwargs['request'].replace(',', ' ').split(' ')
requests = [UpdateRequest.from_string(val) for val in requests]
query = session.query(Update).filter(Update.request.in_(requests))
if kwargs.get('builds'):
query = query.join(Update.builds)
query = query.filter(
or_(*[Build.nvr == build for build in kwargs['builds'].split(',')]))
if kwargs.get('updates'):
query = query.filter(
or_(*[Update.alias == alias for alias in kwargs['updates'].split(',')]))
query = _filter_releases(session, query, kwargs.get('releases'))
for update in query.all():
# Skip unsigned updates (this checks that all builds in the update are signed)
update_sig_status(update)
if not update.signed:
click.echo(
f'Warning: {update.get_title()} has unsigned builds and has been skipped')
continue
updates.append(update)
composes = Compose.from_updates(updates)
for c in composes:
session.add(c)
# We need to flush so the database knows about the new Compose objects, so the
# Compose.updates relationship will work properly. This is due to us overriding the
# primaryjoin on the relationship between Composes and Updates.
session.flush()
# Now we need to refresh the composes so their updates property will not be empty.
for compose in composes:
session.refresh(compose)
# Now we need to sort the composes so their security property can be used to prioritize
# security updates. The security property relies on the updates property being
# non-empty, so this must happen after the refresh above.
composes = sorted(composes)
for compose in composes:
click.echo('\n\n===== {} =====\n'.format(compose))
for update in compose.updates:
click.echo(update.get_title())
if composes:
if yes:
click.echo('\n\nPushing {:d} updates.'.format(
sum([len(c.updates) for c in composes])))
else:
click.confirm('\n\nPush these {:d} updates?'.format(
sum([len(c.updates) for c in composes])), abort=True)
click.echo('\nLocking updates...')
else:
click.echo('\nThere are no updates to push.')
composes = [c.__json__(composer=True) for c in composes]
if composes:
click.echo('\nSending composer.start message')
bodhi.server.notifications.publish(composer_schemas.ComposerStartV1.from_dict(dict(
api_version=2, composes=composes, resume=resume, agent=username)),
force=True)
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
buildsys.setup_buildsystem(config)
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(
f"{update.release.name} doesn't have mandatory days in testing"
)
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
# Only send email notification about the update reaching
# testing approval on releases composed by bodhi
update.comment(
db,
str(config.get('testing_approval_msg')),
author='bodhi',
email_notification=update.release.composed_by_bodhi)
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
# For now only rawhide update can be created using side tag
# Do not add the release.pending_stable_tag if the update
# was created from a side tag.
if update.release.composed_by_bodhi:
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
# For updates that are not included in composes run by bodhi itself,
# mark them as stable
else:
# Single and Multi build update
conflicting_builds = update.find_conflicting_builds()
if conflicting_builds:
builds_str = str.join(", ", conflicting_builds)
update.comment(
db, "This update cannot be pushed to stable. "
f"These builds {builds_str} have a more recent "
f"build in koji's {update.release.stable_tag} tag.",
author="bodhi")
update.request = None
if update.from_tag is not None:
update.status = UpdateStatus.pending
update.remove_tag(
update.release.get_testing_side_tag(
update.from_tag))
else:
update.status = UpdateStatus.obsolete
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
continue
update.add_tag(update.release.stable_tag)
update.status = UpdateStatus.stable
update.request = None
update.pushed = True
update.date_stable = update.date_pushed = func.current_timestamp(
)
update.comment(
db,
"This update has been submitted for stable by bodhi",
author=u'bodhi')
# Multi build update
if update.from_tag:
# Merging the side tag should happen here
pending_signing_tag = update.release.get_pending_signing_side_tag(
update.from_tag)
testing_tag = update.release.get_testing_side_tag(
update.from_tag)
update.remove_tag(pending_signing_tag)
update.remove_tag(testing_tag)
update.remove_tag(update.from_tag)
koji = buildsys.get_session()
koji.deleteTag(pending_signing_tag)
koji.deleteTag(testing_tag)
# Removes the tag and the build target from koji.
koji.removeSideTag(update.from_tag)
else:
# Single build update
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(
update.release.pending_stable_tag)
update.remove_tag(
update.release.pending_signing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def push(username, cert_prefix, **kwargs):
"""Push builds out to the repositories."""
resume = kwargs.pop('resume')
lockfiles = defaultdict(list)
locked_updates = []
locks = '%s/MASHING-*' % config.get('mash_dir')
for lockfile in glob.glob(locks):
with file(lockfile) as lock:
state = json.load(lock)
for update in state['updates']:
lockfiles[lockfile].append(update)
locked_updates.append(update)
update_titles = None
initialize_db(config)
db_factory = transactional_session_maker()
with db_factory() as session:
updates = []
# If we're resuming a push
if resume:
for lockfile in lockfiles:
if not click.confirm('Resume {}?'.format(lockfile)):
continue
for update in lockfiles[lockfile]:
update = session.query(Update).filter(
Update.title == update).first()
updates.append(update)
else:
# Accept both comma and space separated request list
requests = kwargs['request'].replace(',', ' ').split(' ')
requests = [UpdateRequest.from_string(val) for val in requests]
query = session.query(Update).filter(Update.request.in_(requests))
if kwargs.get('builds'):
query = query.join(Update.builds)
query = query.filter(
or_(*[
Build.nvr == build
for build in kwargs['builds'].split(',')
]))
query = _filter_releases(session, query, kwargs.get('releases'))
for update in query.all():
# Skip locked updates that are in a current push
if update.locked:
if update.title in locked_updates:
continue
# Warn about locked updates that aren't a part of a push and
# push them again.
else:
click.echo('Warning: %s is locked but not in a push' %
update.title)
# Skip unsigned updates (this checks that all builds in the update are signed)
if not update.signed:
click.echo(
'Warning: %s has unsigned builds and has been skipped'
% update.title)
continue
updates.append(update)
for update in updates:
click.echo(update.title)
if updates:
click.confirm('Push these {:d} updates?'.format(len(updates)),
abort=True)
click.echo('\nLocking updates...')
for update in updates:
update.locked = True
update.date_locked = datetime.utcnow()
else:
click.echo('\nThere are no updates to push.')
update_titles = list([update.title for update in updates])
if update_titles:
click.echo('\nSending masher.start fedmsg')
# Because we're a script, we want to send to the fedmsg-relay,
# that's why we say active=True
bodhi.server.notifications.init(active=True, cert_prefix=cert_prefix)
bodhi.server.notifications.publish(
topic='masher.start',
msg=dict(
updates=update_titles,
resume=resume,
agent=username,
),
force=True,
)
Used with the bodhi pshell.
pshell /etc/bodhi/production.ini
execfile('shelldb.py')
"""
from pyramid.paster import get_appsettings
import sys
from bodhi.server import Session, initialize_db
config_uri = None
for arg in sys.argv:
if arg.endswith('.ini'):
config_uri = arg
if not config_uri:
config_uri = '/etc/bodhi/production.ini'
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
def delete_update(up):
for b in up.builds:
db.delete(b)
if b.override:
db.delete(b.override)
for c in up.comments:
db.delete(c)
db.delete(up)
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(
f"{update.release.name} doesn't have mandatory days in testing"
)
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
update.comment(db,
str(config.get('testing_approval_msg')),
author='bodhi')
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""This script will print out SAR data for a FAS account given as the SAR_USERNAME env var."""
import json
import os
import sqlalchemy
from bodhi.server import config, initialize_db, models
initialize_db(config.config)
sar_data = {}
if os.environ['SAR_USERNAME']:
user = None
try:
user = models.User.query.filter_by(
name=os.environ['SAR_USERNAME']).one()
except sqlalchemy.orm.exc.NoResultFound:
# User not found so nothing to do.
pass
if user is not None:
sar_data[user.name] = {}
def __init__(self):
"""Initialize the SignedHandler."""
initialize_db(config)
self.db_factory = transactional_session_maker()
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
# If this release does not have any testing requirements, skip it
if not update.release.mandatory_days_in_testing:
print('%s doesn\'t have mandatory days in testing' %
update.release.name)
continue
# If this has already met testing requirements, skip it
if update.met_testing_requirements:
continue
# Approval message when testing based on karma threshold
if update.stable_karma not in (0, None) and update.karma >= update.stable_karma \
and not update.autokarma and update.meets_testing_requirements:
print('%s now reaches stable karma threshold' % update.title)
text = config.get('testing_approval_msg_based_on_karma')
update.comment(db, text, author=u'bodhi')
continue
# If autokarma updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print('%s now meets testing requirements' % update.title)
text = six.text_type(
config.get('testing_approval_msg') %
update.mandatory_days_in_testing)
update.comment(db, text, author=u'bodhi')
notifications.publish(topic='update.requirements_met.stable',
msg=dict(update=update))
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def main(releases=None):
initialize_db(bodhi.server.config)
db = Session()
stats = {} # {release: {'stat': ...}}
feedback = 0 # total number of updates that received feedback
karma = defaultdict(int) # {username: # of karma submissions}
num_updates = db.query(Update).count()
for release in db.query(Release).all():
if releases and release.name not in releases:
continue
updates = db.query(Update).filter_by(release=release)
critpath_pkgs = get_critpath_components(release.name.lower())
total = updates.count()
if not total:
continue
print(header(release.long_name))
stats[release.name] = {
'num_updates':
total,
'num_tested':
0,
'num_tested_without_karma':
0,
'num_feedback':
0,
'num_anon_feedback':
0,
'critpath_pkgs':
defaultdict(int),
'num_critpath':
0,
'num_critpath_approved':
0,
'num_critpath_unapproved':
0,
'num_stablekarma':
0,
'num_testingtime':
0,
'critpath_without_karma':
set(),
'conflicted_proventesters': [],
'critpath_positive_karma_including_proventesters': [],
'critpath_positive_karma_negative_proventesters': [],
'stable_with_negative_karma':
updates.filter(
and_(Update.status == UpdateStatus.stable,
Update.karma < 0)).count(),
'bugs':
set(),
'karma':
defaultdict(int),
'deltas': [],
'occurrences': {},
'accumulative':
timedelta(),
'packages':
defaultdict(int),
'proventesters':
set(),
'proventesters_1':
0,
'proventesters_0':
0,
'proventesters_-1':
0,
'submitters':
defaultdict(int),
# for tracking number of types of karma
'1':
0,
'0':
0,
'-1':
0,
}
data = stats[release.name]
for status in statuses:
data['num_%s' % status] = updates.filter(
and_(Update.status == UpdateStatus.from_string(
status))).count()
for type in types:
data['num_%s' % type] = updates.filter(
Update.type == UpdateType.from_string(type)).count()
for update in updates.all():
assert update.user, update.title
data['submitters'][update.user.name] += 1
for build in update.builds:
data['packages'][build.package.name] += 1
if build.package.name in critpath_pkgs:
data['critpath_pkgs'][build.package.name] += 1
for bug in update.bugs:
data['bugs'].add(bug.bug_id)
feedback_done = False
testingtime_done = False
stablekarma_done = False
for comment in update.comments:
if not comment.user:
print('Error: None comment for %s' % update.title)
if comment.user.name == 'autoqa':
continue
# Track the # of +1's, -1's, and +0's.
if comment.user.name != 'bodhi':
data[str(comment.karma)] += 1
if 'proventesters' in [g.name for g in comment.user.groups]:
data['proventesters'].add(comment.user.name)
data['proventesters_%d' % comment.karma] += 1
if update.status == UpdateStatus.stable:
if not stablekarma_done:
if comment.text == (
'This update has reached the stable karma threshold and will be '
'pushed to the stable updates repository'):
data['num_stablekarma'] += 1
stablekarma_done = True
elif comment.text and comment.text.endswith(
'days in testing and can be pushed to stable now if the maintainer '
'wishes'):
data['num_testingtime'] += 1
stablekarma_done = True
# For figuring out if an update has received feedback or not
if not feedback_done:
if (not comment.user.name == 'bodhi' and comment.karma != 0
and not comment.anonymous):
data[
'num_feedback'] += 1 # per-release tracking of feedback
feedback += 1 # total number of updates that have received feedback
feedback_done = True # so we don't run this for each comment
# Tracking per-author karma & anonymous feedback
if not comment.user.name == 'bodhi':
if comment.anonymous:
# @@: should we track anon +0 comments as "feedback"?
if comment.karma != 0:
data['num_anon_feedback'] += 1
else:
author = comment.user.name
data['karma'][author] += 1
karma[author] += 1
if (not testingtime_done and comment.text
== 'This update has been pushed to testing'):
for othercomment in update.comments:
if othercomment.text == 'This update has been pushed to stable':
delta = othercomment.timestamp - comment.timestamp
data['deltas'].append(delta)
data['occurrences'][delta.days] = \
data['occurrences'].setdefault(
delta.days, 0) + 1
data['accumulative'] += delta
testingtime_done = True
break
if update.critpath:
if update.critpath_approved or update.status == UpdateStatus.stable:
data['num_critpath_approved'] += 1
else:
if update.status in (UpdateStatus.testing,
UpdateStatus.pending):
data['num_critpath_unapproved'] += 1
data['num_critpath'] += 1
if update.status == UpdateStatus.stable and update.karma == 0:
data['critpath_without_karma'].add(update)
# Proventester metrics
proventester_karma = defaultdict(int) # {username: karma}
positive_proventesters = 0
negative_proventesters = 0
for comment in update.comments:
if 'proventesters' in [
g.name for g in comment.user.groups
]:
proventester_karma[comment.user.name] += comment.karma
for _karma in proventester_karma.values():
if _karma > 0:
positive_proventesters += 1
elif _karma < 0:
negative_proventesters += 1
# Conflicting proventesters
if positive_proventesters and negative_proventesters:
data['conflicted_proventesters'] += [short_url(update)]
# Track updates with overall positive karma, including positive
# karma from a proventester
if update.karma > 0 and positive_proventesters:
data[
'critpath_positive_karma_including_proventesters'] += [
short_url(update)
]
# Track updates with overall positive karma, including negative
# karma from a proventester
if update.karma > 0 and negative_proventesters:
data['critpath_positive_karma_negative_proventesters'] += [
short_url(update)
]
if testingtime_done:
data['num_tested'] += 1
if not feedback_done:
data['num_tested_without_karma'] += 1
data['deltas'].sort()
print(" * %d updates" % data['num_updates'])
print(" * %d packages updated" % (len(data['packages'])))
for status in statuses:
print(" * %d %s updates" % (data['num_%s' % status], status))
for type in types:
print(" * %d %s updates (%0.2f%%)" %
(data['num_%s' % type], type,
float(data['num_%s' % type]) / data['num_updates'] * 100))
print(" * %d bugs resolved" % len(data['bugs']))
print(" * %d critical path updates (%0.2f%%)" %
(data['num_critpath'],
float(data['num_critpath']) / data['num_updates'] * 100))
print(" * %d approved critical path updates" %
(data['num_critpath_approved']))
print(" * %d unapproved critical path updates" %
(data['num_critpath_unapproved']))
print(" * %d updates received feedback (%0.2f%%)" %
(data['num_feedback'],
(float(data['num_feedback']) / data['num_updates'] * 100)))
print(" * %d +0 comments" % data['0'])
print(" * %d +1 comments" % data['1'])
print(" * %d -1 comments" % data['-1'])
print(" * %d unique authenticated karma submitters" %
(len(data['karma'])))
print(" * %d proventesters" % len(data['proventesters']))
print(" * %d +1's from proventesters" % data['proventesters_1'])
print(" * %d -1's from proventesters" % data['proventesters_-1'])
if data['num_critpath']:
print(
" * %d critpath updates with conflicting proventesters (%0.2f%% of critpath)"
% (len(data['conflicted_proventesters']),
float(len(data['conflicted_proventesters'])) /
data['num_critpath'] * 100))
for u in sorted(data['conflicted_proventesters']):
print(' <li><a href="%s">%s</a></li>' %
(u, u.split('/')[-1]))
msg = (
" * %d critpath updates with positive karma and negative proventester feedback "
"(%0.2f%% of critpath)")
print(
msg %
(len(data['critpath_positive_karma_negative_proventesters']),
float(
len(data['critpath_positive_karma_negative_proventesters']
)) / data['num_critpath'] * 100))
for u in sorted(
data['critpath_positive_karma_negative_proventesters']):
print(' <li><a href="%s">%s</a></li>' %
(u, u.split('/')[-1]))
msg = (
" * %d critpath updates with positive karma and positive proventester feedback "
"(%0.2f%% of critpath)")
print(msg % (
len(data['critpath_positive_karma_including_proventesters']),
float(
len(data['critpath_positive_karma_including_proventesters']
)) / data['num_critpath'] * 100))
print(
" * %d anonymous users gave feedback (%0.2f%%)" %
(data['num_anon_feedback'], float(data['num_anon_feedback']) /
(data['num_anon_feedback'] + sum(data['karma'].values())) * 100))
print(
" * %d stable updates reached the stable karma threshold (%0.2f%%)"
% (data['num_stablekarma'],
float(data['num_stablekarma']) / data['num_stable'] * 100))
print(
" * %d stable updates reached the minimum time in testing threshold (%0.2f%%)"
% (data['num_testingtime'],
float(data['num_testingtime']) / data['num_stable'] * 100))
print(" * %d went from testing to stable *without* karma (%0.2f%%)" %
(data['num_tested_without_karma'],
float(data['num_tested_without_karma']) / data['num_tested'] *
100))
print(
" * %d updates were pushed to stable with negative karma (%0.2f%%)"
% (data['stable_with_negative_karma'],
float(data['stable_with_negative_karma']) / data['num_stable'] *
100))
print(" * %d critical path updates pushed to stable *without* karma" %
(len(data['critpath_without_karma'])))
print(" * Time spent in testing:")
print(" * mean = %d days" %
(data['accumulative'].days / len(data['deltas'])))
print(" * median = %d days" %
(data['deltas'][len(data['deltas']) / 2].days))
print(" * mode = %d days" % (sorted(
list(data['occurrences'].items()), key=itemgetter(1))[-1][0]))
print("Out of %d packages updated, the top 50 were:" %
(len(data['packages'])))
for package in sorted(six.iteritems(data['packages']),
key=itemgetter(1),
reverse=True)[:50]:
print(" * %s (%d)" % (package[0], package[1]))
print("Out of %d update submitters, the top 50 were:" %
(len(data['submitters'])))
for submitter in sorted(six.iteritems(data['submitters']),
key=itemgetter(1),
reverse=True)[:50]:
print(" * %s (%d)" % (submitter[0], submitter[1]))
print("Out of %d critical path updates, the top 50 updated were:" %
(len(data['critpath_pkgs'])))
for x in sorted(six.iteritems(data['critpath_pkgs']),
key=itemgetter(1),
reverse=True)[:50]:
print(" * %s (%d)" % (x[0], x[1]))
critpath_not_updated = set()
for pkg in critpath_pkgs:
if pkg not in data['critpath_pkgs']:
critpath_not_updated.add(pkg)
print("Out of %d critical path packages, %d were never updated:" %
(len(critpath_pkgs), len(critpath_not_updated)))
for pkg in sorted(critpath_not_updated):
print(' * %s' % pkg)
print()
print()
print("Out of %d total updates, %d received feedback (%0.2f%%)" %
(num_updates, feedback, (float(feedback) / num_updates * 100)))
print("Out of %d total unique commenters, the top 50 were:" % (len(karma)))
for submitter in sorted(six.iteritems(karma),
key=itemgetter(1),
reverse=True)[:50]:
print(" * %s (%d)" % (submitter[0], submitter[1]))