def dequeue_stable():
"""Convert all batched requests to stable requests."""
initialize_db(config.config)
buildsys.setup_buildsystem(config.config)
db = Session()
try:
batched = db.query(models.Update).filter_by(
request=models.UpdateRequest.batched).all()
for update in batched:
try:
update.set_request(db, models.UpdateRequest.stable, u'bodhi')
db.commit()
except Exception as e:
print('Unable to stabilize {}: {}'.format(
update.alias, str(e)))
db.rollback()
msg = u"Bodhi is unable to request this update for stabilization: {}"
update.comment(db, msg.format(str(e)), author=u'bodhi')
db.commit()
except Exception as e:
print(str(e))
sys.exit(1)
finally:
Session.remove()
def _setup_method(self):
"""Set up Bodhi for testing."""
self.config = testing.setUp()
self.app_settings = get_appsettings(os.environ["BODHI_CONFIG"])
config.config.clear()
config.config.load_config(self.app_settings)
# Ensure "cached" objects are cleared before each test.
models.Release.clear_all_releases_cache()
models.Release._tag_cache = None
if engine is None:
self.engine = _configure_test_db(config.config["sqlalchemy.url"])
else:
self.engine = engine
self.connection = self.engine.connect()
models.Base.metadata.create_all(bind=self.connection)
self.transaction = self.connection.begin()
Session.remove()
Session.configure(bind=self.engine,
autoflush=False,
expire_on_commit=False)
self.Session = Session
self.db = Session()
self.db.begin_nested()
if self._populate_db:
populate(self.db)
bugs.set_bugtracker()
buildsys.setup_buildsystem({'buildsystem': 'dev'})
self._request_sesh = mock.patch(
'bodhi.server.webapp._complete_database_session',
webapp._rollback_or_commit)
self._request_sesh.start()
# Create the test WSGI app one time. We should avoid creating too many
# of these since Pyramid holds global references to the objects it creates
# and this results in a substantial memory leak. Long term we should figure
# out how to make Pyramid forget about these.
global _app
if _app is None:
# We don't want to call Session.remove() during the unit tests, because that will
# trigger the restart_savepoint() callback defined above which will remove the data
# added by populate().
with mock.patch('bodhi.server.Session.remove'):
_app = TestApp(
main({},
testing='guest',
session=self.db,
**self.app_settings))
self.app = _app
self.registry = self.app.app.registry
# ensure a clean state of the dev build system
buildsys.DevBuildsys.clear()
def test_koji_buildsystem(self, mock_koji_login):
"""Assert the buildsystem initializes correctly for koji"""
config = {'buildsystem': 'koji', 'some': 'key'}
self.assertTrue(buildsys._buildsystem is None)
buildsys.setup_buildsystem(config)
self.assertFalse(buildsys._buildsystem is None)
buildsys._buildsystem()
mock_koji_login.assert_called_once_with(config=config)
def test_koji_buildsystem(self, mock_koji_login):
"""Assert the buildsystem initializes correctly for koji"""
config = {'buildsystem': 'koji', 'some': 'key'}
assert buildsys._buildsystem is None
buildsys.setup_buildsystem(config)
assert buildsys._buildsystem is not None
buildsys._buildsystem()
mock_koji_login.assert_called_once_with(authenticate=True,
config=config)
def setUp(self):
"""
Initialize our temporary repo.
"""
super(TestAddUpdate, self).setUp()
setup_buildsystem({'buildsystem': 'dev'})
self.tempdir = tempfile.mkdtemp('bodhi')
self.temprepo = join(self.tempdir, 'f17-updates-testing')
mkmetadatadir(join(self.temprepo, 'f17-updates-testing', 'i386'))
def setUp(self):
"""Set up Bodhi for testing."""
# Ensure "cached" objects are cleared before each test.
models.Release._all_releases = None
models.Release._tag_cache = None
if engine is None:
self.engine = _configure_test_db()
else:
self.engine = engine
self.connection = self.engine.connect()
models.Base.metadata.create_all(bind=self.connection)
self.transaction = self.connection.begin()
Session.remove()
Session.configure(bind=self.engine,
autoflush=False,
expire_on_commit=False)
self.Session = Session
self.db = Session()
self.db.begin_nested()
if self._populate_db:
populate(self.db)
bugs.set_bugtracker()
buildsys.setup_buildsystem({'buildsystem': 'dev'})
def request_db(request=None):
"""
Replace the db session function with one that doesn't close the session.
This allows tests to make assertions about the database. Without it, all
the changes would be rolled back to when the nested transaction is started.
"""
def cleanup(request):
if request.exception is not None:
Session().rollback()
else:
Session().commit()
request.add_finished_callback(cleanup)
return Session()
self._request_sesh = mock.patch(
'bodhi.server.get_db_session_for_request', request_db)
self._request_sesh.start()
# Create the test WSGI app one time. We should avoid creating too many
# of these since Pyramid holds global references to the objects it creates
# and this results in a substantial memory leak. Long term we should figure
# out how to make Pyramid forget about these.
global _app
if _app is None:
_app = TestApp(main({}, testing=u'guest', **self.app_settings))
self.app = _app
def test_authenticate_false(self, krb_login):
"""If authenticate is set to False, the Koji client should should be unauthenticated."""
config = {
'krb_ccache': 'a_ccache', 'krb_keytab': 'a_keytab', 'krb_principal': 'a_principal',
'buildsystem': 'koji', 'koji_hub': 'https://example.com/koji'}
self.assertTrue(buildsys._buildsystem is None)
buildsys.setup_buildsystem(config, authenticate=False)
# Instantiating the buildsystem should not cause a krb_login to happen.
buildsys._buildsystem()
self.assertEqual(krb_login.call_count, 0)
def setUp(self):
"""
Initialize our temporary repo.
"""
super(UpdateInfoMetadataTestCase, self).setUp()
setup_buildsystem({'buildsystem': 'dev'})
self.tempdir = tempfile.mkdtemp('bodhi')
self.tempcompdir = join(self.tempdir, 'f17-updates-testing')
self.temprepo = join(self.tempcompdir, 'compose', 'Everything', 'i386', 'os')
base.mkmetadatadir(join(self.temprepo, 'f17-updates-testing', 'i386'), updateinfo=False)
config['cache_dir'] = os.path.join(self.tempdir, 'cache')
os.makedirs(config['cache_dir'])
def __init__(self):
"""Set up the database, build system, bug tracker, and handlers."""
log.info('Initializing Bodhi')
initialize_db(config)
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
self.handler_infos = [
HandlerInfo('.buildsys.tag', "Signed", SignedHandler()),
HandlerInfo('.buildsys.tag', 'Automatic Update', AutomaticUpdateHandler()),
HandlerInfo('.greenwave.decision.update', 'Greenwave', GreenwaveHandler()),
HandlerInfo('.ci.koji-build.test.running', 'CI', CIHandler())
]
def test_authenticate_true(self, krb_login):
"""If authenticate is set to True, the Koji client should should be authenticated."""
config = {
'krb_ccache': 'a_ccache', 'krb_keytab': 'a_keytab', 'krb_principal': 'a_principal',
'buildsystem': 'koji', 'koji_hub': 'https://example.com/koji'}
self.assertTrue(buildsys._buildsystem is None)
buildsys.setup_buildsystem(config, authenticate=True)
# Instantiating the buildsystem should cause a krb_login to happen.
buildsys._buildsystem()
krb_login.assert_called_once_with(ccache='a_ccache', keytab='a_keytab',
principal='a_principal')
def __init__(self):
"""Set up the database, build system, bug tracker, and handlers."""
log.info('Initializing Bodhi')
initialize_db(config)
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
self.handler_infos = [
HandlerInfo('.buildsys.tag', "Signed", SignedHandler()),
HandlerInfo('.buildsys.tag', 'Automatic Update', AutomaticUpdateHandler()),
HandlerInfo('.ci.koji-build.test.running', 'CI', CIHandler()),
HandlerInfo('.waiverdb.waiver.new', 'WaiverDB', WaiverdbHandler()),
HandlerInfo('.resultsdb.result.new', 'ResultsDB', ResultsdbHandler()),
]
def __init__(self):
"""Set up the database, build system, bug tracker, and handlers."""
log.info('Initializing Bodhi')
initialize_db(config)
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
if ComposerHandler:
self.composer_handler = ComposerHandler()
else:
log.info(
'The composer is not installed - Bodhi will ignore composer.start messages.'
)
self.composer_handler = None
self.signed_handler = SignedHandler()
self.updates_handler = UpdatesHandler()
def monitor():
"""Print a simple human-readable report about existing Compose objects."""
initialize_db(config.config)
buildsys.setup_buildsystem(config.config)
click.echo('Locked updates: %s\n' %
models.Update.query.filter_by(locked=True).count())
for c in sorted([c for c in models.Compose.query.all()]):
click.echo(c)
for attr in ('state', 'state_date', 'security', 'error_message'):
if getattr(c, attr) is not None:
click.echo('\t%s: %s' % (attr, getattr(c, attr)))
click.echo('\tcheckpoints: %s' %
', '.join(json.loads(c.checkpoints).keys()))
click.echo('\tlen(updates): %s\n' % len(c.updates))
def update_sig_status(update):
"""Update build signature status for builds in update."""
global _koji
if _koji is None:
# We don't want to authenticate to the buildsystem, because this script is often mistakenly
# run as root and this can cause the ticket cache to become root owned with 0600 perms,
# which will cause the compose to fail when it tries to use it to authenticate to Koji.
buildsys.setup_buildsystem(config, authenticate=False)
_koji = get_koji(None)
for build in update.builds:
if not build.signed:
build_tags = build.get_tags(_koji)
if update.release.pending_signing_tag not in build_tags:
click.echo('Build %s was refreshed as signed' % build.nvr)
build.signed = True
else:
click.echo('Build %s still unsigned' % build.nvr)
def __init__(self, hub, db_factory=None, mash_dir=config.get('mash_dir'),
*args, **kw):
if not db_factory:
config_uri = '/etc/bodhi/production.ini'
settings = get_appsettings(config_uri)
engine = engine_from_config(settings, 'sqlalchemy.')
Base.metadata.create_all(engine)
self.db_factory = transactional_session_maker()
else:
self.db_factory = db_factory
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
self.mash_dir = mash_dir
prefix = hub.config.get('topic_prefix')
env = hub.config.get('environment')
self.topic = prefix + '.' + env + '.' + hub.config.get('masher_topic')
self.valid_signer = hub.config.get('releng_fedmsg_certname')
if not self.valid_signer:
log.warn('No releng_fedmsg_certname defined'
'Cert validation disabled')
super(Masher, self).__init__(hub, *args, **kw)
log.info('Bodhi masher listening on topic: %s' % self.topic)
def setUp(self):
setup_buildsystem({'buildsystem': 'dev'})
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
buildsys.setup_buildsystem(config)
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(
f"{update.release.name} doesn't have mandatory days in testing"
)
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
# Only send email notification about the update reaching
# testing approval on releases composed by bodhi
update.comment(
db,
str(config.get('testing_approval_msg')),
author='bodhi',
email_notification=update.release.composed_by_bodhi)
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
# For now only rawhide update can be created using side tag
# Do not add the release.pending_stable_tag if the update
# was created from a side tag.
if update.release.composed_by_bodhi:
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
# For updates that are not included in composes run by bodhi itself,
# mark them as stable
else:
# Single and Multi build update
conflicting_builds = update.find_conflicting_builds()
if conflicting_builds:
builds_str = str.join(", ", conflicting_builds)
update.comment(
db, "This update cannot be pushed to stable. "
f"These builds {builds_str} have a more recent "
f"build in koji's {update.release.stable_tag} tag.",
author="bodhi")
update.request = None
if update.from_tag is not None:
update.status = UpdateStatus.pending
update.remove_tag(
update.release.get_testing_side_tag(
update.from_tag))
else:
update.status = UpdateStatus.obsolete
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
continue
update.add_tag(update.release.stable_tag)
update.status = UpdateStatus.stable
update.request = None
update.pushed = True
update.date_stable = update.date_pushed = func.current_timestamp(
)
update.comment(
db,
"This update has been submitted for stable by bodhi",
author=u'bodhi')
# Multi build update
if update.from_tag:
# Merging the side tag should happen here
pending_signing_tag = update.release.get_pending_signing_side_tag(
update.from_tag)
testing_tag = update.release.get_testing_side_tag(
update.from_tag)
update.remove_tag(pending_signing_tag)
update.remove_tag(testing_tag)
update.remove_tag(update.from_tag)
koji = buildsys.get_session()
koji.deleteTag(pending_signing_tag)
koji.deleteTag(testing_tag)
# Removes the tag and the build target from koji.
koji.removeSideTag(update.from_tag)
else:
# Single build update
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(
update.release.pending_stable_tag)
update.remove_tag(
update.release.pending_signing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def test_dev_buildsystem(self):
"""Assert the buildsystem initializes correctly for dev"""
self.assertTrue(buildsys._buildsystem is None)
buildsys.setup_buildsystem({'buildsystem': 'dev'})
self.assertTrue(buildsys._buildsystem is buildsys.DevBuildsys)
def test_initialized_buildsystem(self):
"""Assert nothing happens when the buildsystem is already initialized"""
old_buildsystem = buildsys._buildsystem
buildsys.setup_buildsystem({})
self.assertTrue(old_buildsystem is buildsys._buildsystem)
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
buildsys.setup_buildsystem(config)
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(f"{update.release.name} doesn't have mandatory days in testing")
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
# Only send email notification about the update reaching
# testing approval on releases composed by bodhi
update.comment(
db,
str(config.get('testing_approval_msg')),
author='bodhi',
email_notification=update.release.composed_by_bodhi
)
notifications.publish(update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
update.set_request(db=db, action=UpdateRequest.stable, username="******")
# For updates that are not included in composes run by bodhi itself,
# mark them as stable
if not update.release.composed_by_bodhi:
update.add_tag(update.release.stable_tag)
update.remove_tag(update.release.pending_testing_tag)
update.remove_tag(update.release.pending_stable_tag)
update.status = UpdateStatus.stable
update.date_stable = datetime.datetime.utcnow()
update.request = None
update.date_pushed = datetime.datetime.utcnow()
update.pushed = True
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def setup_buildsys():
"""Initialize buildsystem instance from bodhi_config."""
buildsys.setup_buildsystem(bodhi_config)
def setUp(self):
"""Set up the buildsys."""
super(TestGetAllPackages, self).setUp()
buildsys.setup_buildsystem({'buildsystem': 'dev'})
def main(global_config, testing=None, session=None, **settings):
"""
Return a WSGI application.
Args:
global_config (dict): A dictionary with two keys: __file__, a path to the ini file, and
here, the path to the code.
testing (bool or None): Whether or not we are in testing mode.
session (sqlalchemy.orm.session.Session or None): If given, the session will be used instead
of building a new one.
settings (dictionary): Unused.
Returns:
pyramid.router.Router: A WSGI app.
"""
if settings:
bodhi_config.load_config(settings)
# Setup our bugtracker and buildsystem
bugs.set_bugtracker()
buildsys.setup_buildsystem(bodhi_config)
# Sessions & Caching
from pyramid.session import SignedCookieSessionFactory
session_factory = SignedCookieSessionFactory(
bodhi_config['session.secret'])
# Construct a list of all groups we're interested in
default = []
for key in ('important_groups', 'admin_packager_groups',
'mandatory_packager_groups', 'admin_groups'):
default.extend(bodhi_config.get(key))
# pyramid_fas_openid looks for this setting
bodhi_config['openid.groups'] = bodhi_config.get('openid.groups', default)
config = Configurator(settings=bodhi_config,
session_factory=session_factory)
# Plugins
config.include('pyramid_mako')
config.include('cornice')
# Initialize the database scoped session
initialize_db(bodhi_config)
# Lazy-loaded memoized request properties
if session:
config.registry.sessionmaker = lambda: session
else:
config.registry.sessionmaker = Session
config.add_request_method(get_db_session_for_request, 'db', reify=True)
config.add_request_method(get_user, 'user', reify=True)
config.add_request_method(get_koji, 'koji', reify=True)
config.add_request_method(get_cacheregion, 'cache', reify=True)
config.add_request_method(get_buildinfo, 'buildinfo', reify=True)
config.add_request_method(get_releases, 'releases', reify=True)
# Templating
config.add_mako_renderer('.html', settings_prefix='mako.')
config.add_static_view('static', 'bodhi:server/static')
from bodhi.server.renderers import rss, jpeg
config.add_renderer('rss', rss)
config.add_renderer('jpeg', jpeg)
config.add_renderer('jsonp', JSONP(param_name='callback'))
# i18n
config.add_translation_dirs('bodhi:server/locale/')
# Authentication & Authorization
if testing:
# use a permissive security policy while running unit tests
config.testing_securitypolicy(userid=testing, permissive=True)
else:
timeout = bodhi_config.get('authtkt.timeout')
config.set_authentication_policy(
AuthTktAuthenticationPolicy(bodhi_config['authtkt.secret'],
callback=groupfinder,
secure=bodhi_config['authtkt.secure'],
hashalg='sha512',
timeout=timeout,
max_age=timeout))
config.set_authorization_policy(ACLAuthorizationPolicy())
# Frontpage
config.add_route('home', '/')
# Views for creating new objects
config.add_route('new_update', '/updates/new')
config.add_route('new_override', '/overrides/new')
config.add_route('new_stack', '/stacks/new')
# Metrics
config.add_route('metrics', '/metrics')
config.add_route('masher_status', '/masher/')
# Auto-completion search
config.add_route('search_packages', '/search/packages')
config.add_route('latest_candidates', '/latest_candidates')
config.add_route('latest_builds', '/latest_builds')
config.add_route('captcha_image', '/captcha/{cipherkey}/')
# pyramid.openid
config.add_route('login', '/login')
config.add_view('bodhi.server.security.login', route_name='login')
config.add_view('bodhi.server.security.login', context=HTTPForbidden)
config.add_route('logout', '/logout')
config.add_view('bodhi.server.security.logout', route_name='logout')
config.add_route('verify_openid', pattern='/dologin.html')
config.add_view('pyramid_fas_openid.verify_openid',
route_name='verify_openid')
config.add_route('api_version', '/api_version')
# The only user preference we have.
config.add_route('popup_toggle', '/popup_toggle')
config.scan('bodhi.server.views')
config.scan('bodhi.server.services')
config.scan('bodhi.server.captcha')
return config.make_wsgi_app()
def _do_init():
config.load_config()
initialize_db(config)
buildsys.setup_buildsystem(config)
bugs.set_bugtracker()
