def __call__(self):
"""
Manage a database Session object for the life of the context.
Yields a database Session object, then either commits the tranaction if there were no
Exceptions or rolls back the transaction. In either case, it also will close and remove the
Session.
"""
session = Session()
try:
yield session
session.commit()
except Exception as e:
# It is possible for session.rolback() to raise Exceptions, so we will wrap it in an
# Exception handler as well so we can log the rollback failure and still raise the
# original Exception.
try:
session.rollback()
except Exception:
log.exception(
'An Exception was raised while rolling back a transaction.'
)
raise e
finally:
session.close()
Session.remove()
def dequeue_stable():
"""Convert all batched requests to stable requests."""
initialize_db(config.config)
buildsys.setup_buildsystem(config.config)
db = Session()
try:
batched = db.query(models.Update).filter_by(
request=models.UpdateRequest.batched).all()
for update in batched:
try:
update.set_request(db, models.UpdateRequest.stable, u'bodhi')
db.commit()
except Exception as e:
print('Unable to stabilize {}: {}'.format(
update.alias, str(e)))
db.rollback()
msg = u"Bodhi is unable to request this update for stabilization: {}"
update.comment(db, msg.format(str(e)), author=u'bodhi')
db.commit()
except Exception as e:
print(str(e))
sys.exit(1)
finally:
Session.remove()
def tearDown(self):
"""Roll back all the changes from the test and clean up the session."""
self._request_sesh.stop()
self.db.close()
self.transaction.rollback()
self.connection.close()
Session.remove()
def _setup_method(self):
"""Set up Bodhi for testing."""
self.config = testing.setUp()
self.app_settings = get_appsettings(os.environ["BODHI_CONFIG"])
config.config.clear()
config.config.load_config(self.app_settings)
# Ensure "cached" objects are cleared before each test.
models.Release.clear_all_releases_cache()
models.Release._tag_cache = None
if engine is None:
self.engine = _configure_test_db(config.config["sqlalchemy.url"])
else:
self.engine = engine
self.connection = self.engine.connect()
models.Base.metadata.create_all(bind=self.connection)
self.transaction = self.connection.begin()
Session.remove()
Session.configure(bind=self.engine,
autoflush=False,
expire_on_commit=False)
self.Session = Session
self.db = Session()
self.db.begin_nested()
if self._populate_db:
populate(self.db)
bugs.set_bugtracker()
buildsys.setup_buildsystem({'buildsystem': 'dev'})
self._request_sesh = mock.patch(
'bodhi.server.webapp._complete_database_session',
webapp._rollback_or_commit)
self._request_sesh.start()
# Create the test WSGI app one time. We should avoid creating too many
# of these since Pyramid holds global references to the objects it creates
# and this results in a substantial memory leak. Long term we should figure
# out how to make Pyramid forget about these.
global _app
if _app is None:
# We don't want to call Session.remove() during the unit tests, because that will
# trigger the restart_savepoint() callback defined above which will remove the data
# added by populate().
with mock.patch('bodhi.server.Session.remove'):
_app = TestApp(
main({},
testing='guest',
session=self.db,
**self.app_settings))
self.app = _app
self.registry = self.app.app.registry
# ensure a clean state of the dev build system
buildsys.DevBuildsys.clear()
def _end_session(self):
"""
Close and remove the session.
This has been split off the main __call__ method to make it easier to
mock it out in unit tests.
"""
Session.remove()
def setUp(self):
"""Set up Bodhi for testing."""
# Ensure "cached" objects are cleared before each test.
models.Release._all_releases = None
models.Release._tag_cache = None
if engine is None:
self.engine = _configure_test_db()
else:
self.engine = engine
self.connection = self.engine.connect()
models.Base.metadata.create_all(bind=self.connection)
self.transaction = self.connection.begin()
Session.remove()
Session.configure(bind=self.engine,
autoflush=False,
expire_on_commit=False)
self.Session = Session
self.db = Session()
self.db.begin_nested()
if self._populate_db:
populate(self.db)
bugs.set_bugtracker()
buildsys.setup_buildsystem({'buildsystem': 'dev'})
def request_db(request=None):
"""
Replace the db session function with one that doesn't close the session.
This allows tests to make assertions about the database. Without it, all
the changes would be rolled back to when the nested transaction is started.
"""
def cleanup(request):
if request.exception is not None:
Session().rollback()
else:
Session().commit()
request.add_finished_callback(cleanup)
return Session()
self._request_sesh = mock.patch(
'bodhi.server.get_db_session_for_request', request_db)
self._request_sesh.start()
# Create the test WSGI app one time. We should avoid creating too many
# of these since Pyramid holds global references to the objects it creates
# and this results in a substantial memory leak. Long term we should figure
# out how to make Pyramid forget about these.
global _app
if _app is None:
_app = TestApp(main({}, testing=u'guest', **self.app_settings))
self.app = _app
def test_empty_commit(self, mock_fedmsg_publish):
"""Assert calling commit on a session with no changes still triggers fedmsgs."""
# Ensure nothing at all is in our session
Session.remove()
session = Session()
notifications.publish('demo.topic', {'new': 'package'})
session.commit()
mock_fedmsg_publish.assert_called_once_with(topic='demo.topic',
msg={'new': 'package'})
def main(argv=sys.argv):
"""
Remove the pending and testing tags from branched updates.
Args:
argv (list): The arguments passed to the script. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging(config_uri)
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
koji = buildsys.get_session()
one_day = timedelta(days=1)
now = datetime.utcnow()
try:
for release in db.query(Release).filter_by(
state=ReleaseState.pending).all():
log.info(release.name)
for update in db.query(Update).filter_by(
release=release, status=UpdateStatus.stable).all():
if now - update.date_stable > one_day:
for build in update.builds:
tags = build.get_tags()
stable_tag = release.dist_tag
testing_tag = release.testing_tag
pending_signing_tag = release.pending_signing_tag
pending_testing_tag = release.pending_testing_tag
if stable_tag not in tags:
log.error('%s not tagged as stable %s' %
(build.nvr, tags))
continue
if testing_tag in tags:
log.info('Removing %s from %s' %
(testing_tag, build.nvr))
koji.untagBuild(testing_tag, build.nvr)
if pending_signing_tag in tags:
log.info('Removing %s from %s' %
(pending_signing_tag, build.nvr))
koji.untagBuild(pending_signing_tag, build.nvr)
if pending_testing_tag in tags:
log.info('Removing %s from %s' %
(pending_testing_tag, build.nvr))
koji.untagBuild(pending_testing_tag, build.nvr)
db.commit()
except Exception as e:
log.error(e)
db.rollback()
Session.remove()
sys.exit(1)
def main():
"""Wrap ``expire_overrides()``, catching exceptions."""
db = Session()
try:
expire_overrides(db)
db.commit()
except Exception:
log.exception("There was an error expiring overrides")
db.rollback()
Session.remove()
def main(argv=sys.argv):
"""
Search for overrides that are past their expiration date and mark them expired.
Args:
argv (list): The command line arguments. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging()
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
setup_buildsystem(settings)
try:
now = datetime.utcnow()
overrides = db.query(BuildrootOverride)
overrides = overrides.filter(BuildrootOverride.expired_date.is_(None))
overrides = overrides.filter(BuildrootOverride.expiration_date < now)
count = overrides.count()
if not count:
log.info("No active buildroot override to expire")
return
log.info("Expiring %d buildroot overrides...", count)
for override in overrides:
log.debug(
f"Expiring BRO for {override.build.nvr} because it's due to expire."
)
override.expire()
db.add(override)
log.info("Expired %s" % override.build.nvr)
db.commit()
except Exception as e:
log.error(e)
db.rollback()
Session.remove()
sys.exit(1)
def test_publish_sqlalchemy_object(self):
"""Assert publish places the message inside the session info dict."""
message = compose_schemas.ComposeSyncWaitV1.from_dict({
'agent': 'double O seven',
'repo': 'f30'
})
Session.remove()
notifications.publish(message)
session = Session()
assert 'messages' in session.info
assert len(session.info['messages']) == 1
msg = session.info['messages'][0]
assert msg == message
def test_publish_sqlalchemy_object(self):
"""Assert publish places the message inside the session info dict."""
message = compose_schemas.ComposeSyncWaitV1.from_dict({
'agent': 'double O seven',
'repo': 'f30'
})
Session.remove()
notifications.publish(message)
session = Session()
self.assertIn('messages', session.info)
self.assertEqual(len(session.info['messages']), 1)
msg = session.info['messages'][0]
self.assertEqual(msg, message)
def test_publish_sqlalchemy_object(self):
"""Assert publish places the message inside the session info dict."""
Session.remove()
expected_msg = {
u'some_package': {
u'name': u'so good',
u'type': 'base',
u'requirements': None
}
}
package = models.Package(name='so good')
notifications.publish('demo.topic', {'some_package': package})
session = Session()
self.assertIn('messages', session.info)
self.assertEqual(len(session.info['messages']), 1)
msg = session.info['messages'][0]
self.assertEqual(msg.body, expected_msg)
def main():
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, and run approve_update on
them.
"""
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
approve_update(update, db)
db.commit()
except Exception:
log.exception("There was an error approving testing updates.")
db.rollback()
Session.remove()
def test_publish_sqlalchemy_object(self, mock_init):
"""Assert publish places the message inside the session info dict."""
Session.remove()
expected_msg = {
u'some_package': {
u'name': u'so good',
u'type': 'base',
u'requirements': None,
u'stack': None,
u'stack_id': None,
}
}
package = models.Package(name='so good')
notifications.publish('demo.topic', {'some_package': package})
session = Session()
self.assertIn('fedmsg', session.info)
self.assertEqual(session.info['fedmsg']['demo.topic'], [expected_msg])
mock_init.assert_called_once_with()
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
buildsys.setup_buildsystem(config)
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(
f"{update.release.name} doesn't have mandatory days in testing"
)
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
# Only send email notification about the update reaching
# testing approval on releases composed by bodhi
update.comment(
db,
str(config.get('testing_approval_msg')),
author='bodhi',
email_notification=update.release.composed_by_bodhi)
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
# For now only rawhide update can be created using side tag
# Do not add the release.pending_stable_tag if the update
# was created from a side tag.
if update.release.composed_by_bodhi:
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
# For updates that are not included in composes run by bodhi itself,
# mark them as stable
else:
# Single and Multi build update
conflicting_builds = update.find_conflicting_builds()
if conflicting_builds:
builds_str = str.join(", ", conflicting_builds)
update.comment(
db, "This update cannot be pushed to stable. "
f"These builds {builds_str} have a more recent "
f"build in koji's {update.release.stable_tag} tag.",
author="bodhi")
update.request = None
if update.from_tag is not None:
update.status = UpdateStatus.pending
update.remove_tag(
update.release.get_testing_side_tag(
update.from_tag))
else:
update.status = UpdateStatus.obsolete
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
continue
update.add_tag(update.release.stable_tag)
update.status = UpdateStatus.stable
update.request = None
update.pushed = True
update.date_stable = update.date_pushed = func.current_timestamp(
)
update.comment(
db,
"This update has been submitted for stable by bodhi",
author=u'bodhi')
# Multi build update
if update.from_tag:
# Merging the side tag should happen here
pending_signing_tag = update.release.get_pending_signing_side_tag(
update.from_tag)
testing_tag = update.release.get_testing_side_tag(
update.from_tag)
update.remove_tag(pending_signing_tag)
update.remove_tag(testing_tag)
update.remove_tag(update.from_tag)
koji = buildsys.get_session()
koji.deleteTag(pending_signing_tag)
koji.deleteTag(testing_tag)
# Removes the tag and the build target from koji.
koji.removeSideTag(update.from_tag)
else:
# Single build update
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(
update.release.pending_stable_tag)
update.remove_tag(
update.release.pending_signing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
# If this release does not have any testing requirements, skip it
if not update.release.mandatory_days_in_testing:
print('%s doesn\'t have mandatory days in testing' %
update.release.name)
continue
# If this has already met testing requirements, skip it
if update.met_testing_requirements:
continue
# Approval message when testing based on karma threshold
if update.stable_karma not in (0, None) and update.karma >= update.stable_karma \
and not update.autokarma and update.meets_testing_requirements:
print('%s now reaches stable karma threshold' % update.title)
text = config.get('testing_approval_msg_based_on_karma')
update.comment(db, text, author=u'bodhi')
continue
# If autokarma updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print('%s now meets testing requirements' % update.title)
text = six.text_type(
config.get('testing_approval_msg') %
update.mandatory_days_in_testing)
update.comment(db, text, author=u'bodhi')
notifications.publish(topic='update.requirements_met.stable',
msg=dict(update=update))
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(
f"{update.release.name} doesn't have mandatory days in testing"
)
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
update.comment(db,
str(config.get('testing_approval_msg')),
author='bodhi')
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
