def publish(topic, msg, force=False):
"""
Send a message via Fedora Messaging.
This is used to send a message to the AMQP broker.
Args:
topic (str): The topic suffix. The "bodhi" prefix is applied (along with
the "topic_prefix" settings from Fedora Messaging).
msg (dict): The message body to send.
force (bool): If False (the default), the message is only sent after the
currently active database transaction successfully commits. If true,
the messages is sent immediately.
"""
# Dirty, nasty hack that I feel shame for: use the fedmsg encoder that modifies
# messages quietly if they have objects with __json__ methods on them.
# For now, copy that behavior. In the future, callers should pass
# fedora_messaging.api.Message sub-classes or this whole API should go away.
body = json.loads(json.dumps(msg, cls=FedMsgEncoder))
message = api.Message(topic="bodhi.{}".format(topic), body=body)
if force:
api.publish(message)
return
session = Session()
if 'messages' not in session.info:
session.info['messages'] = []
session.info['messages'].append(message)
_log.debug('Queuing message %r for delivery on session commit', message.id)
def __call__(self):
"""
Manage a database Session object for the life of the context.
Yields a database Session object, then either commits the tranaction if there were no
Exceptions or rolls back the transaction. In either case, it also will close and remove the
Session.
"""
session = Session()
try:
yield session
session.commit()
except Exception as e:
# It is possible for session.rolback() to raise Exceptions, so we will wrap it in an
# Exception handler as well so we can log the rollback failure and still raise the
# original Exception.
try:
session.rollback()
except Exception:
log.exception(
'An Exception was raised while rolling back a transaction.'
)
raise e
finally:
session.close()
Session.remove()
def tearDown(self):
"""Roll back all the changes from the test and clean up the session."""
self._request_sesh.stop()
self.db.close()
self.transaction.rollback()
self.connection.close()
Session.remove()
def test_repeated_commit(self, mock_fedmsg_publish):
"""Assert queued fedmsgs are cleared between commits."""
session = Session()
notifications.publish('demo.topic', {'new': 'package'})
session.commit()
session.commit()
mock_fedmsg_publish.assert_called_once_with(topic='demo.topic',
msg={'new': 'package'})
def _end_session(self):
"""
Close and remove the session.
This has been split off the main __call__ method to make it easier to
mock it out in unit tests.
"""
Session.remove()
def main(argv=sys.argv):
"""
Remove the pending and testing tags from branched updates.
Args:
argv (list): The arguments passed to the script. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging(config_uri)
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
koji = buildsys.get_session()
one_day = timedelta(days=1)
now = datetime.utcnow()
try:
for release in db.query(Release).filter_by(
state=ReleaseState.pending).all():
log.info(release.name)
for update in db.query(Update).filter_by(
release=release, status=UpdateStatus.stable).all():
if now - update.date_stable > one_day:
for build in update.builds:
tags = build.get_tags()
stable_tag = release.dist_tag
testing_tag = release.testing_tag
pending_signing_tag = release.pending_signing_tag
pending_testing_tag = release.pending_testing_tag
if stable_tag not in tags:
log.error('%s not tagged as stable %s' %
(build.nvr, tags))
continue
if testing_tag in tags:
log.info('Removing %s from %s' %
(testing_tag, build.nvr))
koji.untagBuild(testing_tag, build.nvr)
if pending_signing_tag in tags:
log.info('Removing %s from %s' %
(pending_signing_tag, build.nvr))
koji.untagBuild(pending_signing_tag, build.nvr)
if pending_testing_tag in tags:
log.info('Removing %s from %s' %
(pending_testing_tag, build.nvr))
koji.untagBuild(pending_testing_tag, build.nvr)
db.commit()
except Exception as e:
log.error(e)
db.rollback()
Session.remove()
sys.exit(1)
def test_clear_messages_on_send(self):
"""Assert the message queue is cleared after the event handler runs."""
session = Session()
session.info['messages'] = [api.Message()]
with fml_testing.mock_sends(api.Message()):
notifications.send_messages_after_commit(session)
assert session.info['messages'] == []
def test_error_logged(self, mock_log, mock_pub):
session = Session()
message = api.Message()
session.info['messages'] = [message]
mock_pub.side_effect = fml_exceptions.BaseException()
notifications.send_messages_after_commit(session)
mock_log.exception.assert_called_once_with(
"An error occurred publishing %r after a database commit", message)
def test_single_topic_many_messages(self, mock_fedmsg_publish):
"""Assert many messages for a single topic are sent."""
session = Session()
notifications.publish('demo.topic', {'new': 'package'})
notifications.publish('demo.topic', {'newer': 'packager'})
session.commit()
self.assertEqual(2, mock_fedmsg_publish.call_count)
mock_fedmsg_publish.assert_any_call(topic='demo.topic',
msg={'new': 'package'})
mock_fedmsg_publish.assert_any_call(topic='demo.topic',
msg={'newer': 'packager'})
def test_multiple_topics(self, mock_fedmsg_publish):
"""Assert messages with different topics are sent."""
session = Session()
notifications.publish('demo.topic', {'new': 'package'})
notifications.publish('other.topic', {'newer': 'packager'})
session.commit()
self.assertEqual(2, mock_fedmsg_publish.call_count)
mock_fedmsg_publish.assert_any_call(topic='demo.topic',
msg={'new': 'package'})
mock_fedmsg_publish.assert_any_call(topic='other.topic',
msg={'newer': 'packager'})
def test_publish_sqlalchemy_object(self):
"""Assert publish places the message inside the session info dict."""
message = compose_schemas.ComposeSyncWaitV1.from_dict({
'agent': 'double O seven',
'repo': 'f30'
})
Session.remove()
notifications.publish(message)
session = Session()
self.assertIn('messages', session.info)
self.assertEqual(len(session.info['messages']), 1)
msg = session.info['messages'][0]
self.assertEqual(msg, message)
def test_publish_sqlalchemy_object(self):
"""Assert publish places the message inside the session info dict."""
message = compose_schemas.ComposeSyncWaitV1.from_dict({
'agent': 'double O seven',
'repo': 'f30'
})
Session.remove()
notifications.publish(message)
session = Session()
assert 'messages' in session.info
assert len(session.info['messages']) == 1
msg = session.info['messages'][0]
assert msg == message
def expire_overrides(db: Session):
"""Search for overrides that are past their expiration date and mark them expired."""
now = datetime.utcnow()
overrides = db.query(BuildrootOverride)
overrides = overrides.filter(BuildrootOverride.expired_date.is_(None))
overrides = overrides.filter(BuildrootOverride.expiration_date < now)
count = overrides.count()
if not count:
log.info("No active buildroot override to expire")
return
log.info("Expiring %d buildroot overrides...", count)
for override in overrides:
log.debug(f"Expiring BRO for {override.build.nvr} because it's due to expire.")
override.expire()
db.add(override)
log.info("Expired %s" % override.build.nvr)
def test_publish_force(self, mock_fedmsg_publish, mock_init):
"""Assert publish with the force flag sends the message immediately."""
notifications.publish('demo.topic', {'such': 'important'}, force=True)
session = Session()
self.assertEqual(dict(), session.info)
mock_fedmsg_publish.assert_called_once_with(topic='demo.topic',
msg={'such': 'important'})
mock_init.assert_called_once_with()
def test_publish_sqlalchemy_object(self):
"""Assert publish places the message inside the session info dict."""
Session.remove()
expected_msg = {
u'some_package': {
u'name': u'so good',
u'type': 'base',
u'requirements': None
}
}
package = models.Package(name='so good')
notifications.publish('demo.topic', {'some_package': package})
session = Session()
self.assertIn('messages', session.info)
self.assertEqual(len(session.info['messages']), 1)
msg = session.info['messages'][0]
self.assertEqual(msg.body, expected_msg)
def main():
"""Wrap ``expire_overrides()``, catching exceptions."""
db = Session()
try:
expire_overrides(db)
db.commit()
except Exception:
log.exception("There was an error expiring overrides")
db.rollback()
Session.remove()
def test_commit_aborted(self, mock_fedmsg_publish):
"""Assert that when commits are aborted, messages aren't sent."""
session = Session()
session.add(models.Package(name=u'ejabberd'))
session.commit()
session.add(models.Package(name=u'ejabberd'))
notifications.publish('demo.topic', {'new': 'package'})
self.assertRaises(exc.IntegrityError, session.commit)
self.assertEqual(0, mock_fedmsg_publish.call_count)
def test_publish(self):
"""Assert publish places the message inside the session info dict."""
notifications.publish('demo.topic', {'such': 'important'})
session = Session()
self.assertIn('messages', session.info)
self.assertEqual(len(session.info['messages']), 1)
msg = session.info['messages'][0]
self.assertEqual(msg.topic, 'bodhi.demo.topic')
self.assertEqual(msg.body, {'such': 'important'})
def test_publish(self, mock_init):
"""Assert publish places the message inside the session info dict."""
notifications.publish('demo.topic', {'such': 'important'})
session = Session()
self.assertIn('fedmsg', session.info)
self.assertEqual(session.info['fedmsg']['demo.topic'],
[{
'such': 'important'
}])
def test_publish_sqlalchemy_object(self, mock_init):
"""Assert publish places the message inside the session info dict."""
Session.remove()
expected_msg = {
u'some_package': {
u'name': u'so good',
u'type': 'base',
u'requirements': None,
u'stack': None,
u'stack_id': None,
}
}
package = models.Package(name='so good')
notifications.publish('demo.topic', {'some_package': package})
session = Session()
self.assertIn('fedmsg', session.info)
self.assertEqual(session.info['fedmsg']['demo.topic'], [expected_msg])
mock_init.assert_called_once_with()
def publish(topic, msg, force=False):
"""Publish a message to fedmsg.
By default, messages are not sent immediately, but are queued in a
transaction "data manager". They will only get published after the
sqlalchemy transaction completes successfully and will not be published at
all if it fails, aborts, or rolls back.
Specifying force=True to this function by-passes that -- messages are sent
immediately.
Args:
msg (dict): A dictionary representing the message to be published via fedmsg.
force (bool): If True, send the message immediately. Else, queue the message to be sent
after the current database transaction is committed.
"""
if not bodhi.server.config.config.get('fedmsg_enabled'):
_log.warn("fedmsg disabled. not sending %r" % topic)
return
# Initialize right before we try to publish, but only if we haven't
# initialized for this thread already.
if not fedmsg_is_initialized():
init()
if force:
_log.debug("fedmsg skipping transaction and sending %r" % topic)
fedmsg.publish(topic=topic, msg=msg)
else:
# We need to do this to ensure all the SQLAlchemy objects that could be in the messages
# are turned into JSON before the session is removed and expires the objects loaded with
# it. The JSON is decoded again because the fedmsg API doesn't state it accepts strings.
# An issue has been filed about this: https://github.com/fedora-infra/fedmsg/issues/407.
json_msg = fedmsg.encoding.dumps(msg)
msg_dict = json.loads(json_msg)
# This gives us the thread-local session which we'll use to stash the fedmsg.
# When commit is called on it, the :func:`send_fedmsgs_after_commit` is triggered.
session = Session()
if 'fedmsg' not in session.info:
_log.debug('Adding a dictionary for fedmsg storage to %r', session)
session.info['fedmsg'] = collections.defaultdict(list)
session.info['fedmsg'][topic].append(msg_dict)
_log.debug('Enqueuing a fedmsg, %r, for topic "%s" on %r', msg_dict,
topic, session)
def publish(message: 'base.BodhiMessage', force: bool = False):
"""
Send a message via Fedora Messaging.
This is used to send a message to the AMQP broker.
Args:
message: The Message you wish to publish.
force: If False (the default), the message is only sent after the
currently active database transaction successfully commits. If true,
the messages is sent immediately.
"""
if force:
_publish_with_retry(message)
return
session = Session()
if 'messages' not in session.info:
session.info['messages'] = []
session.info['messages'].append(message)
_log.debug('Queuing message %r for delivery on session commit', message.id)
def main():
"""Check the enforced policies by Greenwave for each open update."""
session = Session()
updates = models.Update.query.filter(
models.Update.status.in_(
[models.UpdateStatus.pending, models.UpdateStatus.testing])
).filter(models.Update.release_id == models.Release.id).filter(
models.Release.state.in_([
models.ReleaseState.current, models.ReleaseState.pending,
models.ReleaseState.frozen
])
).order_by(
# Check the older updates first so there is more time for the newer to
# get their test results
models.Update.id.asc())
for update in updates:
try:
update.update_test_gating_status()
session.commit()
except Exception:
# If there is a problem talking to Greenwave server, print the error.
log.exception(
f"There was an error checking the policy for {update.alias}")
session.rollback()
def test_no_fedmsgs(self, mock_fedmsg_publish):
"""Assert nothing happens if messages are not explicitly published."""
session = Session()
session.add(models.Package(name=u'ejabberd'))
session.commit()
self.assertEqual(0, mock_fedmsg_publish.call_count)
def check():
"""Check the enforced policies by Greenwave for each open update."""
initialize_db(config.config)
session = Session()
updates = models.Update.query.filter(
models.Update.status.in_(
[models.UpdateStatus.pending, models.UpdateStatus.testing])
).filter(
models.Update.release_id == models.Release.id
).filter(
models.Release.state.in_(
[models.ReleaseState.current, models.ReleaseState.pending])
).order_by(
# Check the older updates first so there is more time for the newer to
# get their test results
models.Update.id.asc()
)
for update in updates:
try:
update.update_test_gating_status()
session.commit()
except Exception as e:
# If there is a problem talking to Greenwave server, print the error.
click.echo(str(e))
session.rollback()
def main():
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, and run approve_update on
them.
"""
db = Session()
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
approve_update(update, db)
db.commit()
except Exception:
log.exception("There was an error approving testing updates.")
db.rollback()
Session.remove()
def _setup_method(self):
"""Set up Bodhi for testing."""
self.config = testing.setUp()
self.app_settings = get_appsettings(os.environ["BODHI_CONFIG"])
config.config.clear()
config.config.load_config(self.app_settings)
# Ensure "cached" objects are cleared before each test.
models.Release.clear_all_releases_cache()
models.Release._tag_cache = None
if engine is None:
self.engine = _configure_test_db(config.config["sqlalchemy.url"])
else:
self.engine = engine
self.connection = self.engine.connect()
models.Base.metadata.create_all(bind=self.connection)
self.transaction = self.connection.begin()
Session.remove()
Session.configure(bind=self.engine,
autoflush=False,
expire_on_commit=False)
self.Session = Session
self.db = Session()
self.db.begin_nested()
if self._populate_db:
populate(self.db)
bugs.set_bugtracker()
buildsys.setup_buildsystem({'buildsystem': 'dev'})
self._request_sesh = mock.patch(
'bodhi.server.webapp._complete_database_session',
webapp._rollback_or_commit)
self._request_sesh.start()
# Create the test WSGI app one time. We should avoid creating too many
# of these since Pyramid holds global references to the objects it creates
# and this results in a substantial memory leak. Long term we should figure
# out how to make Pyramid forget about these.
global _app
if _app is None:
# We don't want to call Session.remove() during the unit tests, because that will
# trigger the restart_savepoint() callback defined above which will remove the data
# added by populate().
with mock.patch('bodhi.server.Session.remove'):
_app = TestApp(
main({},
testing='guest',
session=self.db,
**self.app_settings))
self.app = _app
self.registry = self.app.app.registry
# ensure a clean state of the dev build system
buildsys.DevBuildsys.clear()
def test_single_topic_one_message(self, mock_fedmsg_publish):
"""Assert a single message for a single topic is published."""
session = Session()
session.add(models.Package(name=u'ejabberd'))
notifications.publish('demo.topic', {'new': 'package'})
session.commit()
mock_fedmsg_publish.assert_called_once_with(topic='demo.topic',
msg={'new': 'package'})
def request_db(request=None):
"""
Replace the db session function with one that doesn't close the session.
This allows tests to make assertions about the database. Without it, all
the changes would be rolled back to when the nested transaction is started.
"""
def cleanup(request):
if request.exception is not None:
Session().rollback()
else:
Session().commit()
request.add_finished_callback(cleanup)
return Session()
def test_empty_commit(self, mock_fedmsg_publish):
"""Assert calling commit on a session with no changes still triggers fedmsgs."""
# Ensure nothing at all is in our session
Session.remove()
session = Session()
notifications.publish('demo.topic', {'new': 'package'})
session.commit()
mock_fedmsg_publish.assert_called_once_with(topic='demo.topic',
msg={'new': 'package'})
