def test_publish_to_config_exchange(self):
"""Assert a message can be published to the exchange form config."""
message = "test_message"
expected_pre_publish_signal_data = {
"called": True,
"sender": api.publish,
"args": {"message": message},
}
expected_publish_signal_data = {
"called": True,
"sender": api.publish,
"args": {"message": message},
}
expected_publish_failed_signal_data = {
"called": False,
"sender": None,
"args": None,
}
api.publish(message)
self.mock_publisher_session.assert_called_once_with()
self.mock_publisher_session.publish.assert_called_once_with(
message, exchange="test_publich_exchange"
)
self.assertEqual(self.pre_publish_signal_data, expected_pre_publish_signal_data)
self.assertEqual(self.publish_signal_data, expected_publish_signal_data)
self.assertEqual(
self.publish_failed_signal_data, expected_publish_failed_signal_data
)
def test_twisted_consume_update_callback():
"""Assert a second call to consume updates an existing callback."""
queue = str(uuid.uuid4())
queues = {queue: {"auto_delete": False, "arguments": {"x-expires": 60 * 1000}}}
bindings = [{"queue": queue, "exchange": "amq.topic", "routing_keys": ["#"]}]
callback1 = defer.Deferred()
callback2 = defer.Deferred()
consumers1 = yield api.twisted_consume(
lambda m: reactor.callFromThread(callback1.callback, m), bindings, queues
)
api.publish(message.Message(), "amq.topic")
_add_timeout(callback1, 10)
try:
yield callback1
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Never received message for initial callback")
consumers2 = yield api.twisted_consume(
lambda m: reactor.callFromThread(callback2.callback, m), bindings, queues
)
api.publish(message.Message(), "amq.topic")
_add_timeout(callback2, 10)
try:
yield callback2
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Never received message for updated callback")
assert consumers1[0]._tag == consumers2[0]._tag
yield consumers2[0].cancel()
def fedmsg_publish(*args, **kwargs): # pragma: no cover
""" Try to publish a message on the fedmsg bus. """
if config.config["LEGACY_MESSAGING"]:
# We catch Exception if we want :-p
# pylint: disable=W0703
# Ignore message about fedmsg import
# pylint: disable=F0401
kwargs["modname"] = "anitya"
kwargs["cert_prefix"] = "anitya"
kwargs["name"] = "relay_inbound"
kwargs["active"] = True
try:
import fedmsg
fedmsg.publish(*args, **kwargs)
except Exception as err:
_log.error(str(err))
else:
try:
message_class = message.get_class("anitya." + kwargs["topic"])
api.publish(
message_class(
topic="anitya.{}".format(kwargs["topic"]), body=kwargs["msg"]
)
)
except (
fm_exceptions.ConnectionException,
fm_exceptions.PublishException,
) as err:
# For now, continue just logging the error. Once the messaging has
# been untangled into SQLAlchemy events, it should probably result
# in an exception and the client should try again later.
_log.error(str(err))
def test_publish_to_exchange(self, mock_twisted_publish):
"""Assert a message can be published to the exchange."""
message = "test_message"
exchange = "test_exchange"
expected_pre_publish_signal_data = {
"called": True,
"sender": api.publish,
"args": {"message": message},
}
expected_publish_signal_data = {
"called": True,
"sender": api.publish,
"args": {"message": message},
}
expected_publish_failed_signal_data = {
"called": False,
"sender": None,
"args": None,
}
api.publish(message, exchange)
mock_twisted_publish.assert_called_once_with(message, exchange)
mock_twisted_publish.return_value.wait.assert_called_once_with(timeout=30)
self.assertEqual(self.pre_publish_signal_data, expected_pre_publish_signal_data)
self.assertEqual(self.publish_signal_data, expected_publish_signal_data)
self.assertEqual(
self.publish_failed_signal_data, expected_publish_failed_signal_data
)
def test_twisted_consume_halt_consumer_requeue():
"""Assert raising HaltConsumer with requeue=True re-queues the message."""
queue = str(uuid.uuid4())
queues = {queue: {"auto_delete": False, "arguments": {"x-expires": 60 * 1000}}}
bindings = [{"queue": queue, "exchange": "amq.topic", "routing_keys": ["#"]}]
msg = message.Message(
topic=u"nice.message",
headers={u"niceness": u"very"},
body={u"encouragement": u"You're doing great!"},
)
def callback(message):
"""Count to 3 and quit."""
raise exceptions.HaltConsumer(exit_code=1, requeue=True)
# Assert that the number of consumers we think we started is the number the
# server things we started. This will fail if other tests don't clean up properly.
# If it becomes problematic perhaps each test should have a vhost.
consumers = yield api.twisted_consume(callback, bindings, queues)
api.publish(msg, "amq.topic")
_add_timeout(consumers[0].result, 10)
try:
yield consumers[0].result
except exceptions.HaltConsumer as e:
# Assert there are no consumers for the queue, and that there's a ready message
assert e.exit_code == 1
server_queue = yield get_queue(queue)
assert server_queue["consumers"] == 0
assert server_queue["messages_ready"] == 1
except (defer.TimeoutError, defer.CancelledError):
yield consumers[0].cancel()
pytest.fail("Timeout reached without consumer halting!")
def test_pub_sub_default_settings(self):
"""
Assert publishing and subscribing works with the default configuration.
This should work because the publisher uses the 'amq.topic' exchange by
default and the consumer also uses the 'amq.topic' exchange with its
auto-named queue and a default subscription key of '#'.
"""
# Consumer setup
def counting_callback(message, storage=defaultdict(int)):
storage[message.topic] += 1
if storage[message.topic] == 3:
raise exceptions.HaltConsumer()
consumer_process = multiprocessing.Process(
target=api.consume, args=(counting_callback,))
msg = message.Message(topic=u'nice.message', headers={u'niceness': u'very'},
body={u'encouragement': u"You're doing great!"})
consumer_process.start()
# Allow the consumer time to create the queues and bindings
time.sleep(5)
for _ in range(0, 3):
api.publish(msg)
consumer_process.join(timeout=30)
self.assertEqual(0, consumer_process.exitcode)
def test_session_cache_has_session(self):
"""Assert a TLS vaiable _session_cache contains 'session'."""
message = "test_message"
exchange = "test_exchange"
expected_pre_publish_signal_data = {
"called": True,
"sender": api.publish,
"args": {"message": message},
}
expected_publish_signal_data = {
"called": True,
"sender": api.publish,
"args": {"message": message},
}
expected_publish_failed_signal_data = {
"called": False,
"sender": None,
"args": None,
}
self.mock_session_cache.session = self.mock_publisher_session
api.publish(message, exchange)
self.mock_publisher_session.assert_not_called()
self.mock_publisher_session.publish.assert_called_once_with(
message, exchange=exchange
)
self.assertEqual(self.pre_publish_signal_data, expected_pre_publish_signal_data)
self.assertEqual(self.publish_signal_data, expected_publish_signal_data)
self.assertEqual(
self.publish_failed_signal_data, expected_publish_failed_signal_data
)
def fedmsg_publish(*args, **kwargs): # pragma: no cover
""" Try to publish a message on the fedmsg bus. """
if config.config["LEGACY_MESSAGING"]:
# We catch Exception if we want :-p
# pylint: disable=W0703
# Ignore message about fedmsg import
# pylint: disable=F0401
kwargs["modname"] = "anitya"
kwargs["cert_prefix"] = "anitya"
kwargs["name"] = "relay_inbound"
kwargs["active"] = True
try:
import fedmsg
fedmsg.publish(*args, **kwargs)
except Exception as err:
_log.error(str(err))
else:
try:
message_class = message.get_class("anitya." + kwargs["topic"])
api.publish(
message_class(
topic="anitya.{}".format(kwargs["topic"]), body=kwargs["msg"]
)
)
except (
fm_exceptions.ConnectionException,
fm_exceptions.PublishException,
) as err:
# For now, continue just logging the error. Once the messaging has
# been untangled into SQLAlchemy events, it should probably result
# in an exception and the client should try again later.
_log.error(str(err))
def publish_message(topic,
project=None,
distro=None,
message=None): # pragma: no cover
"""Try to publish a message.
Args:
topic (str): Topic of the message
project (dict): Dictionary representing project
distro (str): Name of the distribution
message (dict): Additional data needed for the topic
"""
msg = dict(project=project, distro=distro, message=message)
try:
message_class = fm_message.get_class("anitya." + topic)
api.publish(message_class(topic="anitya.{}".format(topic), body=msg))
except (
fm_exceptions.ConnectionException,
fm_exceptions.PublishException,
) as err:
# For now, continue just logging the error. Once the messaging has
# been untangled into SQLAlchemy events, it should probably result
# in an exception and the client should try again later.
_log.error(str(err))
def v2_playbook_on_play_start(self, play):
# This gets called once for each play.. but we just issue a message once
# for the first one. One per "playbook"
if self.playbook:
# figure out where the playbook FILE is
path = os.path.abspath(self.playbook._file_name)
# Bail out early without publishing if we're in --check mode
if self.play_context.check_mode:
return
if not self.playbook_path:
try:
msg = Message(
topic="ansible.playbook.start",
body={
'playbook': path,
'userid': getlogin(),
'extra_vars': play._variable_manager.extra_vars,
'inventory':
play._variable_manager._inventory._sources,
'playbook_checksum': secure_hash(path),
'check': self.play_context.check_mode
})
publish(msg)
except PublishReturned as e:
LOGGER.warning(
"Fedora Messaging broker rejected message %s: %s",
msg.id, e)
except ConnectionException as e:
LOGGER.warning("Error sending message %s: %s", msg.id, e)
self.playbook_path = path
def admin_new_release():
if not is_admin():
flask.flash("You are not an admin", "error")
return flask.redirect(flask.url_for("ui.index"))
form = forms.ReleaseForm()
if form.validate_on_submit():
release = db.Release()
form.populate_obj(obj=release)
db.Session.add(release)
db.Session.commit()
message = fm_api.Message(
topic="kerneltest.release.new",
body={
"agent": flask.g.user.username,
"release": {
"releasenum": release.version,
"support": release.support
},
},
)
try:
fm_api.publish(message)
except (
fm_api.exceptions.ConnectionException,
fm_api.exceptions.ConnectionException,
):
pass
flask.flash('Release "%s" added' % release.version)
return flask.redirect(flask.url_for("ui.index"))
return flask.render_template("release_new.html",
form=form,
submit_text="Create release")
def publish(topic, msg, force=False):
"""
Send a message via Fedora Messaging.
This is used to send a message to the AMQP broker.
Args:
topic (str): The topic suffix. The "bodhi" prefix is applied (along with
the "topic_prefix" settings from Fedora Messaging).
msg (dict): The message body to send.
force (bool): If False (the default), the message is only sent after the
currently active database transaction successfully commits. If true,
the messages is sent immediately.
"""
# Dirty, nasty hack that I feel shame for: use the fedmsg encoder that modifies
# messages quietly if they have objects with __json__ methods on them.
# For now, copy that behavior. In the future, callers should pass
# fedora_messaging.api.Message sub-classes or this whole API should go away.
body = json.loads(json.dumps(msg, cls=FedMsgEncoder))
message = api.Message(topic="bodhi.{}".format(topic), body=body)
if force:
api.publish(message)
return
session = Session()
if 'messages' not in session.info:
session.info['messages'] = []
session.info['messages'].append(message)
_log.debug('Queuing message %r for delivery on session commit', message.id)
def test_consume_halt_with_exitcode(callback, exit_code, msg, queue):
"""Assert user execution halt with reason and exit_code is reported."""
args = [
"fedora-messaging",
"--conf={}".format(CLI_CONF),
"consume",
"--callback=fedora_messaging.tests.integration.test_cli:{}".format(
callback),
"--queue-name={}".format(queue),
"--exchange=amq.topic",
"--routing-key=#",
]
process = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(5)
api.publish(message.Message())
for _ in range(5):
time.sleep(1)
if process.poll() is not None:
break
else:
process.kill()
pytest.fail("Process never stopped!: {}".format(process.stdout.read()))
assert process.returncode == exit_code
assert msg in process.stdout.read()
def _publish_with_retry(message: 'base.BodhiMessage'):
"""
Call fedora_messaging.api.publish with the given message, and retry upon temporary failures.
The goal of this function is to try to recover from temporary failures by trying again for a
while. If it is unable to succeed, it will ultimately raise the Exception.
"""
api.publish(message)
def fedora_messaging_change(change_id):
change = Change.objects.get(pk=change_id)
publish(
Message(
topic=get_change_topic(change),
headers=get_change_headers(change),
body=get_change_body(change),
))
def test_twisted_consume_drop_message():
"""Assert raising Drop causes the message to be dropped, but processing continues."""
queue = str(uuid.uuid4())
queues = {
queue: {
"auto_delete": False,
"arguments": {
"x-expires": 60 * 1000
}
}
}
bindings = [{
"queue": queue,
"exchange": "amq.topic",
"routing_keys": ["#"]
}]
msg = message.Message(
topic=u"nice.message",
headers={u"niceness": u"very"},
body={u"encouragement": u"You're doing great!"},
)
dropped_messages = []
def callback(message):
"""Drop 1 message and then halt on the second message."""
dropped_messages.append(message)
if len(dropped_messages) == 2:
raise exceptions.HaltConsumer()
raise exceptions.Drop()
# Assert that the number of consumers we think we started is the number the
# server things we started. This will fail if other tests don't clean up properly.
# If it becomes problematic perhaps each test should have a vhost.
consumers = yield api.twisted_consume(callback, bindings, queues)
api.publish(msg, "amq.topic")
api.publish(msg, "amq.topic")
_add_timeout(consumers[0].result, 10)
try:
yield consumers[0].result
except exceptions.HaltConsumer:
# Assert both messages are delivered, no messages are un-acked, and only one
# message got a positive acknowledgment.
server_queue = yield task.deferLater(
reactor,
5,
treq.get,
HTTP_API + "queues/%2F/" + queue,
auth=HTTP_AUTH,
timeout=3,
)
server_queue = yield server_queue.json()
assert server_queue["consumers"] == 0
assert server_queue["messages"] == 0
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Timeout reached without consumer halting!")
finally:
yield consumers[0].cancel()
def publish_fedmsg(session, message):
if not get_config('fedmsg-publisher.enabled', False):
return
message = fedmsg.Message(
topic='{modname}.{topic}'.format(**message),
body=message['msg'],
)
session.log.info('Publishing fedmsg:\n' + str(message))
fedmsg.publish(message)
def test_twisted_consume_general_exception():
"""
Assert if the callback raises an unhandled exception, it is passed on to the
consumer.result and the message is re-queued.
"""
queue = str(uuid.uuid4())
queues = {
queue: {
"auto_delete": False,
"arguments": {
"x-expires": 60 * 1000
}
}
}
bindings = [{
"queue": queue,
"exchange": "amq.topic",
"routing_keys": ["#"]
}]
msg = message.Message(
topic=u"nice.message",
headers={u"niceness": u"very"},
body={u"encouragement": u"You're doing great!"},
)
def callback(message):
"""An *exceptionally* useless callback"""
raise Exception("Oh the huge manatee")
# Assert that the number of consumers we think we started is the number the
# server things we started. This will fail if other tests don't clean up properly.
# If it becomes problematic perhaps each test should have a vhost.
consumers = yield api.twisted_consume(callback, bindings, queues)
api.publish(msg, "amq.topic")
_add_timeout(consumers[0].result, 10)
try:
yield consumers[0].result
pytest.fail("Expected an exception to be raised.")
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Timeout reached without consumer halting!")
except Exception as e:
# Assert the message was delivered and re-queued when the consumer crashed.
assert e.args[0] == "Oh the huge manatee"
server_queue = yield task.deferLater(
reactor,
10,
treq.get,
HTTP_API + "queues/%2F/" + queue,
auth=HTTP_AUTH,
timeout=3,
)
server_queue = yield server_queue.json()
assert server_queue["consumers"] == 0
assert server_queue["messages"] == 1
finally:
yield consumers[0].cancel()
def _do_publish(session, topic, body):
namespace = get_config('osci.namespace')
artifact_type = get_config('osci.build_group_artifact_type')
message = fedmsg.Message(
topic=f'ci.{namespace}.{artifact_type}.test.{topic}',
body=body,
)
session.log.info('Publishing fedmsg:\n' + str(message))
fedmsg.publish(message)
def publish(message):
"""
Publish message through fedora-messaging.
Params:
message (dict): JSON representation of the message
"""
msg = fm_message.Message(topic=message["topic"], body=message["msg"])
api.publish(msg)
def publish(message):
"""
Publish message through fedora-messaging.
Params:
message (dict): JSON representation of the message
"""
msg = fm_message.Message(topic=message["topic"], body=message["msg"])
api.publish(msg)
def _fedora_messaging_publish(topic, msg):
from fedora_messaging import api, message
if conf.dry_run:
logger.info(
'DRY-RUN: send message to fedora-messaging, topic: %s, msg: %s',
topic, msg)
else:
fm_msg = message.Message(topic=topic, body=msg)
logger.debug('Send message: %s', fm_msg)
api.publish(fm_msg)
def publish(self, message):
try:
msg = Message(topic='{}.result.new'.format(self.modname),
body=message)
publish(msg)
except PublishReturned as e:
log.error('Fedora Messaging broker rejected message {}: {}'.format(
msg.id, e))
except ConnectionException as e:
log.error('Error sending message {}: {}'.format(msg.id, e))
def notify(self, package: Package, message: str, opts: dict) -> dict:
"""
This method is inherited from `hotness.notifiers.Notifier`.
It publishes messages to Fedora messaging broker.
Params:
package: Package to create notification for
message: Topic for the message
opts: Additional options for fedora message. Example:
{
"body": {} # Body of the message we want to sent. Look at the
# hotness_schema for more info
}
Returns:
Dictionary containing message id
Example:
{
"msg_id": "ae68f"
}
Raises:
NotifierException: When the required `opts` parameters are missing or
error happens during publishing.
"""
topic = self.prefix + "." + message
body = opts.get("body", {})
output = {}
if not body:
raise NotifierException(
"Additional parameters are missing! Please provide `body` for the message."
)
message_class = fm_message.get_class(topic)
if not message_class:
raise NotifierException("Unknown topic provided '{}'".format(topic))
_logger.info("publishing topic %r" % topic)
try:
msg = message_class(topic=topic, body=body)
api.publish(msg)
except PublishException as e:
raise NotifierException(
"Fedora messaging broker rejected message {}:{}".format(msg.id, e)
)
except ConnectionException as e:
raise NotifierException("Error sending the message {}:{}".format(msg.id, e))
finally:
output["msg_id"] = msg.id
return output
def test_twisted_consume_halt_consumer():
"""
Assert raising HaltConsumer works with :func:`fedora_messaging.api.twisted_consume`
API.
"""
queue = str(uuid.uuid4())
queues = {queue: {"auto_delete": False, "arguments": {"x-expires": 60 * 1000}}}
bindings = [{"queue": queue, "exchange": "amq.topic", "routing_keys": ["#"]}]
msg = message.Message(
topic=u"nice.message",
headers={u"niceness": u"very"},
body={u"encouragement": u"You're doing great!"},
)
expected_headers = {
u"fedora_messaging_severity": 20,
u"fedora_messaging_schema": u"base.message",
u"niceness": u"very",
}
messages_received = []
def callback(message):
"""Count to 3 and quit."""
messages_received.append(message)
if len(messages_received) == 3:
raise exceptions.HaltConsumer()
consumers = yield api.twisted_consume(callback, bindings, queues)
# Assert the server reports a consumer
server_queue = yield get_queue(queue)
assert server_queue["consumers"] == 1
for _ in range(0, 3):
api.publish(msg, "amq.topic")
_add_timeout(consumers[0].result, 10)
try:
yield consumers[0].result
except exceptions.HaltConsumer as e:
assert len(messages_received) == 3
assert e.exit_code == 0
for m in messages_received:
assert u"nice.message" == m.topic
assert {u"encouragement": u"You're doing great!"} == m.body
assert "sent-at" in m._headers
del m._headers["sent-at"]
assert expected_headers == m._headers
server_queue = yield get_queue(queue)
assert server_queue["consumers"] == 0
except (defer.TimeoutError, defer.CancelledError):
yield consumers[0].cancel()
pytest.fail("Timeout reached without consumer halting!")
def publish_taskotron_message(result, include_job_url=False):
"""
Publish a fedmsg on the taskotron topic with Taskotron-compatible structure.
These messages are deprecated, consumers should consume from the resultsdb
topic instead.
"""
prev_result = get_prev_result(result)
if prev_result is not None and prev_result.outcome == result.outcome:
# If the previous result had the same outcome, skip publishing
# a message for this new result.
# This was intended as a workaround to avoid spammy messages from the
# dist.depcheck task, which tends to produce a very large number of
# identical results for any given build, because of the way that it is
# designed.
log.debug(
"Skipping Taskotron message for result %d, outcome has not changed",
result.id)
return
task = dict((datum.key, datum.value) for datum in result.data
if datum.key in (
'item',
'type',
))
task['name'] = result.testcase.name
body = {
'task': task,
'result': {
'id': result.id,
'submit_time':
result.submit_time.strftime("%Y-%m-%d %H:%M:%S UTC"),
'prev_outcome': prev_result.outcome if prev_result else None,
'outcome': result.outcome,
'log_url': result.ref_url,
}
}
if include_job_url: # only in the v1 API
body['result'][
'job_url'] = result.groups[0].ref_url if result.groups else None
try:
msg = Message(topic='taskotron.result.new', body=body)
publish(msg)
except PublishReturned as e:
log.error('Fedora Messaging broker rejected message {}: {}'.format(
msg.id, e))
except ConnectionException as e:
log.error('Error sending message {}: {}'.format(msg.id, e))
def test_twisted_stop_service():
"""
Assert stopping the service, which happens when the reactor shuts down, waits
for consumers to finish processing and then cancels them before closing the connection.
"""
queue = str(uuid.uuid4())
queues = {
queue: {
"auto_delete": False,
"arguments": {
"x-expires": 60 * 1000
}
}
}
bindings = [{
"queue": queue,
"exchange": "amq.topic",
"routing_keys": ["#"]
}]
message_received, message_processed = defer.Deferred(), defer.Deferred()
def callback(message):
"""Callback when the message is received, introduce a delay, then callback again."""
reactor.callFromThread(message_received.callback, None)
time.sleep(5)
reactor.callFromThread(message_processed.callback, None)
consumers = yield api.twisted_consume(callback, bindings, queues)
api.publish(message.Message(), "amq.topic")
_add_timeout(consumers[0].result, 10)
_add_timeout(message_received, 10)
try:
yield message_received
except (defer.TimeoutError, defer.CancelledError):
yield consumers[0].cancel()
pytest.fail("Timeout reached without consumer receiving message!")
assert not message_processed.called
deferred_stop = api._twisted_service.stopService()
_add_timeout(message_processed, 10)
try:
yield message_processed
# The request to stop should wait on the message to be processed
assert deferred_stop.called is False
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Timeout reached without consumer processing message")
yield deferred_stop
def send_messages_after_commit(session):
"""
Send messages via AMQP after a database commit occurs.
Args:
session (sqlalchemy.orm.session.Session): The session that was committed.
"""
if 'messages' in session.info:
for m in session.info['messages']:
try:
api.publish(m)
except fml_exceptions.BaseException:
# In the future we should handle errors more gracefully
_log.exception("An error occurred publishing %r after a database commit", m)
session.info['messages'] = []
def v2_playbook_on_stats(self, stats):
if not self.playbook_path:
return
results = dict([(h, stats.summarize(h)) for h in stats.processed])
try:
msg = Message(topic="ansible.playbook.complete",
body={
'playbook': self.playbook_path,
'userid': getlogin(),
'results': results
})
publish(msg)
except PublishReturned as e:
LOGGER.warning("Fedora Messaging broker rejected message %s: %s",
msg.id, e)
except ConnectionException as e:
LOGGER.warning("Error sending message %s: %s", msg.id, e)
def enqueue(data, topic, headers):
DQ.append((data, topic, headers))
while DQ:
_data, _topic, _headers = DQ.popleft()
try:
if get_bool_env('IS_PRODUCTION', DEFAULT_IS_PRODUCTION):
msg = message.Message(topic=_topic,
headers=_headers,
body=_data)
api.publish(msg)
app.logger.info(f'Sending payload to {_topic}')
else:
app.logger.info(f'\n---\n{_topic}\n{_headers}\n{_data}\n---')
except Exception:
app.logger.exception(
'Error during sending message, will be retried')
DQ.appendleft((_data, _topic, _headers))
def admin_edit_release(relnum):
if not is_admin():
flask.flash("You are not an admin", "error")
return flask.redirect(flask.url_for("ui.index"))
release = db.Release.query.filter_by(version=relnum).one_or_none()
if not release:
flask.flash("No release %s found" % relnum)
return flask.redirect(flask.url_for("ui.index"))
form = forms.ReleaseForm(obj=release)
if form.validate_on_submit():
form.populate_obj(obj=release)
db.Session.commit()
message = fm_api.Message(
topic="kerneltest.release.edit",
body={
"agent": flask.g.user.username,
"release": {
"releasenum": release.version,
"support": release.support
},
},
)
try:
fm_api.publish(message)
except (
fm_api.exceptions.ConnectionException,
fm_api.exceptions.ConnectionException,
):
pass
flask.flash('Release "%s" updated' % release.version)
return flask.redirect(flask.url_for("ui.index"))
return flask.render_template("release_new.html",
form=form,
release=release,
submit_text="Edit release")
def get(self, request, *args, **kwargs):
try:
message = Message.objects.get(identifier=kwargs['identifier'], sender_email_token=kwargs['token'])
except Message.DoesNotExist:
response = render(request, self.template_name, {'not_found': True})
response.status_code = 404
return response
if message.sender_email != self.request.user.email:
response = render(request, self.template_name, {'not_sender': True})
response.status_code = 400
return response
if message.status != Message.STATUS.pending_sender_confirmation:
response = render(request, self.template_name, {'already_confirmed': True})
return response
message.send_to_recipient(self.request.is_secure(), self.request.get_host())
sender_name = self.request.user.username if message.sender_named else "Anonymous"
recipient_name = message.recipient_username if message.recipient_username else message.recipient_name
message = MessageV1(
topic="happinesspacket.send",
body={
"id": message.identifier,
"sender": sender_name,
"recipient": recipient_name
}
)
try:
publish(message)
except PublishReturned:
response = render(request, self.template_name, {'publish_returned': True})
response.status_code = 500
return response
except ConnectionException:
response = render(request, self.template_name, {'connection_exception': True})
response.status_code = 500
return response
return HttpResponseRedirect(reverse('messaging:sender_confirmed'))
def run(self, cmdline, db):
levels = tuple(10**n for n in range(7))
if cmdline.reports:
# Sum of counts until yesterday
q_yesterday = (
db.session
.query(Report.id.label("y_report_id"),
func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day < cmdline.date)
.group_by(Report.id)
.subquery()
)
# Sum of counts until today
q_today = (
db.session
.query(Report.id.label("t_report_id"),
func.sum(ReportHistoryDaily.count).label("sum_today"))
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day <= cmdline.date)
.group_by(Report.id)
.subquery()
)
q = (db.session.query(Report,
q_today.c.sum_today,
q_yesterday.c.sum_yesterday)
.outerjoin(q_today, Report.id == q_today.c.t_report_id)
.outerjoin(q_yesterday, Report.id == q_yesterday.c.y_report_id)
.filter(or_(Report.max_certainty.isnot(None), Report.max_certainty != 100))
.filter(or_(and_(q_yesterday.c.sum_yesterday.is_(None),
q_today.c.sum_today.isnot(None)),
q_today.c.sum_today != q_yesterday.c.sum_yesterday))
)
for db_report, sum_today, sum_yesterday in q.yield_per(100):
# avoid None
sum_yesterday = sum_yesterday or 0
for level in levels:
if sum_yesterday < level <= sum_today:
self.log_info("Notifying about report #{0} level {1}"
.format(db_report.id, level))
msg_body = {
"report_id": db_report.id,
"function": db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence": db_report.first_occurrence
.strftime("%Y-%m-%d"),
"count": sum_today,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse("reports.item",
report_id=db_report.id)
if db_report.problem_id:
msg_body["problem_id"] = db_report.problem_id
try:
msg = FafReportMessage(topic="faf.report.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
self.log_warn("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
except ConnectionException as e:
self.log_warn("Error sending message {0}: {1}".format(msg.id, e))
if cmdline.problems:
# Sum of counts until yesterday
q_yesterday = (
db.session
.query(Problem.id.label("y_problem_id"),
func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
.join(Report, Report.problem_id == Problem.id)
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day < cmdline.date)
.group_by(Problem.id)
.subquery()
)
# Sum of counts until today
q_today = (
db.session
.query(Problem.id.label("t_problem_id"),
func.sum(ReportHistoryDaily.count).label("sum_today"))
.join(Report, Report.problem_id == Problem.id)
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day <= cmdline.date)
.group_by(Problem.id)
.subquery()
)
q = (db.session
.query(Problem, q_today.c.sum_today, q_yesterday.c.sum_yesterday)
.outerjoin(q_today, Problem.id == q_today.c.t_problem_id)
.outerjoin(q_yesterday, Problem.id == q_yesterday.c.y_problem_id)
.filter(or_(and_(q_yesterday.c.sum_yesterday.is_(None),
q_today.c.sum_today.isnot(None)),
q_today.c.sum_today != q_yesterday.c.sum_yesterday))
)
for db_problem, sum_today, sum_yesterday in q.yield_per(100):
# avoid None
sum_yesterday = sum_yesterday or 0
for level in levels:
if sum_yesterday < level <= sum_today:
self.log_info("Notifying about problem #{0} level {1}"
.format(db_problem.id, level))
msg_body = {
"problem_id": db_problem.id,
"function": db_problem.crash_function,
"components": list(db_problem.unique_component_names),
"first_occurrence": db_problem.first_occurrence
.strftime("%Y-%m-%d"),
"count": sum_today,
"type": db_problem.type,
"level": level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse("problems.item",
problem_id=db_problem.id)
try:
msg = FafProblemMessage(topic="faf.problem.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
self.log_warn("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
except ConnectionException as e:
self.log_warn("Error sending message {0}: {1}".format(msg.id, e))
def fedmsg_report(target, value, oldvalue, initiator): # pylint: disable=unused-argument
"""
Send Fedora Messaging notifications when Report.count reaches specified threshold.
"""
try:
db_report = target
if notify_reports:
oldcount = oldvalue
newcount = value
for level in levels:
if oldcount < level <= newcount:
logger.info("Notifying about report #{0} level {1}"
.format(db_report.id, level))
msg_body = {
"report_id": db_report.id,
"function": db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence": db_report.first_occurrence
.strftime("%Y-%m-%d"),
"count": newcount,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed() and db_report.hashes:
msg_body["url"] = web.reverse("reports.bthash_forward",
bthash=db_report.hashes[0].hash)
if db_report.problem_id:
msg_body["problem_id"] = db_report.problem_id
try:
msg = FafReportMessage(topic="faf.report.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
logger.exception("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
except ConnectionException as e:
logger.exception("Error sending message {0}: {1}".format(msg.id, e))
if notify_problems and db_report.problem is not None:
oldcount = db_report.problem.reports_count
newcount = oldcount + value - oldvalue
for level in levels:
if oldcount < level <= newcount:
logger.info("Notifying about problem #{0} level {1}"
.format(db_report.problem.id, level))
msg_body = {
"problem_id": db_report.problem.id,
"function": db_report.problem.crash_function,
"components": list(db_report.problem.unique_component_names),
"first_occurrence": db_report.problem.first_occurrence
.strftime("%Y-%m-%d"),
"count": newcount,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse("problems.item", problem_id=db_report.problem.id)
try:
msg = FafProblemMessage(topic="faf.problem.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
logger.exception("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
except ConnectionException as e:
logger.exception("Error sending message {0}: {1}".format(msg.id, e))
# Catch any exception. This is non-critical and mustn't break stuff
# elsewhere.
except Exception as e: # pylint: disable=broad-except
logger.exception(e, exc_info=True)