def test_twisted_consume_update_callback(queue_and_binding):
"""Assert a second call to consume updates an existing callback."""
queues, bindings = queue_and_binding
callback1 = defer.Deferred()
callback2 = defer.Deferred()
consumers1 = yield api.twisted_consume(
lambda m: reactor.callFromThread(callback1.callback, m), bindings,
queues)
yield threads.deferToThread(api.publish, message.Message(), "amq.topic")
_add_timeout(callback1, 10)
try:
yield callback1
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Never received message for initial callback")
consumers2 = yield api.twisted_consume(
lambda m: reactor.callFromThread(callback2.callback, m), bindings,
queues)
yield threads.deferToThread(api.publish, message.Message(), "amq.topic")
_add_timeout(callback2, 10)
try:
yield callback2
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Never received message for updated callback")
assert consumers1[0]._tag == consumers2[0]._tag
yield consumers2[0].cancel()
def test_proper_message_multiple(self):
"""Assert proper json is returned"""
test_topic = "test topic"
test_body = {"test_key": "test_value"}
test_queue = "test queue"
test_id = "test id"
test_headers = {
"fedora_messaging_schema": "base.message",
"fedora_messaging_severity": message.WARNING,
}
test_properties = pika.BasicProperties(
content_type="application/json",
content_encoding="utf-8",
delivery_mode=2,
headers=test_headers,
message_id=test_id,
)
test_msg = message.Message(
body=test_body, topic=test_topic, properties=test_properties
)
test_msg2 = message.Message(
body=test_body, topic=test_topic, properties=test_properties
)
test_msg.queue = test_queue
test_msg2.queue = test_queue
expected_json = (
'{"body": {"test_key": "test_value"}, "headers": {"fedora_messaging_schema": '
'"base.message", "fedora_messaging_severity": 30}, "id": "test id", "queue": '
'"test queue", "topic": "test topic"}\n'
'{"body": {"test_key": "test_value"}, "headers": {"fedora_messaging_schema": '
'"base.message", "fedora_messaging_severity": 30}, "id": "test id", "queue": '
'"test queue", "topic": "test topic"}\n'
)
self.assertEqual(expected_json, message.dumps([test_msg, test_msg2]))
def test_equality_different_sent_at(self):
"""Assert the "sent-at" key is not included in the equality check."""
m1 = message.Message(topic="test.topic", body={"my": "key"})
m2 = message.Message(topic="test.topic", body={"my": "key"})
m2._headers["sent-at"] = datetime.datetime(1970, 1, 2).isoformat()
self.assertEqual(m1, m2)
def test_twisted_consume_update_callback():
"""Assert a second call to consume updates an existing callback."""
queue = str(uuid.uuid4())
queues = {queue: {"auto_delete": False, "arguments": {"x-expires": 60 * 1000}}}
bindings = [{"queue": queue, "exchange": "amq.topic", "routing_keys": ["#"]}]
callback1 = defer.Deferred()
callback2 = defer.Deferred()
consumers1 = yield api.twisted_consume(
lambda m: reactor.callFromThread(callback1.callback, m), bindings, queues
)
api.publish(message.Message(), "amq.topic")
_add_timeout(callback1, 10)
try:
yield callback1
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Never received message for initial callback")
consumers2 = yield api.twisted_consume(
lambda m: reactor.callFromThread(callback2.callback, m), bindings, queues
)
api.publish(message.Message(), "amq.topic")
_add_timeout(callback2, 10)
try:
yield callback2
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Never received message for updated callback")
assert consumers1[0]._tag == consumers2[0]._tag
yield consumers2[0].cancel()
def delayed_publish():
"""Publish, break the channel, and publish again."""
yield threads.deferToThread(api.publish, message.Message(),
"amq.topic")
protocol = yield api._twisted_service._service.factory.when_connected()
yield protocol._publish_channel.close()
yield threads.deferToThread(api.publish, message.Message(),
"amq.topic")
def test_equality(self):
"""
Assert two messages of the same class with the same topic, headers, and
body are equivalent.
"""
self.assertEqual(
message.Message(topic='test.topic', body={'my': 'key'}),
message.Message(topic='test.topic', body={'my': 'key'}))
def test_equality(self):
"""
Assert two messages of the same class with the same topic, headers, and
body are equivalent.
"""
self.assertEqual(
message.Message(topic="test.topic", body={"my": "key"}),
message.Message(topic="test.topic", body={"my": "key"}),
)
def test_missing_severity(self):
"""Assert the default severity is INFO if it's not in the headers."""
msg = message.Message(severity=message.ERROR)
del msg._headers["fedora_messaging_severity"]
recv_msg = message.get_message("", msg._properties, b"{}")
self.assertEqual(recv_msg.severity, message.INFO)
def test_repr(self):
"""Assert the message produces a valid representation of the message."""
msg = message.Message(topic="test.topic", body={"my": "key"})
expected = "Message(id='{}', topic='test.topic', body={{'my': 'key'}})".format(
msg.id
)
self.assertEqual(expected, repr(msg))
def test_signed_implicit_cert(self):
"""Assert signing certificate is properly autodetected."""
zmq_bridge = bridges.AmqpToZmq()
msg = message.Message(topic="my.topic", body={"my": "message"})
hostname = socket.gethostname().split(".", 1)[0]
base_conf = {"sign_messages": True, "ssldir": FIXTURES_DIR}
sign_configs = [
{
"name": "fedmsg",
"certnames": {
"fedmsg": "fedmsg"
}
},
{
"cert_prefix": "fedmsg",
"certnames": {
"fedmsg.{}".format(hostname): "fedmsg"
},
},
]
for sign_config in sign_configs:
conf = base_conf.copy()
conf.update(sign_config)
with mock.patch.dict(
"fedmsg_migration_tools.bridges.fedmsg_config.conf", conf):
with mock.patch(
"fedmsg_migration_tools.bridges.fedmsg.crypto.sign"
) as mock_sign:
mock_sign.side_effect = lambda *a, **kw: a[0]
zmq_bridge(msg)
sign_call_kw = mock_sign.call_args_list[-1][1]
self.assertIn("certname", sign_call_kw)
self.assertEqual(sign_call_kw["certname"], "fedmsg")
def test_drop_handled():
"""Assert raising Drop in a consumer works and messages are not re-delivered"""
queue = str(uuid.uuid4())
messages = []
serv = service.FedoraMessagingService(amqp_url="amqp://")
serv.startService()
client = yield serv.getFactory().whenConnected()
queues = [
{"queue": queue, "auto_delete": True, "arguments": {"x-expires": 60 * 1000}}
]
yield client.declare_queues(queues)
yield client.bind_queues(
[{"queue": queue, "exchange": "amq.topic", "routing_key": "#"}]
)
def callback(message):
messages.append(message)
raise exceptions.Drop()
yield client.consume(callback, queue)
assert len(client._consumers) == 1
yield client.publish(message.Message(), "amq.topic")
yield task.deferLater(reactor, 3.0, lambda: True) # Just wait a few seconds
assert len(messages) == 1
assert len(client._consumers) == 1
yield client.cancel(queue)
serv.stopService()
def test_signed(self):
"""Assert messages are signed if fedmsg is configured for signatures."""
year = datetime.datetime.utcnow().year
zmq_bridge = bridges.AmqpToZmq()
msg = message.Message(topic="my.topic", body={"my": "message"})
expected = {
"topic": "my.topic",
"msg": {
"my": "message"
},
"timestamp": 101,
"msg_id": "{}-{}".format(year, msg.id),
"i": 1,
"username": "******",
"crypto": "x509",
}
conf = {
"sign_messages": True,
"ssldir": FIXTURES_DIR,
"certname": "fedmsg"
}
with mock.patch.dict(
"fedmsg_migration_tools.bridges.fedmsg_config.conf", conf):
zmq_bridge(msg)
body = json.loads(
zmq_bridge.pub_socket.send_multipart.call_args_list[0][0][0]
[1].decode("utf-8"))
self.assertIn("signature", body)
self.assertIn("certificate", body)
del body["signature"]
del body["certificate"]
self.assertEqual(body, expected)
def test_twisted_consume_halt_consumer_requeue(queue_and_binding):
"""Assert raising HaltConsumer with requeue=True re-queues the message."""
queues, bindings = queue_and_binding
msg = message.Message(
topic=u"nice.message",
headers={u"niceness": u"very"},
body={u"encouragement": u"You're doing great!"},
)
def callback(message):
"""Count to 3 and quit."""
raise exceptions.HaltConsumer(exit_code=1, requeue=True)
# Assert that the number of consumers we think we started is the number the
# server things we started. This will fail if other tests don't clean up properly.
# If it becomes problematic perhaps each test should have a vhost.
consumers = yield api.twisted_consume(callback, bindings, queues)
yield threads.deferToThread(api.publish, msg, "amq.topic")
_add_timeout(consumers[0].result, 10)
try:
yield consumers[0].result
except exceptions.HaltConsumer as e:
# Assert there are no consumers for the queue, and that there's a ready message
assert e.exit_code == 1
server_queue = yield get_queue(queues)
assert server_queue["consumers"] == 0
assert server_queue["messages_ready"] == 1
except (defer.TimeoutError, defer.CancelledError):
yield consumers[0].cancel()
pytest.fail("Timeout reached without consumer halting!")
def test_msg_becomes_body(self):
"""Assert the "msg" key of a fedmsg is the fedora-messaging body."""
expected = message.Message(body={"hello": "world"}, topic="hi")
zmq_message = b'{"msg": {"hello": "world"}, "msg_id": "abc123"}'
with fml_testing.mock_sends(expected):
bridges._convert_and_maybe_publish(b"hi", zmq_message, "amq.topic")
def test_twisted_consume_nack_message(queue_and_binding):
"""Assert raising Nack causes the message to be replaced in the queue."""
queues, bindings = queue_and_binding
msg = message.Message(
topic=u"nice.message",
headers={u"niceness": u"very"},
body={u"encouragement": u"You're doing great!"},
)
nacked_messages = []
def callback(message):
"""Nack the message, then halt."""
nacked_messages.append(message)
if len(nacked_messages) == 2:
raise exceptions.HaltConsumer()
raise exceptions.Nack()
# Assert that the number of consumers we think we started is the number the
# server things we started. This will fail if other tests don't clean up properly.
# If it becomes problematic perhaps each test should have a vhost.
consumers = yield api.twisted_consume(callback, bindings, queues)
yield threads.deferToThread(api.publish, msg, "amq.topic")
_add_timeout(consumers[0].result, 10)
try:
yield consumers[0].result
except exceptions.HaltConsumer:
# Assert the message was delivered, redelivered when Nacked, then acked by HaltConsumer
server_queue = yield get_queue(queues)
assert server_queue["consumers"] == 0
assert server_queue["messages"] == 0
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Timeout reached without consumer halting!")
finally:
yield consumers[0].cancel()
def test_consume_halt_with_exitcode(callback, exit_code, msg, queue):
"""Assert user execution halt with reason and exit_code is reported."""
args = [
"fedora-messaging",
"--conf={}".format(CLI_CONF),
"consume",
"--callback=fedora_messaging.tests.integration.test_cli:{}".format(
callback),
"--queue-name={}".format(queue),
"--exchange=amq.topic",
"--routing-key=#",
]
process = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(5)
api.publish(message.Message())
for _ in range(5):
time.sleep(1)
if process.poll() is not None:
break
else:
process.kill()
pytest.fail("Process never stopped!: {}".format(process.stdout.read()))
assert process.returncode == exit_code
assert msg in process.stdout.read()
def test_blank_headers(self):
"""Assert ZMQ messages with blank headers still get the defaults."""
expected = message.Message(body={"hello": "world"}, topic="hi")
zmq_message = b'{"headers": {}, "msg": {"hello": "world"}, "msg_id": "abc123"}'
with fml_testing.mock_sends(expected):
bridges._convert_and_maybe_publish(b"hi", zmq_message, "amq.topic")
def test_invalid_severity(self):
"""Assert the invalid severity fails validation."""
msg = message.Message()
msg._headers["fedora_messaging_severity"] = 42
self.assertRaises(exceptions.ValidationError, message.get_message, "",
msg._properties, b"{}")
def test_dump(self):
"""Assert proper dict is generated on Message._dump."""
test_topic = "test topic"
test_body = {"test_key": "test_value"}
test_queue = "test queue"
test_id = "test id"
test_headers = {
"fedora_messaging_schema": "base.message",
"fedora_messaging_severity": message.WARNING,
}
test_properties = pika.BasicProperties(
content_type="application/json",
content_encoding="utf-8",
delivery_mode=2,
headers=test_headers,
message_id=test_id,
)
expected_dict = {
"topic": test_topic,
"headers": test_headers,
"id": test_id,
"body": test_body,
"queue": test_queue,
}
test_msg = message.Message(body=test_body,
topic=test_topic,
properties=test_properties)
test_msg.queue = test_queue
test_msg_dict = test_msg._dump()
self.assertEqual(expected_dict, test_msg_dict)
def test_headers(self):
msg = message.Message(headers={"foo": "bar"})
self.assertIn("foo", msg._properties.headers)
self.assertEqual(msg._properties.headers["foo"], "bar")
# The fedora_messaging_schema key must also be added when headers are given.
self.assertEqual(msg._properties.headers["fedora_messaging_schema"],
"base.message")
def test_unhandled_exception_cancels_consumer():
"""Assert any unhandled Exception results in the consumer being canceled."""
queue = str(uuid.uuid4())
queues = [{
"queue": queue,
"auto_delete": True,
"durable": False,
"exclusive": False,
"arguments": {
"x-expires": 60 * 1000
},
}]
serv = service.FedoraMessagingService(amqp_url="amqp://")
serv.startService()
client = yield serv.getFactory().whenConnected()
yield client.declare_queues(queues)
yield client.bind_queues([{
"queue": queue,
"exchange": "amq.topic",
"routing_key": "#"
}])
def callback(message):
raise Exception("Panic!")
yield client.consume(callback, queue)
assert len(client._consumers) == 1
yield client.publish(message.Message(), "amq.topic")
yield task.deferLater(reactor, 3.0, lambda: True)
assert len(client._consumers) == 0
yield serv.stopService()
def test_pub_sub_default_settings(self):
"""
Assert publishing and subscribing works with the default configuration.
This should work because the publisher uses the 'amq.topic' exchange by
default and the consumer also uses the 'amq.topic' exchange with its
auto-named queue and a default subscription key of '#'.
"""
# Consumer setup
def counting_callback(message, storage=defaultdict(int)):
storage[message.topic] += 1
if storage[message.topic] == 3:
raise exceptions.HaltConsumer()
consumer_process = multiprocessing.Process(
target=api.consume, args=(counting_callback,))
msg = message.Message(topic=u'nice.message', headers={u'niceness': u'very'},
body={u'encouragement': u"You're doing great!"})
consumer_process.start()
# Allow the consumer time to create the queues and bindings
time.sleep(5)
for _ in range(0, 3):
api.publish(msg)
consumer_process.join(timeout=30)
self.assertEqual(0, consumer_process.exitcode)
def test_repr(self):
"""Assert the message produces a valid representation of the message."""
Message = message.Message # noqa
expected = "Message(body={'my': 'key'}, headers={}, topic='test.topic')"
msg = message.Message(topic='test.topic', body={'my': 'key'})
self.assertEqual(expected, repr(msg))
self.assertEqual(msg, eval(repr(msg)))
def test_missing_headers(self):
"""Assert missing headers results in a default message."""
msg = message.Message()
msg._headers = None
received_msg = message.get_message(msg._encoded_routing_key,
msg._properties, msg._encoded_body)
self.assertIsInstance(received_msg, message.Message)
def test_publish_topic_prefix(self):
# Check that the topic prefix is correctly prepended to outgoing messages.
with mock.patch.dict(config.conf, {"topic_prefix": "prefix"}):
msg = message.Message(topic="test.topic")
self.publisher.publish(msg)
self.publisher_channel_publish.assert_called_once()
publish_call = self.publisher_channel_publish.call_args_list[0][1]
self.assertEqual(publish_call["routing_key"], b"prefix.test.topic")
def test_twisted_consume_drop_message():
"""Assert raising Drop causes the message to be dropped, but processing continues."""
queue = str(uuid.uuid4())
queues = {
queue: {
"auto_delete": False,
"arguments": {
"x-expires": 60 * 1000
}
}
}
bindings = [{
"queue": queue,
"exchange": "amq.topic",
"routing_keys": ["#"]
}]
msg = message.Message(
topic=u"nice.message",
headers={u"niceness": u"very"},
body={u"encouragement": u"You're doing great!"},
)
dropped_messages = []
def callback(message):
"""Drop 1 message and then halt on the second message."""
dropped_messages.append(message)
if len(dropped_messages) == 2:
raise exceptions.HaltConsumer()
raise exceptions.Drop()
# Assert that the number of consumers we think we started is the number the
# server things we started. This will fail if other tests don't clean up properly.
# If it becomes problematic perhaps each test should have a vhost.
consumers = yield api.twisted_consume(callback, bindings, queues)
api.publish(msg, "amq.topic")
api.publish(msg, "amq.topic")
_add_timeout(consumers[0].result, 10)
try:
yield consumers[0].result
except exceptions.HaltConsumer:
# Assert both messages are delivered, no messages are un-acked, and only one
# message got a positive acknowledgment.
server_queue = yield task.deferLater(
reactor,
5,
treq.get,
HTTP_API + "queues/%2F/" + queue,
auth=HTTP_AUTH,
timeout=3,
)
server_queue = yield server_queue.json()
assert server_queue["consumers"] == 0
assert server_queue["messages"] == 0
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Timeout reached without consumer halting!")
finally:
yield consumers[0].cancel()
def test_str(self):
"""Assert calling str on a message produces a human-readable result."""
msg = message.Message(topic='test.topic', body={'my': 'key'})
expected_headers = json.dumps(msg._headers, sort_keys=True, indent=4)
expected = ('Id: {}\nTopic: test.topic\n'
'Headers: {}'
'\nBody: {{\n "my": "key"\n}}').format(
msg.id, expected_headers)
self.assertEqual(expected, str(msg))
def test_properties_default(self):
msg = message.Message()
self.assertEqual(msg._properties.content_type, "application/json")
self.assertEqual(msg._properties.content_encoding, "utf-8")
self.assertEqual(msg._properties.delivery_mode, 2)
self.assertIn("sent-at", msg._properties.headers)
self.assertIn("fedora_messaging_schema", msg._properties.headers)
self.assertEqual(msg._properties.headers["fedora_messaging_schema"],
"base.message")
def test_twisted_consume_general_exception():
"""
Assert if the callback raises an unhandled exception, it is passed on to the
consumer.result and the message is re-queued.
"""
queue = str(uuid.uuid4())
queues = {
queue: {
"auto_delete": False,
"arguments": {
"x-expires": 60 * 1000
}
}
}
bindings = [{
"queue": queue,
"exchange": "amq.topic",
"routing_keys": ["#"]
}]
msg = message.Message(
topic=u"nice.message",
headers={u"niceness": u"very"},
body={u"encouragement": u"You're doing great!"},
)
def callback(message):
"""An *exceptionally* useless callback"""
raise Exception("Oh the huge manatee")
# Assert that the number of consumers we think we started is the number the
# server things we started. This will fail if other tests don't clean up properly.
# If it becomes problematic perhaps each test should have a vhost.
consumers = yield api.twisted_consume(callback, bindings, queues)
api.publish(msg, "amq.topic")
_add_timeout(consumers[0].result, 10)
try:
yield consumers[0].result
pytest.fail("Expected an exception to be raised.")
except (defer.TimeoutError, defer.CancelledError):
pytest.fail("Timeout reached without consumer halting!")
except Exception as e:
# Assert the message was delivered and re-queued when the consumer crashed.
assert e.args[0] == "Oh the huge manatee"
server_queue = yield task.deferLater(
reactor,
10,
treq.get,
HTTP_API + "queues/%2F/" + queue,
auth=HTTP_AUTH,
timeout=3,
)
server_queue = yield server_queue.json()
assert server_queue["consumers"] == 0
assert server_queue["messages"] == 1
finally:
yield consumers[0].cancel()
def test_sent_at(self):
"""Assert a timestamp is inserted and contains explicit timezone information."""
mock_datetime = mock.Mock()
mock_datetime.utcnow.return_value = datetime.datetime(1970, 1, 1, 0, 0, 0)
with mock.patch("datetime.datetime", mock_datetime):
msg = message.Message()
self.assertEqual("1970-01-01T00:00:00+00:00", msg._headers["sent-at"])
