def test_message_receipt(client, random_topic):
processor: missive.Processor[missive.JSONMessage] = missive.Processor()
messages = set()
@processor.handle_for(always)
def catch_all(message, ctx):
messages.add(message.get_json()["test_id"])
adapted.shutdown_handler.set_flag()
logger.info("caught %s", message.get_json())
adapted = MQTTAdapter(
missive.JSONMessage, processor, [random_topic], host=MQTT_HOST, port=MQTT_PORT,
)
test_event = make_test_message_body()
thread = threading.Thread(target=adapted.run)
thread.start()
# FIXME: There has to be a better way than this but there are a lot of
# small races here and it's difficult to do it more elegantly.
while thread.is_alive():
client.publish(random_topic, json.dumps(test_event).encode("utf-8"))
sleep(0.1)
assert test_event["test_id"] in messages
def test_message_receipt(redis_client):
processor: missive.Processor[missive.JSONMessage] = missive.Processor()
flag = False
@processor.handle_for(always)
def catch_all(message, ctx):
nonlocal flag
flag = message.get_json()
ctx.ack()
adapted.shutdown_handler.set_flag()
adapted = RedisPubSubAdapter(missive.JSONMessage, processor,
["test-channel"])
thread = threading.Thread(target=adapted.run)
thread.start()
while adapted.thread is None:
time.sleep(0.01)
test_event = {"test-event": True}
redis_client.publish("test-channel", json.dumps(test_event))
adapted.thread.join(1)
assert flag == test_event
def test_passing_a_conn(channel, random_queue, connection):
processor: missive.Processor[missive.JSONMessage] = missive.Processor()
flag = False
@processor.handle_for(always)
def catch_all(message, ctx):
nonlocal flag
flag = message.get_json()
ctx.ack()
adapted.shutdown_handler.set_flag()
adapted = RabbitMQAdapter(
missive.JSONMessage,
processor,
[random_queue.name],
url_or_conn=connection,
disable_shutdown_handler=True,
)
test_event = {"test-event": True}
producer = kombu.Producer(channel)
producer.publish(json.dumps(test_event).encode("utf-8"),
routing_key=random_queue.name)
thread = threading.Thread(target=adapted.run)
thread.start()
thread.join(1)
assert flag == test_event
# Assert nothing left on the queue
assert random_queue.get() is None
def init_proc(pool) -> missive.Processor[missive.JSONMessage]:
proc: missive.Processor[missive.JSONMessage] = missive.Processor()
@proc.before_processing
def create_session(
proc_ctx: missive.ProcessingContext[missive.JSONMessage]):
# pretend connection pool
proc_ctx.state.pool = pool
@proc.before_handling
def create_connection(
proc_ctx: missive.ProcessingContext,
handling_ctx: missive.HandlingContext[missive.JSONMessage],
):
handling_ctx.state.conn = proc_ctx.state.pool.pop()
handling_ctx.state.conn.open()
@proc.handle_for(TypeMatcher("happy"))
def happy_handler(
message: missive.JSONMessage,
handling_ctx: missive.HandlingContext[missive.JSONMessage],
):
handling_ctx.ack()
@proc.after_handling
def return_connection(proc_ctx, handling_ctx):
handling_ctx.state.conn.commit()
proc_ctx.state.pool.append(handling_ctx.state.conn)
@proc.after_processing
def close_session(proc_ctx):
for conn in proc_ctx.state.pool:
conn.close()
return proc
def test_two_handlers_with_the_same_matcher():
processor: missive.Processor[missive.JSONMessage] = missive.Processor()
@processor.handle_for(always)
def handler_1(message, ctx):
ctx.ack(message)
with pytest.raises(RuntimeError):
@processor.handle_for(always)
def handler_2(message, ctx):
ctx.ack(message)
def test_exception_raised_no_dlq():
"""If an exception is raised and there's no DLQ, the proc should crash,
(without acking)."""
proc: missive.Processor[missive.RawMessage] = missive.Processor()
@proc.handle_for(always)
def crash(message, ctx):
raise RuntimeError("bad bytes!")
with proc.test_client() as test_client:
blank_message = missive.RawMessage(b"")
with pytest.raises(RuntimeError):
test_client.send(blank_message)
def test_no_matching_handler():
processor: m.Processor[m.RawMessage] = m.Processor()
@processor.handle_for(never)
def non_matching_handler(message, ctx):
assert False
with processor.test_client() as test_client:
blank_message = m.RawMessage(b"")
with pytest.raises(RuntimeError):
test_client.send(blank_message)
assert blank_message not in test_client.acked
def test_exception_raised_with_a_dlq():
"""If an exception is raised and there's a DLQ, it's written to the DLQ,
acked and the processor gets on with it's life."""
proc: missive.Processor[missive.RawMessage] = missive.Processor()
dlq: missive.DLQ = {}
proc.set_dlq(dlq)
@proc.handle_for(always)
def crash(message, ctx):
raise RuntimeError("bad bytes!")
with proc.test_client() as test_client:
blank_message = missive.RawMessage(b"")
test_client.send(blank_message)
assert dlq == {blank_message.message_id: (blank_message, "bad bytes!")}
assert blank_message in test_client.acked
def test_no_matching_handler():
"""Messages for which no handler matches should be written to the DLQ"""
dlq: Dict = {}
processor: m.Processor[m.RawMessage] = m.Processor()
processor.set_dlq(dlq)
@processor.handle_for(never)
def non_matching_handler(message, ctx):
assert False
with processor.test_client() as test_client:
blank_message = m.RawMessage(b"")
test_client.send(blank_message)
assert blank_message in test_client.acked
assert list(dlq.values()) == [(blank_message, "no matching handlers")]
def test_one_matching_handler():
processor: m.Processor[m.RawMessage] = m.Processor()
flag = False
@processor.handle_for(always)
def flip_bit(message: m.RawMessage,
ctx: m.HandlingContext[m.RawMessage]) -> None:
nonlocal flag
flag = True
ctx.ack()
with processor.test_client() as test_client:
blank_message = m.RawMessage(b"")
test_client.send(blank_message)
assert flag
assert blank_message in test_client.acked
def test_multiple_matching_handlers():
processor: m.Processor[m.RawMessage] = m.Processor()
@processor.handle_for(always)
def a_matching_handler(message, ctx):
message.ack()
@processor.handle_for(lambda m: True)
def another_matching_handler(message, ctx):
message.ack()
with processor.test_client() as test_client:
blank_message = m.RawMessage(b"")
with pytest.raises(RuntimeError):
test_client.send(blank_message)
assert blank_message not in test_client.acked
def test_multiple_matching_handlers():
dlq: Dict = {}
processor: m.Processor[m.RawMessage] = m.Processor()
processor.set_dlq(dlq)
@processor.handle_for(always)
def a_matching_handler(message, ctx):
ctx.ack(message)
@processor.handle_for(lambda m: True)
def another_matching_handler(message, ctx):
message.ack()
with processor.test_client() as test_client:
blank_message = m.RawMessage(b"")
test_client.send(blank_message)
assert blank_message in test_client.acked
assert list(dlq.values()) == [(blank_message, "multiple matching handlers")]
def test_no_dlq_required():
"""Test the happy path where the DLQ is never used"""
dlq: Dict = {}
processor: m.Processor[m.RawMessage] = m.Processor()
processor.set_dlq(dlq)
flag = False
@processor.handle_for(always)
def flip_bit(message: m.RawMessage, ctx: m.HandlingContext[m.RawMessage]) -> None:
nonlocal flag
flag = True
ctx.ack()
with processor.test_client() as test_client:
blank_message = m.RawMessage(b"")
test_client.send(blank_message)
assert flag
assert blank_message in test_client.acked
def test_one_matching_handler_among_multiple():
processor: m.Processor[m.RawMessage] = m.Processor()
flag = False
@processor.handle_for(always)
def a_matching_handler(message, ctx):
nonlocal flag
flag = True
ctx.ack()
@processor.handle_for(never)
def another_handler(message, ctx):
message.ack()
with processor.test_client() as test_client:
blank_message = m.RawMessage(b"")
test_client.send(blank_message)
assert flag
assert blank_message in test_client.acked
def test_receipt_from_multiple_queues(channel):
q1 = make_random_queue(channel)
q1.declare()
q2 = make_random_queue(channel)
q2.declare()
messages = set()
processor: missive.Processor[missive.JSONMessage] = missive.Processor()
@processor.handle_for(always)
def catch_all(message, ctx):
messages.add(message.get_json()["n"])
ctx.ack()
if len(messages) >= 2:
adapted.shutdown_handler.set_flag()
adapted = RabbitMQAdapter(
missive.JSONMessage,
processor,
[q1.name, q2.name],
url_or_conn=RABBITMQ_URL,
disable_shutdown_handler=True,
)
producer = kombu.Producer(channel)
producer.publish(json.dumps({"n": 1}).encode("utf-8"), routing_key=q1.name)
producer.publish(json.dumps({"n": 2}).encode("utf-8"), routing_key=q2.name)
thread = threading.Thread(target=adapted.run)
thread.start()
thread.join(1)
assert len(messages) == 2
q1.delete()
q2.delete()
def test_nack(channel, random_queue):
processor: missive.Processor[missive.JSONMessage] = missive.Processor()
flag = False
@processor.handle_for(always)
def catch_all(message, ctx):
nonlocal flag
flag = message.get_json()
ctx.nack()
adapted.shutdown_handler.set_flag()
adapted = RabbitMQAdapter(
missive.JSONMessage,
processor,
[random_queue.name],
url_or_conn=RABBITMQ_URL,
disable_shutdown_handler=True,
)
test_event = {"test-event": True}
producer = kombu.Producer(channel)
producer.publish(json.dumps(test_event).encode("utf-8"),
routing_key=random_queue.name)
thread = threading.Thread(target=adapted.run)
thread.start()
thread.join(1)
assert flag == test_event
# Assert it's left on the queue
message = random_queue.get()
assert message.delivery_info["redelivered"]
message.ack() # clear it
import json
from typing import cast, Dict
import missive
processor: missive.Processor[missive.JSONMessage] = missive.Processor()
db = []
@processor.handle_for(lambda m: cast(str,
cast(Dict, m.get_json())["flag"]) == "a")
def handle_as(message, ctx):
db.append(message.get_json())
ctx.ack()
def test_json():
with processor.test_client() as tc:
body = {"flag": "a"}
tc.send(missive.JSONMessage(json.dumps(body).encode("utf-8")))
assert db == [body]
)
from quarchive.messaging.publication import publish_message
from quarchive.messaging.message_lib import (
BookmarkCreated,
CrawlRequested,
Event,
FetchDiscussionsCommand,
HelloEvent,
IndexRequested,
NewIconFound,
)
from quarchive.messaging.receipt import PickleMessage
log = getLogger(__name__)
proc: missive.Processor[PickleMessage] = missive.Processor()
@proc.before_processing
def create_session_cls(proc_ctx: missive.ProcessingContext[PickleMessage]):
proc_ctx.state.sessionmaker = get_session_cls()
@proc.before_processing
def create_http_clients(proc_ctx):
http_client = requests.Session()
proc_ctx.state.http_client = http_client
proc_ctx.state.reddit_client = discussion_clients.RedditDiscussionClient(
http_client,
environ["QM_REDDIT_CLIENT_ID"],
environ["QM_REDDIT_CLIENT_SECRET"],
