def test_message_receipt(client, random_topic): processor: missive.Processor[missive.JSONMessage] = missive.Processor() messages = set() @processor.handle_for(always) def catch_all(message, ctx): messages.add(message.get_json()["test_id"]) adapted.shutdown_handler.set_flag() logger.info("caught %s", message.get_json()) adapted = MQTTAdapter( missive.JSONMessage, processor, [random_topic], host=MQTT_HOST, port=MQTT_PORT, ) test_event = make_test_message_body() thread = threading.Thread(target=adapted.run) thread.start() # FIXME: There has to be a better way than this but there are a lot of # small races here and it's difficult to do it more elegantly. while thread.is_alive(): client.publish(random_topic, json.dumps(test_event).encode("utf-8")) sleep(0.1) assert test_event["test_id"] in messages
def test_message_receipt(redis_client): processor: missive.Processor[missive.JSONMessage] = missive.Processor() flag = False @processor.handle_for(always) def catch_all(message, ctx): nonlocal flag flag = message.get_json() ctx.ack() adapted.shutdown_handler.set_flag() adapted = RedisPubSubAdapter(missive.JSONMessage, processor, ["test-channel"]) thread = threading.Thread(target=adapted.run) thread.start() while adapted.thread is None: time.sleep(0.01) test_event = {"test-event": True} redis_client.publish("test-channel", json.dumps(test_event)) adapted.thread.join(1) assert flag == test_event
def test_passing_a_conn(channel, random_queue, connection): processor: missive.Processor[missive.JSONMessage] = missive.Processor() flag = False @processor.handle_for(always) def catch_all(message, ctx): nonlocal flag flag = message.get_json() ctx.ack() adapted.shutdown_handler.set_flag() adapted = RabbitMQAdapter( missive.JSONMessage, processor, [random_queue.name], url_or_conn=connection, disable_shutdown_handler=True, ) test_event = {"test-event": True} producer = kombu.Producer(channel) producer.publish(json.dumps(test_event).encode("utf-8"), routing_key=random_queue.name) thread = threading.Thread(target=adapted.run) thread.start() thread.join(1) assert flag == test_event # Assert nothing left on the queue assert random_queue.get() is None
def init_proc(pool) -> missive.Processor[missive.JSONMessage]: proc: missive.Processor[missive.JSONMessage] = missive.Processor() @proc.before_processing def create_session( proc_ctx: missive.ProcessingContext[missive.JSONMessage]): # pretend connection pool proc_ctx.state.pool = pool @proc.before_handling def create_connection( proc_ctx: missive.ProcessingContext, handling_ctx: missive.HandlingContext[missive.JSONMessage], ): handling_ctx.state.conn = proc_ctx.state.pool.pop() handling_ctx.state.conn.open() @proc.handle_for(TypeMatcher("happy")) def happy_handler( message: missive.JSONMessage, handling_ctx: missive.HandlingContext[missive.JSONMessage], ): handling_ctx.ack() @proc.after_handling def return_connection(proc_ctx, handling_ctx): handling_ctx.state.conn.commit() proc_ctx.state.pool.append(handling_ctx.state.conn) @proc.after_processing def close_session(proc_ctx): for conn in proc_ctx.state.pool: conn.close() return proc
def test_two_handlers_with_the_same_matcher(): processor: missive.Processor[missive.JSONMessage] = missive.Processor() @processor.handle_for(always) def handler_1(message, ctx): ctx.ack(message) with pytest.raises(RuntimeError): @processor.handle_for(always) def handler_2(message, ctx): ctx.ack(message)
def test_exception_raised_no_dlq(): """If an exception is raised and there's no DLQ, the proc should crash, (without acking).""" proc: missive.Processor[missive.RawMessage] = missive.Processor() @proc.handle_for(always) def crash(message, ctx): raise RuntimeError("bad bytes!") with proc.test_client() as test_client: blank_message = missive.RawMessage(b"") with pytest.raises(RuntimeError): test_client.send(blank_message)
def test_no_matching_handler(): processor: m.Processor[m.RawMessage] = m.Processor() @processor.handle_for(never) def non_matching_handler(message, ctx): assert False with processor.test_client() as test_client: blank_message = m.RawMessage(b"") with pytest.raises(RuntimeError): test_client.send(blank_message) assert blank_message not in test_client.acked
def test_exception_raised_with_a_dlq(): """If an exception is raised and there's a DLQ, it's written to the DLQ, acked and the processor gets on with it's life.""" proc: missive.Processor[missive.RawMessage] = missive.Processor() dlq: missive.DLQ = {} proc.set_dlq(dlq) @proc.handle_for(always) def crash(message, ctx): raise RuntimeError("bad bytes!") with proc.test_client() as test_client: blank_message = missive.RawMessage(b"") test_client.send(blank_message) assert dlq == {blank_message.message_id: (blank_message, "bad bytes!")} assert blank_message in test_client.acked
def test_no_matching_handler(): """Messages for which no handler matches should be written to the DLQ""" dlq: Dict = {} processor: m.Processor[m.RawMessage] = m.Processor() processor.set_dlq(dlq) @processor.handle_for(never) def non_matching_handler(message, ctx): assert False with processor.test_client() as test_client: blank_message = m.RawMessage(b"") test_client.send(blank_message) assert blank_message in test_client.acked assert list(dlq.values()) == [(blank_message, "no matching handlers")]
def test_one_matching_handler(): processor: m.Processor[m.RawMessage] = m.Processor() flag = False @processor.handle_for(always) def flip_bit(message: m.RawMessage, ctx: m.HandlingContext[m.RawMessage]) -> None: nonlocal flag flag = True ctx.ack() with processor.test_client() as test_client: blank_message = m.RawMessage(b"") test_client.send(blank_message) assert flag assert blank_message in test_client.acked
def test_multiple_matching_handlers(): processor: m.Processor[m.RawMessage] = m.Processor() @processor.handle_for(always) def a_matching_handler(message, ctx): message.ack() @processor.handle_for(lambda m: True) def another_matching_handler(message, ctx): message.ack() with processor.test_client() as test_client: blank_message = m.RawMessage(b"") with pytest.raises(RuntimeError): test_client.send(blank_message) assert blank_message not in test_client.acked
def test_multiple_matching_handlers(): dlq: Dict = {} processor: m.Processor[m.RawMessage] = m.Processor() processor.set_dlq(dlq) @processor.handle_for(always) def a_matching_handler(message, ctx): ctx.ack(message) @processor.handle_for(lambda m: True) def another_matching_handler(message, ctx): message.ack() with processor.test_client() as test_client: blank_message = m.RawMessage(b"") test_client.send(blank_message) assert blank_message in test_client.acked assert list(dlq.values()) == [(blank_message, "multiple matching handlers")]
def test_no_dlq_required(): """Test the happy path where the DLQ is never used""" dlq: Dict = {} processor: m.Processor[m.RawMessage] = m.Processor() processor.set_dlq(dlq) flag = False @processor.handle_for(always) def flip_bit(message: m.RawMessage, ctx: m.HandlingContext[m.RawMessage]) -> None: nonlocal flag flag = True ctx.ack() with processor.test_client() as test_client: blank_message = m.RawMessage(b"") test_client.send(blank_message) assert flag assert blank_message in test_client.acked
def test_one_matching_handler_among_multiple(): processor: m.Processor[m.RawMessage] = m.Processor() flag = False @processor.handle_for(always) def a_matching_handler(message, ctx): nonlocal flag flag = True ctx.ack() @processor.handle_for(never) def another_handler(message, ctx): message.ack() with processor.test_client() as test_client: blank_message = m.RawMessage(b"") test_client.send(blank_message) assert flag assert blank_message in test_client.acked
def test_receipt_from_multiple_queues(channel): q1 = make_random_queue(channel) q1.declare() q2 = make_random_queue(channel) q2.declare() messages = set() processor: missive.Processor[missive.JSONMessage] = missive.Processor() @processor.handle_for(always) def catch_all(message, ctx): messages.add(message.get_json()["n"]) ctx.ack() if len(messages) >= 2: adapted.shutdown_handler.set_flag() adapted = RabbitMQAdapter( missive.JSONMessage, processor, [q1.name, q2.name], url_or_conn=RABBITMQ_URL, disable_shutdown_handler=True, ) producer = kombu.Producer(channel) producer.publish(json.dumps({"n": 1}).encode("utf-8"), routing_key=q1.name) producer.publish(json.dumps({"n": 2}).encode("utf-8"), routing_key=q2.name) thread = threading.Thread(target=adapted.run) thread.start() thread.join(1) assert len(messages) == 2 q1.delete() q2.delete()
def test_nack(channel, random_queue): processor: missive.Processor[missive.JSONMessage] = missive.Processor() flag = False @processor.handle_for(always) def catch_all(message, ctx): nonlocal flag flag = message.get_json() ctx.nack() adapted.shutdown_handler.set_flag() adapted = RabbitMQAdapter( missive.JSONMessage, processor, [random_queue.name], url_or_conn=RABBITMQ_URL, disable_shutdown_handler=True, ) test_event = {"test-event": True} producer = kombu.Producer(channel) producer.publish(json.dumps(test_event).encode("utf-8"), routing_key=random_queue.name) thread = threading.Thread(target=adapted.run) thread.start() thread.join(1) assert flag == test_event # Assert it's left on the queue message = random_queue.get() assert message.delivery_info["redelivered"] message.ack() # clear it
import json from typing import cast, Dict import missive processor: missive.Processor[missive.JSONMessage] = missive.Processor() db = [] @processor.handle_for(lambda m: cast(str, cast(Dict, m.get_json())["flag"]) == "a") def handle_as(message, ctx): db.append(message.get_json()) ctx.ack() def test_json(): with processor.test_client() as tc: body = {"flag": "a"} tc.send(missive.JSONMessage(json.dumps(body).encode("utf-8"))) assert db == [body]
) from quarchive.messaging.publication import publish_message from quarchive.messaging.message_lib import ( BookmarkCreated, CrawlRequested, Event, FetchDiscussionsCommand, HelloEvent, IndexRequested, NewIconFound, ) from quarchive.messaging.receipt import PickleMessage log = getLogger(__name__) proc: missive.Processor[PickleMessage] = missive.Processor() @proc.before_processing def create_session_cls(proc_ctx: missive.ProcessingContext[PickleMessage]): proc_ctx.state.sessionmaker = get_session_cls() @proc.before_processing def create_http_clients(proc_ctx): http_client = requests.Session() proc_ctx.state.http_client = http_client proc_ctx.state.reddit_client = discussion_clients.RedditDiscussionClient( http_client, environ["QM_REDDIT_CLIENT_ID"], environ["QM_REDDIT_CLIENT_SECRET"],