def after_process_message(
self,
broker: Broker,
message: Message,
*,
result: Any = None,
exception: Optional[BaseException] = None,
) -> None:
message.fail()
def enqueue(self, message: dramatiq.Message, *, delay: Optional[int] = None) -> dramatiq.Message:
queue_name = message.queue_name
if delay is None:
queue = self.queues[queue_name]
delay_seconds = 0
elif delay <= 900000:
queue = self.queues[queue_name]
delay_seconds = int(delay / 1000)
else:
raise ValueError("Messages in SQS cannot be delayed for longer than 15 minutes.")
encoded_message = b64encode(message.encode()).decode()
if len(encoded_message) > MAX_MESSAGE_SIZE:
raise RuntimeError("Messages in SQS can be at most 256KiB large.")
self.logger.debug("Enqueueing message %r on queue %r.", message.message_id, queue_name)
self.emit_before("enqueue", message, delay)
send_message_args = {
"MessageBody": encoded_message,
"DelaySeconds": delay_seconds
}
message_group_id = message.options.get("message_group_id")
if message_group_id is not None:
send_message_args["MessageGroupId"] = message_group_id
queue.send_message({**queue_args})
self.emit_after("enqueue", message, delay)
return message
def test_redis_consumer_nack_does_not_raise_on_missing_id(redis_worker):
"""
Reproduces issue 320, exception when the message id is not found.
https://github.com/Bogdanp/dramatiq/issues/320
Before the change would raise:
redis.exceptions.ResponseError: Error running script (call to f_e9668bc413bd4a2d63c8108b124f5b7df0d01263):
@user_script:218: @user_script: 218: Lua redis() command arguments must be strings or integers
"""
# Given that I have an actor
@dramatiq.actor(max_retries=0)
def do_work():
raise RuntimeError
consumer = redis_worker.consumers[do_work.queue_name].consumer
# If I send a bogus message to nack, I expect no exception to be raised.
message = Message(
queue_name=do_work.queue_name,
actor_name=do_work.actor_name,
args=(), kwargs={},
options={"redis_message_id": "XXXXXXXXX"}
) # Bogus ID
consumer.nack(message)
def __consume_one(self, db):
# First check if we're still allowed to fetch more.
have_count = (db.query(DramatiqMessage).filter(
DramatiqMessage.consumer_id == self.__consumer_id).count())
if have_count >= self.__prefetch:
LOG.debug(
"Too many pending messages (%s), not consuming more",
have_count,
)
return
# Take any message in the queue not yet assigned to a consumer.
db_message = (db.query(DramatiqMessage).with_for_update().filter(
DramatiqMessage.consumer_id == None).filter(
DramatiqMessage.queue == self.__queue_name).first())
if db_message:
LOG.info("%s: consumed %s", self.__consumer_id, db_message.id)
body = db_message.body
out = Message(
queue_name=db_message.queue,
actor_name=db_message.actor,
message_id=db_message.id,
**body,
)
# Mark it as ours.
db_message.consumer_id = self.__consumer_id
return MessageProxy(out)
LOG.debug("%s: did not find any messages", self.__consumer_id)
def _call_actor(self, actor_name, *args, **kwargs):
return self.broker.enqueue(Message(
queue_name="default",
actor_name=actor_name,
args=args, kwargs=kwargs,
options={},
))
def mocked_next_message():
consumer_thread.stop()
message = Message(queue_name='queue',
actor_name='actor',
args=[],
kwargs={},
options=message_options)
return MessageProxy(message)
def test_actors_can_be_sent_messages(stub_broker):
# Given that I have an actor
@dramatiq.actor
def add(x, y):
return x + y
# If I send it a message,
# I expect it to enqueue a message
enqueued_message = add.send(1, 2)
enqueued_message_data = stub_broker.queues["default"].get(timeout=1)
assert enqueued_message == Message.decode(enqueued_message_data)
def lookups(self, request, model_admin):
lookup_choices = set()
messages = (Message.decode(bytes(encoded_message))
for encoded_message in model_admin.model.tasks.values_list(
"message_data", flat=True))
for message in messages:
choice = getattr(message, self.parameter_name)
lookup_choices.add((choice, "{} (slow)".format(choice)))
return sorted(lookup_choices)
def create_message(cls, actor, *args, **kwargs):
"""Prepare a message and add an entry in the Message Model
:param actor: an Actor instance
:param delay: use for postcommit hook send2broker
:param _*args: args of the actor
:param _*_*kwargs: kwargs of the actor
:rtype: dramatiq message instance
"""
delay = kwargs.pop('delay', None)
run_at = kwargs.pop('run_at', None)
if not isinstance(actor, Actor):
logger.warning("[create_message] can't work without an actor")
return None
message = DramatiqMessage(queue_name=actor.queue_name,
actor_name=actor.actor_name,
args=args,
kwargs=kwargs,
options=dict())
message.broker = actor.broker
cls.Message.insert(id=message.message_id,
message=loads(message.encode()))
cls.postcommit_hook('send2broker', message, delay=delay, run_at=run_at)
return message
def test_redis_messages_revived_on_maintenance(redis_broker, redis_worker):
# Given that I have an actor that takes a long time to run
@dramatiq.actor
def do_work():
time.sleep(5)
do_work.send()
# When I kill the worker prematurely
redis_worker.stop(timeout=1)
# And maintenance runs for that actor's queue
redis_broker.heartbeat_timeout = 0
redis_broker.maintenance_chance = MAINTENANCE_SCALE
redis_broker.do_qsize(do_work.queue_name)
# My message should have a revived key on it
messages = redis_broker.do_fetch(do_work.queue_name, 1)
assert len(messages) == 1
assert Message.decode(messages[0]).options["revives"] == 1
redis_worker.start()
time.sleep(1)
redis_worker.stop(timeout=1)
# And maintenance runs for that actor"s queue
redis_broker.heartbeat_timeout = 0
redis_broker.maintenance_chance = MAINTENANCE_SCALE
redis_broker.do_qsize(do_work.queue_name)
# My message should have a revived key on it
messages = redis_broker.do_fetch(do_work.queue_name, 1)
assert len(messages) == 1
assert Message.decode(messages[0]).options["revives"] == 2
def test_messages_belonging_to_missing_actors_are_rejected(stub_broker, stub_worker):
# Given that I have a broker without actors
# If I send it a message
message = Message(
queue_name="some-queue",
actor_name="some-actor",
args=(), kwargs={},
options={},
)
stub_broker.declare_queue("some-queue")
stub_broker.enqueue(message)
# Then join on the queue
stub_broker.join("some-queue")
stub_worker.join()
# I expect the message to end up on the dead letter queue
assert stub_broker.dead_letters == [message]
def test_rabbitmq_messages_belonging_to_missing_actors_are_rejected(rabbitmq_broker, rabbitmq_worker):
# Given that I have a broker without actors
# If I send it a message
message = Message(
queue_name="some-queue",
actor_name="some-actor",
args=(), kwargs={},
options={},
)
rabbitmq_broker.declare_queue(message.queue_name)
rabbitmq_broker.enqueue(message)
# Then join on the queue
rabbitmq_broker.join(message.queue_name)
rabbitmq_worker.join()
# I expect the message to end up on the dead letter queue
_, _, dead = rabbitmq_broker.get_queue_message_counts(message.queue_name)
assert dead == 1
def enqueue(self,
message: dramatiq.Message,
*,
delay: Optional[int] = None) -> dramatiq.Message:
queue_name = message.queue_name
if delay is None:
queue = self.queues[queue_name]
delay_seconds = 0
elif delay <= 900000:
queue = self.queues[queue_name]
delay_seconds = int(delay / 1000)
else:
raise ValueError(
"Messages in SQS cannot be delayed for longer than 15 minutes."
)
encoded_message = b64encode(message.encode()).decode()
if len(encoded_message) > MAX_MESSAGE_SIZE:
raise RuntimeError("Messages in SQS can be at most 256KiB large.")
self.logger.debug("Enqueueing message %r on queue %r.",
message.message_id, queue_name)
self.emit_before("enqueue", message, delay)
if is_fifo(queue_name):
message_group_id = message.kwargs.get("message_group_id",
DEFAULT_MESSAGE_GROUP_ID)
message_deduplication_id = message.kwargs.get(
"message_deduplication_id")
if message_deduplication_id:
queue.send_message(
MessageBody=encoded_message,
MessageGroupId=message_group_id,
MessageDeduplicationId=message_deduplication_id)
else:
queue.send_message(MessageBody=encoded_message,
MessageGroupId=message_group_id)
else:
queue.send_message(
MessageBody=encoded_message,
DelaySeconds=delay_seconds,
)
self.emit_after("enqueue", message, delay)
return message
def test_redis_messages_belonging_to_missing_actors_are_rejected(redis_broker, redis_worker):
# Given that I have a broker without actors
# If I send it a message
message = Message(
queue_name="some-queue",
actor_name="some-actor",
args=(), kwargs={},
options={},
)
redis_broker.declare_queue("some-queue")
message = redis_broker.enqueue(message)
# Then join on the queue
redis_broker.join("some-queue")
redis_worker.join()
# I expect the message to end up on the dead letter queue
dead_queue_name = "dramatiq:%s" % xq_name("some-queue")
dead_ids = redis_broker.client.zrangebyscore(dead_queue_name, 0, "+inf")
assert message.options["redis_message_id"].encode("utf-8") in dead_ids
def enqueue(self,
message: dramatiq.Message,
*,
delay: Optional[int] = None) -> dramatiq.Message:
queue_name = message.queue_name
queue = self.queues[queue_name]
queue_url = queue.url
if not queue_url.endswith(".fifo"):
if delay is None:
delay_seconds = 0
elif delay <= 900000:
delay_seconds = int(delay / 1000)
else:
raise ValueError(
"Messages in SQS cannot be delayed for longer than 15 minutes."
)
encoded_message = b64encode(message.encode()).decode()
if len(encoded_message) > MAX_MESSAGE_SIZE:
raise RuntimeError("Messages in SQS can be at most 256KiB large.")
self.logger.debug("Enqueueing message %r on queue %r.",
message.message_id, queue_name)
self.emit_before("enqueue", message, delay)
if queue_url.endswith(".fifo"):
queue.send_message(
MessageBody=encoded_message,
MessageGroupId="4", # 4 is a great number
)
else:
queue.send_message(
MessageBody=encoded_message,
DelaySeconds=delay_seconds,
)
self.emit_after("enqueue", message, delay)
return message
def enqueue_dramatiq_job(**message):
broker = get_broker()
broker.enqueue(Message(**message))
def message(self):
if django.VERSION[0] < 2:
return Message.decode(self.message_data.tobytes())
return Message.decode(self.message_data)
def message(self):
return Message.decode(bytes(self.message_data))
