def test_redis_broker_maintains_backwards_compat_with_old_acks(redis_broker):
# Given that I have an actor
@dramatiq.actor
def do_work(self):
pass
# And that actor has some old-style unacked messages
expired_message_ids = set()
valid_message_ids = set()
for i in range(LUA_MAX_UNPACK_SIZE * 2):
expired_message_id = b"expired-old-school-ack-%r" % i
valid_message_id = b"valid-old-school-ack-%r" % i
expired_message_ids.add(expired_message_id)
valid_message_ids.add(valid_message_id)
if redis.__version__.startswith("2."):
redis_broker.client.zadd("dramatiq:default.acks", 0, expired_message_id)
redis_broker.client.zadd("dramatiq:default.acks", current_millis(), valid_message_id)
else:
redis_broker.client.zadd("dramatiq:default.acks", {expired_message_id: 0})
redis_broker.client.zadd("dramatiq:default.acks", {valid_message_id: current_millis()})
# When maintenance runs for that actor's queue
redis_broker.maintenance_chance = MAINTENANCE_SCALE
redis_broker.do_qsize(do_work.queue_name)
# Then maintenance should move the expired message to the new style acks set
unacked = redis_broker.client.smembers("dramatiq:__acks__.%s.default" % redis_broker.broker_id)
assert set(unacked) == expired_message_ids
# And the valid message should stay in that set
compat_unacked = redis_broker.client.zrangebyscore("dramatiq:default.acks", 0, "+inf")
assert set(compat_unacked) == valid_message_ids
def do_work():
nonlocal failure_time, success_time
if not failure_time:
failure_time = current_millis()
raise RuntimeError("First failure.")
else:
success_time = current_millis()
def test_redis_broker_maintains_backwards_compat_with_old_acks(redis_broker):
# Given that I have an actor
@dramatiq.actor
def do_work(self):
pass
# And that actor has some old-style unacked messages
expired_message_id = b"expired-old-school-ack"
valid_message_id = b"valid-old-school-ack"
redis_broker.client.zadd("dramatiq:default.acks", 0, expired_message_id)
redis_broker.client.zadd("dramatiq:default.acks", current_millis(),
valid_message_id)
# When maintenance runs for that actor's queue
redis_broker.maintenance_chance = MAINTENANCE_SCALE
redis_broker.do_qsize(do_work.queue_name)
# Then maintenance should move the expired message to the new style acks set
unacked = redis_broker.client.smembers("dramatiq:__acks__.%s.default" %
redis_broker.broker_id)
assert set(unacked) == {expired_message_id}
# And the valid message should stay in that set
compat_unacked = redis_broker.client.zrangebyscore("dramatiq:default.acks",
0, "+inf")
assert set(compat_unacked) == {valid_message_id}
def enqueue_using_session(self, db, message, delay=None):
# Given a dramatiq message, saves it to the queue in the DB.
queue_name = message.queue_name
if delay is not None:
queue_name = dq_name(queue_name)
message.options["eta"] = current_millis() + delay
db_message = DramatiqMessage(id=message.message_id,
actor=message.actor_name,
queue=queue_name)
message_dict = message.asdict()
# Drop these so we're not storing them in two places.
del message_dict["message_id"]
del message_dict["queue_name"]
del message_dict["actor_name"]
db_message.body = message_dict
# Use merge rather than add since the message may already exist;
# for instance this happens in case of retry.
db_message = db.merge(db_message)
# Explicitly wipe out the consumer, since if we've updated an existing
# message it'll have to be consumed again.
db_message.consumer_id = None
def after_process_message(
self,
broker: dramatiq.broker.Broker,
message: dramatiq.message.Message,
*,
# This is on purpose, our tasks never return anything useful.
result: None = None,
exception: Optional[BaseException] = None) -> None:
from . import get_logger
from .metrics import TaskMetrics
logger = get_logger()
labels = (message.queue_name, message.actor_name)
actor = broker.get_actor(message.actor_name)
actor_arguments = _actor_arguments(logger, message, actor)
message_start_time = self._message_start_times.pop(
message.message_id, current_millis())
message_duration = current_millis() - message_start_time
# Extract the poolname. `None` is a good starting value, but it turns out that most of the tasks
# relate to a particular pool in one way or another. Some tasks are given the poolname as a parameter,
# and some can tell us by attaching a note to the message.
poolname: ActorArgumentType = None
if 'poolname' in actor_arguments:
poolname = actor_arguments['poolname']
elif message.options:
poolname = get_metric_note(NOTE_POOLNAME)
TaskMetrics.inc_message_durations(message.queue_name,
message.actor_name, message_duration,
poolname)
TaskMetrics.dec_current_messages(*labels)
TaskMetrics.inc_overall_messages(*labels)
if exception is not None:
TaskMetrics.inc_overall_errored_messages(*labels)
def before_process_message(self, broker: dramatiq.broker.Broker,
message: dramatiq.message.Message) -> None:
from .metrics import TaskMetrics
labels = (message.queue_name, message.actor_name)
if message.message_id in self._delayed_messages:
self._delayed_messages.remove(message.message_id)
TaskMetrics.dec_current_delayed_messages(message.queue_name,
message.actor_name)
TaskMetrics.inc_current_messages(*labels)
self._message_start_times[message.message_id] = current_millis()
def test_redis_actors_can_have_their_messages_delayed(redis_broker, redis_worker):
# Given that I have a database
start_time, run_time = current_millis(), None
# And an actor that records the time it ran
@dramatiq.actor()
def record():
nonlocal run_time
run_time = current_millis()
# If I send it a delayed message
record.send_with_options(delay=1000)
# Then join on the queue
redis_broker.join(record.queue_name)
redis_worker.join()
# I expect that message to have been processed at least delayed milliseconds later
assert run_time - start_time >= 1000
def record():
nonlocal run_time
run_time = current_millis()
import dramatiq
from dramatiq.brokers.redis import RedisBroker
from dramatiq.common import current_millis
from .common import skip_on_windows
fakebroker = object()
class BrokerHolder:
fakebroker = object()
broker = RedisBroker()
loaded_at = current_millis()
@dramatiq.actor(broker=broker)
def write_loaded_at(filename):
with open(filename, "w") as f:
f.write(str(loaded_at))
@skip_on_windows
def test_cli_fails_to_start_given_an_invalid_broker_name(start_cli):
# Given that this module doesn't define a broker called "idontexist"
# When I start the cli and point it at that broker
proc = start_cli("tests.test_cli:idontexist", stdout=PIPE, stderr=STDOUT)
proc.wait(60)
