def test_redis_broker_maintains_backwards_compat_with_old_acks(redis_broker): # Given that I have an actor @dramatiq.actor def do_work(self): pass # And that actor has some old-style unacked messages expired_message_ids = set() valid_message_ids = set() for i in range(LUA_MAX_UNPACK_SIZE * 2): expired_message_id = b"expired-old-school-ack-%r" % i valid_message_id = b"valid-old-school-ack-%r" % i expired_message_ids.add(expired_message_id) valid_message_ids.add(valid_message_id) if redis.__version__.startswith("2."): redis_broker.client.zadd("dramatiq:default.acks", 0, expired_message_id) redis_broker.client.zadd("dramatiq:default.acks", current_millis(), valid_message_id) else: redis_broker.client.zadd("dramatiq:default.acks", {expired_message_id: 0}) redis_broker.client.zadd("dramatiq:default.acks", {valid_message_id: current_millis()}) # When maintenance runs for that actor's queue redis_broker.maintenance_chance = MAINTENANCE_SCALE redis_broker.do_qsize(do_work.queue_name) # Then maintenance should move the expired message to the new style acks set unacked = redis_broker.client.smembers("dramatiq:__acks__.%s.default" % redis_broker.broker_id) assert set(unacked) == expired_message_ids # And the valid message should stay in that set compat_unacked = redis_broker.client.zrangebyscore("dramatiq:default.acks", 0, "+inf") assert set(compat_unacked) == valid_message_ids
def do_work(): nonlocal failure_time, success_time if not failure_time: failure_time = current_millis() raise RuntimeError("First failure.") else: success_time = current_millis()
def test_redis_broker_maintains_backwards_compat_with_old_acks(redis_broker): # Given that I have an actor @dramatiq.actor def do_work(self): pass # And that actor has some old-style unacked messages expired_message_id = b"expired-old-school-ack" valid_message_id = b"valid-old-school-ack" redis_broker.client.zadd("dramatiq:default.acks", 0, expired_message_id) redis_broker.client.zadd("dramatiq:default.acks", current_millis(), valid_message_id) # When maintenance runs for that actor's queue redis_broker.maintenance_chance = MAINTENANCE_SCALE redis_broker.do_qsize(do_work.queue_name) # Then maintenance should move the expired message to the new style acks set unacked = redis_broker.client.smembers("dramatiq:__acks__.%s.default" % redis_broker.broker_id) assert set(unacked) == {expired_message_id} # And the valid message should stay in that set compat_unacked = redis_broker.client.zrangebyscore("dramatiq:default.acks", 0, "+inf") assert set(compat_unacked) == {valid_message_id}
def enqueue_using_session(self, db, message, delay=None): # Given a dramatiq message, saves it to the queue in the DB. queue_name = message.queue_name if delay is not None: queue_name = dq_name(queue_name) message.options["eta"] = current_millis() + delay db_message = DramatiqMessage(id=message.message_id, actor=message.actor_name, queue=queue_name) message_dict = message.asdict() # Drop these so we're not storing them in two places. del message_dict["message_id"] del message_dict["queue_name"] del message_dict["actor_name"] db_message.body = message_dict # Use merge rather than add since the message may already exist; # for instance this happens in case of retry. db_message = db.merge(db_message) # Explicitly wipe out the consumer, since if we've updated an existing # message it'll have to be consumed again. db_message.consumer_id = None
def after_process_message( self, broker: dramatiq.broker.Broker, message: dramatiq.message.Message, *, # This is on purpose, our tasks never return anything useful. result: None = None, exception: Optional[BaseException] = None) -> None: from . import get_logger from .metrics import TaskMetrics logger = get_logger() labels = (message.queue_name, message.actor_name) actor = broker.get_actor(message.actor_name) actor_arguments = _actor_arguments(logger, message, actor) message_start_time = self._message_start_times.pop( message.message_id, current_millis()) message_duration = current_millis() - message_start_time # Extract the poolname. `None` is a good starting value, but it turns out that most of the tasks # relate to a particular pool in one way or another. Some tasks are given the poolname as a parameter, # and some can tell us by attaching a note to the message. poolname: ActorArgumentType = None if 'poolname' in actor_arguments: poolname = actor_arguments['poolname'] elif message.options: poolname = get_metric_note(NOTE_POOLNAME) TaskMetrics.inc_message_durations(message.queue_name, message.actor_name, message_duration, poolname) TaskMetrics.dec_current_messages(*labels) TaskMetrics.inc_overall_messages(*labels) if exception is not None: TaskMetrics.inc_overall_errored_messages(*labels)
def before_process_message(self, broker: dramatiq.broker.Broker, message: dramatiq.message.Message) -> None: from .metrics import TaskMetrics labels = (message.queue_name, message.actor_name) if message.message_id in self._delayed_messages: self._delayed_messages.remove(message.message_id) TaskMetrics.dec_current_delayed_messages(message.queue_name, message.actor_name) TaskMetrics.inc_current_messages(*labels) self._message_start_times[message.message_id] = current_millis()
def test_redis_actors_can_have_their_messages_delayed(redis_broker, redis_worker): # Given that I have a database start_time, run_time = current_millis(), None # And an actor that records the time it ran @dramatiq.actor() def record(): nonlocal run_time run_time = current_millis() # If I send it a delayed message record.send_with_options(delay=1000) # Then join on the queue redis_broker.join(record.queue_name) redis_worker.join() # I expect that message to have been processed at least delayed milliseconds later assert run_time - start_time >= 1000
def record(): nonlocal run_time run_time = current_millis()
import dramatiq from dramatiq.brokers.redis import RedisBroker from dramatiq.common import current_millis from .common import skip_on_windows fakebroker = object() class BrokerHolder: fakebroker = object() broker = RedisBroker() loaded_at = current_millis() @dramatiq.actor(broker=broker) def write_loaded_at(filename): with open(filename, "w") as f: f.write(str(loaded_at)) @skip_on_windows def test_cli_fails_to_start_given_an_invalid_broker_name(start_cli): # Given that this module doesn't define a broker called "idontexist" # When I start the cli and point it at that broker proc = start_cli("tests.test_cli:idontexist", stdout=PIPE, stderr=STDOUT) proc.wait(60)