def after_process_message( self, broker: Broker, message: Message, *, result: Any = None, exception: Optional[BaseException] = None, ) -> None: message.fail()
def enqueue(self, message: dramatiq.Message, *, delay: Optional[int] = None) -> dramatiq.Message: queue_name = message.queue_name if delay is None: queue = self.queues[queue_name] delay_seconds = 0 elif delay <= 900000: queue = self.queues[queue_name] delay_seconds = int(delay / 1000) else: raise ValueError("Messages in SQS cannot be delayed for longer than 15 minutes.") encoded_message = b64encode(message.encode()).decode() if len(encoded_message) > MAX_MESSAGE_SIZE: raise RuntimeError("Messages in SQS can be at most 256KiB large.") self.logger.debug("Enqueueing message %r on queue %r.", message.message_id, queue_name) self.emit_before("enqueue", message, delay) send_message_args = { "MessageBody": encoded_message, "DelaySeconds": delay_seconds } message_group_id = message.options.get("message_group_id") if message_group_id is not None: send_message_args["MessageGroupId"] = message_group_id queue.send_message({**queue_args}) self.emit_after("enqueue", message, delay) return message
def test_redis_consumer_nack_does_not_raise_on_missing_id(redis_worker): """ Reproduces issue 320, exception when the message id is not found. https://github.com/Bogdanp/dramatiq/issues/320 Before the change would raise: redis.exceptions.ResponseError: Error running script (call to f_e9668bc413bd4a2d63c8108b124f5b7df0d01263): @user_script:218: @user_script: 218: Lua redis() command arguments must be strings or integers """ # Given that I have an actor @dramatiq.actor(max_retries=0) def do_work(): raise RuntimeError consumer = redis_worker.consumers[do_work.queue_name].consumer # If I send a bogus message to nack, I expect no exception to be raised. message = Message( queue_name=do_work.queue_name, actor_name=do_work.actor_name, args=(), kwargs={}, options={"redis_message_id": "XXXXXXXXX"} ) # Bogus ID consumer.nack(message)
def __consume_one(self, db): # First check if we're still allowed to fetch more. have_count = (db.query(DramatiqMessage).filter( DramatiqMessage.consumer_id == self.__consumer_id).count()) if have_count >= self.__prefetch: LOG.debug( "Too many pending messages (%s), not consuming more", have_count, ) return # Take any message in the queue not yet assigned to a consumer. db_message = (db.query(DramatiqMessage).with_for_update().filter( DramatiqMessage.consumer_id == None).filter( DramatiqMessage.queue == self.__queue_name).first()) if db_message: LOG.info("%s: consumed %s", self.__consumer_id, db_message.id) body = db_message.body out = Message( queue_name=db_message.queue, actor_name=db_message.actor, message_id=db_message.id, **body, ) # Mark it as ours. db_message.consumer_id = self.__consumer_id return MessageProxy(out) LOG.debug("%s: did not find any messages", self.__consumer_id)
def _call_actor(self, actor_name, *args, **kwargs): return self.broker.enqueue(Message( queue_name="default", actor_name=actor_name, args=args, kwargs=kwargs, options={}, ))
def mocked_next_message(): consumer_thread.stop() message = Message(queue_name='queue', actor_name='actor', args=[], kwargs={}, options=message_options) return MessageProxy(message)
def test_actors_can_be_sent_messages(stub_broker): # Given that I have an actor @dramatiq.actor def add(x, y): return x + y # If I send it a message, # I expect it to enqueue a message enqueued_message = add.send(1, 2) enqueued_message_data = stub_broker.queues["default"].get(timeout=1) assert enqueued_message == Message.decode(enqueued_message_data)
def lookups(self, request, model_admin): lookup_choices = set() messages = (Message.decode(bytes(encoded_message)) for encoded_message in model_admin.model.tasks.values_list( "message_data", flat=True)) for message in messages: choice = getattr(message, self.parameter_name) lookup_choices.add((choice, "{} (slow)".format(choice))) return sorted(lookup_choices)
def create_message(cls, actor, *args, **kwargs): """Prepare a message and add an entry in the Message Model :param actor: an Actor instance :param delay: use for postcommit hook send2broker :param _*args: args of the actor :param _*_*kwargs: kwargs of the actor :rtype: dramatiq message instance """ delay = kwargs.pop('delay', None) run_at = kwargs.pop('run_at', None) if not isinstance(actor, Actor): logger.warning("[create_message] can't work without an actor") return None message = DramatiqMessage(queue_name=actor.queue_name, actor_name=actor.actor_name, args=args, kwargs=kwargs, options=dict()) message.broker = actor.broker cls.Message.insert(id=message.message_id, message=loads(message.encode())) cls.postcommit_hook('send2broker', message, delay=delay, run_at=run_at) return message
def test_redis_messages_revived_on_maintenance(redis_broker, redis_worker): # Given that I have an actor that takes a long time to run @dramatiq.actor def do_work(): time.sleep(5) do_work.send() # When I kill the worker prematurely redis_worker.stop(timeout=1) # And maintenance runs for that actor's queue redis_broker.heartbeat_timeout = 0 redis_broker.maintenance_chance = MAINTENANCE_SCALE redis_broker.do_qsize(do_work.queue_name) # My message should have a revived key on it messages = redis_broker.do_fetch(do_work.queue_name, 1) assert len(messages) == 1 assert Message.decode(messages[0]).options["revives"] == 1 redis_worker.start() time.sleep(1) redis_worker.stop(timeout=1) # And maintenance runs for that actor"s queue redis_broker.heartbeat_timeout = 0 redis_broker.maintenance_chance = MAINTENANCE_SCALE redis_broker.do_qsize(do_work.queue_name) # My message should have a revived key on it messages = redis_broker.do_fetch(do_work.queue_name, 1) assert len(messages) == 1 assert Message.decode(messages[0]).options["revives"] == 2
def test_messages_belonging_to_missing_actors_are_rejected(stub_broker, stub_worker): # Given that I have a broker without actors # If I send it a message message = Message( queue_name="some-queue", actor_name="some-actor", args=(), kwargs={}, options={}, ) stub_broker.declare_queue("some-queue") stub_broker.enqueue(message) # Then join on the queue stub_broker.join("some-queue") stub_worker.join() # I expect the message to end up on the dead letter queue assert stub_broker.dead_letters == [message]
def test_rabbitmq_messages_belonging_to_missing_actors_are_rejected(rabbitmq_broker, rabbitmq_worker): # Given that I have a broker without actors # If I send it a message message = Message( queue_name="some-queue", actor_name="some-actor", args=(), kwargs={}, options={}, ) rabbitmq_broker.declare_queue(message.queue_name) rabbitmq_broker.enqueue(message) # Then join on the queue rabbitmq_broker.join(message.queue_name) rabbitmq_worker.join() # I expect the message to end up on the dead letter queue _, _, dead = rabbitmq_broker.get_queue_message_counts(message.queue_name) assert dead == 1
def enqueue(self, message: dramatiq.Message, *, delay: Optional[int] = None) -> dramatiq.Message: queue_name = message.queue_name if delay is None: queue = self.queues[queue_name] delay_seconds = 0 elif delay <= 900000: queue = self.queues[queue_name] delay_seconds = int(delay / 1000) else: raise ValueError( "Messages in SQS cannot be delayed for longer than 15 minutes." ) encoded_message = b64encode(message.encode()).decode() if len(encoded_message) > MAX_MESSAGE_SIZE: raise RuntimeError("Messages in SQS can be at most 256KiB large.") self.logger.debug("Enqueueing message %r on queue %r.", message.message_id, queue_name) self.emit_before("enqueue", message, delay) if is_fifo(queue_name): message_group_id = message.kwargs.get("message_group_id", DEFAULT_MESSAGE_GROUP_ID) message_deduplication_id = message.kwargs.get( "message_deduplication_id") if message_deduplication_id: queue.send_message( MessageBody=encoded_message, MessageGroupId=message_group_id, MessageDeduplicationId=message_deduplication_id) else: queue.send_message(MessageBody=encoded_message, MessageGroupId=message_group_id) else: queue.send_message( MessageBody=encoded_message, DelaySeconds=delay_seconds, ) self.emit_after("enqueue", message, delay) return message
def test_redis_messages_belonging_to_missing_actors_are_rejected(redis_broker, redis_worker): # Given that I have a broker without actors # If I send it a message message = Message( queue_name="some-queue", actor_name="some-actor", args=(), kwargs={}, options={}, ) redis_broker.declare_queue("some-queue") message = redis_broker.enqueue(message) # Then join on the queue redis_broker.join("some-queue") redis_worker.join() # I expect the message to end up on the dead letter queue dead_queue_name = "dramatiq:%s" % xq_name("some-queue") dead_ids = redis_broker.client.zrangebyscore(dead_queue_name, 0, "+inf") assert message.options["redis_message_id"].encode("utf-8") in dead_ids
def enqueue(self, message: dramatiq.Message, *, delay: Optional[int] = None) -> dramatiq.Message: queue_name = message.queue_name queue = self.queues[queue_name] queue_url = queue.url if not queue_url.endswith(".fifo"): if delay is None: delay_seconds = 0 elif delay <= 900000: delay_seconds = int(delay / 1000) else: raise ValueError( "Messages in SQS cannot be delayed for longer than 15 minutes." ) encoded_message = b64encode(message.encode()).decode() if len(encoded_message) > MAX_MESSAGE_SIZE: raise RuntimeError("Messages in SQS can be at most 256KiB large.") self.logger.debug("Enqueueing message %r on queue %r.", message.message_id, queue_name) self.emit_before("enqueue", message, delay) if queue_url.endswith(".fifo"): queue.send_message( MessageBody=encoded_message, MessageGroupId="4", # 4 is a great number ) else: queue.send_message( MessageBody=encoded_message, DelaySeconds=delay_seconds, ) self.emit_after("enqueue", message, delay) return message
def enqueue_dramatiq_job(**message): broker = get_broker() broker.enqueue(Message(**message))
def message(self): if django.VERSION[0] < 2: return Message.decode(self.message_data.tobytes()) return Message.decode(self.message_data)
def message(self): return Message.decode(bytes(self.message_data))