async def nsq_publisher(self, queue: TopicQueue):
"""
Publisher for NSQ topic
:return:
"""
topic = queue.topic
self.logger.info("[nsq|%s] Starting NSQ publisher", topic)
while not queue.to_shutdown or not queue.is_empty():
# Message throttling. Wait and allow to collect more messages
await queue.wait(timeout=10, rate=config.nsqd.topic_mpub_rate)
# Get next batch up to `mpub_messages` messages or up to `mpub_size` size
messages = list(
queue.iter_get(
n=config.nsqd.mpub_messages,
size=config.nsqd.mpub_size,
total_overhead=4,
message_overhead=4,
)
)
if not messages:
continue
try:
self.logger.debug("[nsq|%s] Publishing %d messages", topic, len(messages))
await mpub(topic, messages, dcs=self.dcs)
except NSQPubError:
if queue.to_shutdown:
self.logger.debug(
"[nsq|%s] Failed to publish during shutdown. Dropping messages", topic
)
else:
# Return to queue
self.logger.info(
"[nsq|%s] Failed to publish. %d messages returned to queue",
topic,
len(messages),
)
queue.return_messages(messages)
del messages # Release memory
self.logger.info("[nsq|%s] Stopping NSQ publisher", topic)
# Queue is shut down and empty, notify
queue.notify_shutdown()
def test_is_empty(items, expected):
queue = TopicQueue("test_is_empty")
for item in items:
queue.put(item)
assert queue.is_empty() is expected