def test_reader_messages(self):
self.msg_count = 0
num_messages = 500
topic = "test_reader_msgs_%s" % time.time()
self._send_messages(topic, num_messages, b"sup")
def handler(msg):
assert msg.body == b"sup"
self.msg_count += 1
if self.msg_count >= num_messages:
self.stop()
return True
r = Reader(
nsqd_tcp_addresses=["127.0.0.1:4150"],
topic=topic,
channel="ch",
io_loop=self.io_loop,
message_handler=handler,
max_in_flight=100,
**self.identify_options
)
self.wait()
r.close()
def test_reader_messages(self):
self.msg_count = 0
num_messages = 1
topic = 'test_lookupd_msgs_%s' % int(time.time())
self._send_messages(topic, num_messages, b'sup')
def handler(msg):
assert msg.body == b'sup'
self.msg_count += 1
if self.msg_count >= num_messages:
self.stop()
return True
r = Reader(lookupd_http_addresses=['http://127.0.0.1:4161'],
topic=topic,
channel='ch',
io_loop=self.io_loop,
message_handler=handler,
lookupd_poll_interval=1,
lookupd_poll_jitter=0.01,
max_in_flight=100,
**self.identify_options)
self.wait()
r.close()
def test_reader_coro(self):
self.msg_count = 0
num_messages = 20
topic = 'test_reader_msgs_%s' % time.time()
self._send_messages(topic, num_messages, b'sup')
@tornado.gen.coroutine
def handler(msg):
yield tornado.gen.sleep(0.1)
self.msg_count += 1
if self.msg_count >= num_messages:
self.stop()
raise tornado.gen.Return(True)
r = Reader(nsqd_tcp_addresses=['127.0.0.1:4150'],
topic=topic,
channel='ch',
message_handler=handler,
max_in_flight=10,
**self.identify_options)
self.wait()
r.close()
assert self.msg_count == num_messages
def test_reader_messages(self):
self.msg_count = 0
num_messages = 1
topic = 'test_lookupd_msgs_%s' % int(time.time())
self._send_messages(topic, num_messages, b'sup')
def handler(msg):
assert msg.body == b'sup'
self.msg_count += 1
if self.msg_count >= num_messages:
self.stop()
return True
r = Reader(lookupd_http_addresses=['http://127.0.0.1:4161'],
topic=topic,
channel='ch',
message_handler=handler,
lookupd_poll_interval=1,
lookupd_poll_jitter=0.01,
max_in_flight=100,
**self.identify_options)
self.wait()
r.close()
def test_reader_messages(self):
self.msg_count = 0
num_messages = 300
topic = 'test_reader_msgs_%s' % time.time()
self._send_messages(topic, num_messages, b'sup')
def handler(msg):
assert msg.body == b'sup'
self.msg_count += 1
if self.msg_count >= num_messages:
self.stop()
return True
r = Reader(nsqd_tcp_addresses=['127.0.0.1:4150'],
topic=topic, channel='ch',
message_handler=handler, max_in_flight=100,
**self.identify_options)
self.wait()
r.close()
def test_reader_messages(self):
self.msg_count = 0
num_messages = 500
topic = 'test_reader_msgs_%s' % time.time()
self._send_messages(topic, num_messages, b'sup')
def handler(msg):
assert msg.body == b'sup'
self.msg_count += 1
if self.msg_count >= num_messages:
self.stop()
return True
r = Reader(nsqd_tcp_addresses=['127.0.0.1:4150'], topic=topic, channel='ch',
io_loop=self.io_loop, message_handler=handler, max_in_flight=100,
**self.identify_options)
self.wait()
r.close()
def forward(self):
targets = []
for t in self.args.target:
imp_path, args = self._parse_forwarder_target_arg(t)
target_class = util.load_object(imp_path)
target_obj = target_class(**args)
targets.append(target_obj)
nsq_receiver = Reader(
topic=self.args.nsqtopic,
channel=self.args.nsqchannel,
nsqd_tcp_addresses=[self.args.nsqd_tcp_address],
max_in_flight=2500,
)
forwarder = LogForwarder(nsq_receiver, targets, self.log)
forwarder.start()
def basic(topic='topic',
channel='channel',
count=1e6,
size=10,
gevent=False,
max_in_flight=2500,
profile=False):
'''Basic benchmark'''
if gevent:
from gevent import monkey
monkey.patch_all()
# Check the types of the arguments
count = int(count)
size = int(size)
max_in_flight = int(max_in_flight)
from nsq.http import nsqd
from nsq.reader import Reader
print('Publishing messages...')
for batch in grouper(messages(count, size), 1000):
nsqd.Client('http://localhost:4151').mpub(topic, batch)
print('Consuming messages')
client = Reader(topic,
channel,
nsqd_tcp_addresses=['localhost:4150'],
max_in_flight=max_in_flight)
with closing(client):
start = -time.time()
if profile:
with profiler():
for message in islice(client, count):
message.fin()
else:
for message in islice(client, count):
message.fin()
start += time.time()
print('Finished %i messages in %fs (%5.2f messages / second)' %
(count, start, count / start))
def get(self):
"""
Sample uri: /tail?nsqd_tcp_address=localhost:4150&nsqd_http_address=localhost:4151&topic=Heartbeat&empty_lines='yes/no'
"""
loop = tornado.ioloop.IOLoop.current()
nsqd_tcp_addresses, nsqd_http_address, topic, empty_lines = self._parse_query(
self.request.query_arguments)
if not nsqd_tcp_addresses:
self.finish("nsqd_tcp_addresses not found")
elif not nsqd_http_address:
self.finish("nsqd_http_address not found")
elif not topic:
self.finish("topic not found")
else:
channel = generate_random_string(self.CHANNEL_NAME_LENGTH)
nsq_reader = Reader(
nsqd_tcp_addresses=[nsqd_tcp_addresses],
topic=topic,
channel=channel,
max_in_flight=self.NSQ_MAX_IN_FLIGHT,
)
def cleanup():
nsq_reader.close()
self._remove_channel(nsqd_http_address=nsqd_http_address,
topic=topic,
channel=channel)
self.log.debug("channel_deleted", channel=channel)
self.finish()
try:
with nsq_reader.connection_checker():
start = time.time()
msg_list = list()
while True:
if self.request.connection.stream.closed():
self.log.info("stream_closed")
cleanup()
break
msg = nsq_reader.read()
for m in msg:
self.log.debug(
"preparing_list",
nsqd_tcp_addresses=nsqd_tcp_addresses,
topic=topic,
channel=channel,
)
if isinstance(m, Message):
msg_list.append(m.body + "\n")
m.fin()
if time.time() - start >= 1:
if msg_list:
result = "".join(msg_list)
self.log.debug(
"yield",
nsqd_tcp_addresses=nsqd_tcp_addresses,
topic=topic,
channel=channel,
result=result,
)
self.write(result)
yield self.flush()
msg_list = list()
start = time.time()
else:
self.log.debug(
"empty",
nsqd_tcp_addresses=nsqd_tcp_addresses,
topic=topic,
channel=channel,
)
if empty_lines:
self.write("\n")
yield self.flush()
except KeyboardInterrupt:
cleanup()
sys.exit(0)