def __initializeNsqdReader(self):
nsq.Reader(message_handler=self.__onFastLaneHandler,
nsqd_tcp_addresses=self.fastLaneAddrList,
lookupd_http_addresses=self.nsqlookupdList,
topic=TOPIC["FAST_LANE"],
channel="processor",
max_in_flight=self.__slowLaneCacheLimit,
lookupd_poll_interval=1)
nsq.Reader(message_handler=self.__onSlowLaneHandler,
nsqd_tcp_addresses=self.slowLaneAddrList,
lookupd_http_addresses=self.nsqlookupdList,
topic=TOPIC["SLOW_LANE"],
channel="processor",
max_in_flight=self.__slowLaneCacheLimit,
lookupd_poll_interval=1)
nsq.Reader(message_handler=self.__onPublishHandler,
nsqd_tcp_addresses=self.requestConsumerAddrList,
lookupd_http_addresses=self.nsqlookupdList,
topic=TOPIC["PUBLISH"],
channel="processor",
max_in_flight=self.__slowLaneCacheLimit,
lookupd_poll_interval=1)
nsq.Reader(message_handler=self.__onRequestHandler,
nsqd_tcp_addresses=self.requestProducerAddrList,
lookupd_http_addresses=self.nsqlookupdList,
topic=TOPIC["REQUEST"],
channel="processor",
lookupd_poll_interval=1)
def pull(self, topic, channel, runFunc, **args):
try:
def process_message(message):
global buf
message.enable_async()
buf.append(message)
if len(buf) >= 3:
for msg in buf:
msg.finish()
buf = []
else:
# print 'deferring processing'
return
result = json.loads(message.body)
response = runFunc(result, **args)
r = nsq.Reader(message_handler=process_message,
nsqd_tcp_addresses=["%s:%s" % (nsq_ip, nsq_port)],
topic=topic,
channel=channel,
max_in_flight=9)
nsq.run()
except Exception as e:
print e
return
def consume(self,
exchange,
routing_key,
callback,
durable=False,
ack=False,
queue=None):
'''
Inicia o consumo de mensagens do servidor. A mensagem será recebida por <code>callback</code>.
'''
nsq.Reader(message_handler=functools.partial(self._handler,
routing_key=routing_key,
callback=callback),
max_in_flight=1000,
lookupd_http_addresses=[
'http://{}:4161'.format(self.connection.server)
],
topic='{}{}'.format(queue, self.connection.path),
channel='proc',
lookupd_poll_interval=15)
#if hasattr(self.connection, 'time_in_secs') and self.connection.time_in_secs:
#nsq.tornado.ioloop.add_timeout(self.connection.time_in_secs * 1000, self.connection.callback) # @UndefinedVariable
#t.start()
nsq.run()
def subscribe(self, topic, callback):
"""
Used to subscribe a callback to a topic.
It will spin up a new :class:`nsq.Reader` for the given topic if one
does not exist already and register the given callback with it.
Only one callback per topic exists.
:param topic: a string which identifies the topic on which to listen.
:type string:
:param callback: the callback which will be called when a message is
read. The callback must take a single argument (`message`),
call :meth:`nsq.Message.finish()`/:meth:`nsq.Message.requeue()` and
return `True`/`False` when it is done with the message. The callback
can expect `message` to be :class:`nsq.Message`, with an additional
attribute `parsed_body` which contains the `json` parsed body of the
message (still available in raw from through the `body` attribute).
:type callable:
"""
# Check if topic exists, if not - create it
# Check if channel exists, if not - create it
if not self._readers.get(topic, None):
_create_topic(topic, self._lookupd_addresses)
_create_channel(topic, self._server_name, self._lookupd_addresses)
reader = nsq.Reader(message_handler=self.filter_callback(callback),
lookupd_http_addresses=self._lookupd_addresses,
topic=topic,
channel=self._server_name,
lookupd_poll_interval=5)
self._readers[topic] = reader
log.msg("Subscribed on %s on %s" % (topic, self._server_name))
def get_reader(io_loop=None, max_in_flight=5):
return nsq.Reader("test",
"test",
message_handler=_message_handler,
lookupd_http_addresses=["http://test.local:4161"],
max_in_flight=max_in_flight,
io_loop=io_loop)
def __init__(self):
nsq.Reader(
topic=self.name,
channel='mq',
name="mq_output.mq",
# nsqd_tcp_addresses=['127.0.0.1:4150'],
lookupd_http_addresses=['http://127.0.0.1:4161'],
message_handler=self._handle_monitor,
heartbeat_interval=10,
output_buffer_size=4096,
output_buffer_timeout=100,
max_tries=5,
max_in_flight=9,
lookupd_poll_interval=60,
low_rdy_idle_timeout=10,
max_backoff_duration=128,
lookupd_poll_jitter=0.3,
lookupd_connect_timeout=1,
lookupd_request_timeout=2,
)
import test
self.code_path = test.__path__[0]
print self.code_path
self.redis_client = redis.Redis(host='127.0.0.1', port=6379)
_p = self.code_path.split("/").pop()
print(_p)
for name, code in self.upload_code():
self.redis_client.hset("__code__." + _p, name, code)
def run(self):
self.r = nsq.Reader(message_handler=self.handler,
nsqd_tcp_addresses=[self.nsqd_addr],
topic=self.topic,
channel="default",
lookupd_poll_interval=15)
nsq.run()
def start(self):
nsq.Reader(message_handler=self.nsq_kwargs["handler"],
lookupd_http_addresses=[self.nsq_kwargs["nsq_url"]],
topic=self.nsq_kwargs["topic"],
channel=self.nsq_kwargs["channel"],
max_in_flight=9)
nsq.run()
def __init__(self, topic: str, channel: str):
nsq.Reader(message_handler=self.async_handler,
nsqd_tcp_addresses=[Config.TCP],
topic=topic,
channel=channel,
max_in_flight=10,
lookupd_poll_interval=3)
def __init__(self):
self.__code_obj = {}
nsq.Reader(
topic='mq_input',
channel='mq',
name="mq_input.mq",
nsqd_tcp_addresses=['127.0.0.1:4150'],
lookupd_http_addresses=['http://127.0.0.1:4161'],
message_handler=self._handle_monitor,
heartbeat_interval=10,
tls_options={'cert_reqs': ssl.CERT_NONE},
output_buffer_size=4096,
output_buffer_timeout=100,
max_tries=5,
max_in_flight=9,
lookupd_poll_interval=60,
low_rdy_idle_timeout=10,
max_backoff_duration=128,
lookupd_poll_jitter=0.3,
lookupd_connect_timeout=1,
lookupd_request_timeout=2,
)
self.writer = nsq.Writer(['127.0.0.1:4150'])
self.redis_client = redis.Redis(host='127.0.0.1', port=6379)
def create_nsq_reader(topic, message_handler, channel='default'):
return nsq.Reader(
topic=topic,
channel=channel,
message_handler=message_handler,
nsqd_tcp_addresses=config.NSQD_TCP_ADDRESSES,
)
def __init__(self, topic, channel, max_in_flight=5, auto_confirm=True, task_cls=Task, **conf):
self.task_cls = task_cls
self._auto_confirm = auto_confirm
conf['max_in_flight'] = max_in_flight
self._inner_q = asyncio.Queue(0)
self._nsq_reader = nsq.Reader(topic=topic, channel=channel,
message_handler=self._message_handler, **conf)
def test_backoff_hard(mock_ioloop, mock_iostream):
conn = _get_test_conn()
msg = _get_test_message()
instance = Mock()
mock_ioloop.instance.return_value = instance
r = nsq.Reader({"test": two_arg_fxn},
"test",
"test",
nsqd_tcp_addresses=[
"test:9999",
],
max_in_flight=5)
r.conns["test:9999:test"] = conn
r.total_ready = 0
num_fails = 0
fail = True
last_timeout_time = 0
for i in range(50):
if fail:
r._client_callback(nsq.REQ, task="test", conn=conn, message=msg)
num_fails += 1
send_args, send_kwargs = conn.send.call_args
assert send_args[0] == 'RDY 0\n'
else:
r._client_callback(nsq.FIN, task="test", conn=conn, message=msg)
num_fails -= 1
assert r.backoff_block["test"] == True
assert r.backoff_timer["test"].get_interval() > 0
assert instance.add_timeout.called
timeout_args, timeout_kwargs = instance.add_timeout.call_args
if timeout_args[0] != last_timeout_time:
timeout_args[1]()
last_timeout_time = timeout_args[0]
assert r.backoff_block["test"] == False
send_args, send_kwargs = conn.send.call_args
assert send_args[0] == 'RDY 1\n'
fail = True
if random.random() < 0.1 and num_fails > 1:
fail = False
for i in range(num_fails + 1):
r._client_callback(nsq.FIN, task="test", conn=conn, message=msg)
timeout_args, timeout_kwargs = instance.add_timeout.call_args
if timeout_args[0] != last_timeout_time:
timeout_args[1]()
last_timeout_time = timeout_args[0]
r._client_callback(nsq.FIN, task="test", conn=conn, message=msg)
assert r.backoff_block["test"] == False
assert r.backoff_timer["test"].get_interval() == 0
send_args, send_kwargs = conn.send.call_args_list[-3]
assert send_args[0] == 'RDY 5\n'
def subscribe(conversation, client_id, callback):
reader = nsq.Reader(message_handler=callback,
lookupd_http_addresses=['http://nsqlookupd:4161'],
topic='conversations.{}'.format(conversation['id']),
channel='web-{}#ephemeral'.format(client_id),
max_in_flight=10)
return reader
def register_message_readers():
'''为nsq消息系统建立消费者'''
for topic in MessageTopic.all_topic:
handler = None
if topic == MessageTopic.CHAT_MESSAGE_NEW:
handler = chat_message_handler
elif topic == MessageTopic.SEND_ACTIVATION_EMAIL:
handler = send_activation_email_handler
elif topic == MessageTopic.SEND_RESET_PASSWORD_EMAIL:
handler = send_reset_password_email_handler
elif topic == MessageTopic.SEND_HAS_UNREAD_MESSAGE_EMAIL:
handler = send_has_unread_message_email_handler
else:
handler = message_handler
if handler is not None:
reader = {
'message_handler': handler,
"nsqd_tcp_addresses": ['127.0.0.1:4150'],
'topic': topic,
'channel': topic,
'lookupd_poll_interval': 15
}
nsq.Reader(**reader)
def consume():
r = nsq.Reader(message_handler=handler,
nsqd_tcp_addresses=['127.0.0.1:4150'],
topic='tag',
channel='a',
lookupd_poll_interval=15)
nsq.run() # tornado.ioloop.IOLoop.instance().start()
def get_reader(max_in_flight=5):
return nsq.Reader(
"test",
"test",
message_handler=_message_handler,
lookupd_http_addresses=["http://localhost:4161"],
max_in_flight=max_in_flight,
max_backoff_duration=2.0,
)
def nsq_run(topic, channel, tcp_address, handler):
nsq.Reader(message_handler=handler,
topic=topic,
channel=channel,
lookupd_poll_interval=5,
nsqd_tcp_addresses=tcp_address,
lookupd_request_timeout=30,
max_in_flight=3)
nsq.run()
def run(self):
self.reader = nsq.Reader(
self.tasks,
topic=self.config['topic'],
channel=self.config['channel'],
nsqd_tcp_addresses=self.config.get('nsqd_tcp_addresses',
['127.0.0.1:4150']),
max_in_flight=self.config.get('max_in_flight', 1))
nsq.run()
def listen_abuses(lookupd_addresses):
"""
Listens for abuse messages.
"""
nsq.Reader(message_handler=abuses_handler,
lookupd_http_addresses=lookupd_addresses,
topic=ABUSES_TOPIC_NAME,
channel='send-email',
lookupd_poll_interval=15)
def listen_registrations(lookupd_addresses):
"""
Listens for registration messages.
"""
nsq.Reader(message_handler=registrations_handler,
lookupd_http_addresses=lookupd_addresses,
topic=REGISTRATIONS_TOPIC_NAME,
channel='send-email',
lookupd_poll_interval=15)
def listen_feedbacks(lookupd_addresses):
"""
Listens for feedback messages.
"""
nsq.Reader(message_handler=feedbacks_handler,
lookupd_http_addresses=lookupd_addresses,
topic=FEEDBACKS_TOPIC_NAME,
channel='send-email',
lookupd_poll_interval=15)
def __init__(self, nsqlookupd_addresses: List, topic: str, channel: str):
self.reader = nsq.Reader(topic,
channel,
message_handler=self._handle_message,
lookupd_poll_interval=5,
lookupd_http_addresses=nsqlookupd_addresses)
self.mediator = Queue()
# Or client will have to wait `lookupd_pool_interval`
# seconds after the service start.
IOLoop.current().add_callback(self.reader.query_lookupd)
def run(workers_module="nsqworker.workers", **kw):
worker = load_worker(workers_module)
logging.debug("Starting worker: %s" % worker)
topic = "%s%s" % (options.topic_prefix, options.topic)
legacy = False
try:
from nsq import LegacyReader
except ImportError:
legacy = True
if not legacy:
# nsq 0.5+
for name, handler in worker.handlers.iteritems():
r = nsq.Reader(
topic,
"%s_%s" % (options.channel, name[0:-len("_handler")]),
message_handler=handler,
lookupd_http_addresses=options.nsqlookupd_http_addresses,
**kw)
# override default preprocess and validate method with worker's
# method
r.validate_message = worker.validate_message
r.preprocess_message = worker.preprocess_message
else:
# nsq 0.4
r = nsq.Reader(
all_tasks=worker.handlers,
topic=topic,
channel=options.channel,
lookupd_http_addresses=options.nsqlookupd_http_addresses,
**kw)
r.validate_message = worker.validate_message
r.preprocess_message = worker.preprocess_message
if options.debug:
autoreload.start()
nsq.run()
def run(self):
tornado.platform.asyncio.AsyncIOMainLoop().install()
nsq.Reader(message_handler=self._handle,
nsqd_tcp_addresses=self.nsqd,
lookupd_http_addresses=self.nsqlookupd,
topic=self.topic,
channel=self.channel,
max_in_flight=self.workers)
executor = ThreadPoolExecutor(max_workers=self.workers)
loop = asyncio.get_event_loop()
loop.set_default_executor(executor)
loop.run_forever()
def create_subscriber(nsqd_config, topic, channel, on_received_message):
def _message_handler(message, callback):
message.enable_async()
data = message.body
message.finish()
callback(data)
return nsq.Reader(
topic=topic,
channel=channel,
message_handler=functools.partial(_message_handler,
callback=on_received_message),
nsqd_tcp_addresses=nsqd_config.tcp_addresses,
)
def subscribe_worker(self):
kwargs = {k: v for k, v in self.kwargs.items()}
kwargs["message_handler"] = self._message_handler
kwargs["max_in_flight"] = self.max_in_flight
self.reader = nsq.Reader(**kwargs)
self.logger.info(
"Added an handler for NSQD messages on [service_name={}] [topic={}], [channel={}]."
.format(self.service_name, self.kwargs["topic"],
self.kwargs["channel"]))
self.logger.info(
"Handling messages with {} threads and {} max_in_flight.".format(
self.concurrency, self.max_in_flight))
def _start(self):
import nsq
for topic, channel, func in self.handlers:
func = alert_exceptions(func)
reader = nsq.Reader(
topic=topic,
channel=channel,
message_handler=func,
nsqd_tcp_addresses=self.nsqd_tcp_addresses,
lookupd_http_addresses=self.lookupd_http_addresses,
**self.nsq_kwargs
)
self.readers.append(reader)
nsq.run()
def register_message_readers():
'''为nsq消息系统建立消费者'''
handlers = {
MessageTopic.CHAT_MESSAGE_NEW:
chat_message_handler,
MessageTopic.SEND_ACTIVATION_EMAIL:
send_activation_email_handler,
MessageTopic.SEND_RESET_PASSWORD_EMAIL:
send_reset_password_email_handler,
MessageTopic.SEND_HAS_UNREAD_MESSAGE_EMAIL:
send_has_unread_message_email_handler,
}
for topic in MessageTopic.all_topic:
nsq.Reader(message_handler=handlers.get(topic, message_handler),
nsqd_tcp_addresses=['127.0.0.1:4150'],
topic=topic,
channel=topic,
lookupd_poll_interval=15)
def test_backoff_easy(mock_ioloop, mock_iostream):
conn = _get_test_conn()
msg = _get_test_message()
instance = Mock()
mock_ioloop.instance.return_value = instance
r = nsq.Reader({"test": two_arg_fxn},
"test",
"test",
nsqd_tcp_addresses=[
"test:9999",
],
max_in_flight=5)
r.conns["test:9999:test"] = conn
r.total_ready = 0
r._client_callback(nsq.REQ, task="test", conn=conn, message=msg)
assert r.backoff_block["test"] == True
assert r.backoff_timer["test"].get_interval() > 0
assert instance.add_timeout.called
send_args, send_kwargs = conn.send.call_args
assert send_args[0] == 'RDY 0\n'
timeout_args, timeout_kwargs = instance.add_timeout.call_args
timeout_args[1]()
assert r.backoff_block["test"] == False
send_args, send_kwargs = conn.send.call_args
assert send_args[0] == 'RDY 1\n'
r._client_callback(nsq.FIN, task="test", conn=conn, message=msg)
assert r.backoff_block["test"] == False
assert r.backoff_timer["test"].get_interval() == 0
send_args, send_kwargs = conn.send.call_args
assert send_args[0] == 'RDY 5\n'
