def __initializeNsqdReader(self): nsq.Reader(message_handler=self.__onFastLaneHandler, nsqd_tcp_addresses=self.fastLaneAddrList, lookupd_http_addresses=self.nsqlookupdList, topic=TOPIC["FAST_LANE"], channel="processor", max_in_flight=self.__slowLaneCacheLimit, lookupd_poll_interval=1) nsq.Reader(message_handler=self.__onSlowLaneHandler, nsqd_tcp_addresses=self.slowLaneAddrList, lookupd_http_addresses=self.nsqlookupdList, topic=TOPIC["SLOW_LANE"], channel="processor", max_in_flight=self.__slowLaneCacheLimit, lookupd_poll_interval=1) nsq.Reader(message_handler=self.__onPublishHandler, nsqd_tcp_addresses=self.requestConsumerAddrList, lookupd_http_addresses=self.nsqlookupdList, topic=TOPIC["PUBLISH"], channel="processor", max_in_flight=self.__slowLaneCacheLimit, lookupd_poll_interval=1) nsq.Reader(message_handler=self.__onRequestHandler, nsqd_tcp_addresses=self.requestProducerAddrList, lookupd_http_addresses=self.nsqlookupdList, topic=TOPIC["REQUEST"], channel="processor", lookupd_poll_interval=1)
def pull(self, topic, channel, runFunc, **args): try: def process_message(message): global buf message.enable_async() buf.append(message) if len(buf) >= 3: for msg in buf: msg.finish() buf = [] else: # print 'deferring processing' return result = json.loads(message.body) response = runFunc(result, **args) r = nsq.Reader(message_handler=process_message, nsqd_tcp_addresses=["%s:%s" % (nsq_ip, nsq_port)], topic=topic, channel=channel, max_in_flight=9) nsq.run() except Exception as e: print e return
def consume(self, exchange, routing_key, callback, durable=False, ack=False, queue=None): ''' Inicia o consumo de mensagens do servidor. A mensagem será recebida por <code>callback</code>. ''' nsq.Reader(message_handler=functools.partial(self._handler, routing_key=routing_key, callback=callback), max_in_flight=1000, lookupd_http_addresses=[ 'http://{}:4161'.format(self.connection.server) ], topic='{}{}'.format(queue, self.connection.path), channel='proc', lookupd_poll_interval=15) #if hasattr(self.connection, 'time_in_secs') and self.connection.time_in_secs: #nsq.tornado.ioloop.add_timeout(self.connection.time_in_secs * 1000, self.connection.callback) # @UndefinedVariable #t.start() nsq.run()
def subscribe(self, topic, callback): """ Used to subscribe a callback to a topic. It will spin up a new :class:`nsq.Reader` for the given topic if one does not exist already and register the given callback with it. Only one callback per topic exists. :param topic: a string which identifies the topic on which to listen. :type string: :param callback: the callback which will be called when a message is read. The callback must take a single argument (`message`), call :meth:`nsq.Message.finish()`/:meth:`nsq.Message.requeue()` and return `True`/`False` when it is done with the message. The callback can expect `message` to be :class:`nsq.Message`, with an additional attribute `parsed_body` which contains the `json` parsed body of the message (still available in raw from through the `body` attribute). :type callable: """ # Check if topic exists, if not - create it # Check if channel exists, if not - create it if not self._readers.get(topic, None): _create_topic(topic, self._lookupd_addresses) _create_channel(topic, self._server_name, self._lookupd_addresses) reader = nsq.Reader(message_handler=self.filter_callback(callback), lookupd_http_addresses=self._lookupd_addresses, topic=topic, channel=self._server_name, lookupd_poll_interval=5) self._readers[topic] = reader log.msg("Subscribed on %s on %s" % (topic, self._server_name))
def get_reader(io_loop=None, max_in_flight=5): return nsq.Reader("test", "test", message_handler=_message_handler, lookupd_http_addresses=["http://test.local:4161"], max_in_flight=max_in_flight, io_loop=io_loop)
def __init__(self): nsq.Reader( topic=self.name, channel='mq', name="mq_output.mq", # nsqd_tcp_addresses=['127.0.0.1:4150'], lookupd_http_addresses=['http://127.0.0.1:4161'], message_handler=self._handle_monitor, heartbeat_interval=10, output_buffer_size=4096, output_buffer_timeout=100, max_tries=5, max_in_flight=9, lookupd_poll_interval=60, low_rdy_idle_timeout=10, max_backoff_duration=128, lookupd_poll_jitter=0.3, lookupd_connect_timeout=1, lookupd_request_timeout=2, ) import test self.code_path = test.__path__[0] print self.code_path self.redis_client = redis.Redis(host='127.0.0.1', port=6379) _p = self.code_path.split("/").pop() print(_p) for name, code in self.upload_code(): self.redis_client.hset("__code__." + _p, name, code)
def run(self): self.r = nsq.Reader(message_handler=self.handler, nsqd_tcp_addresses=[self.nsqd_addr], topic=self.topic, channel="default", lookupd_poll_interval=15) nsq.run()
def start(self): nsq.Reader(message_handler=self.nsq_kwargs["handler"], lookupd_http_addresses=[self.nsq_kwargs["nsq_url"]], topic=self.nsq_kwargs["topic"], channel=self.nsq_kwargs["channel"], max_in_flight=9) nsq.run()
def __init__(self, topic: str, channel: str): nsq.Reader(message_handler=self.async_handler, nsqd_tcp_addresses=[Config.TCP], topic=topic, channel=channel, max_in_flight=10, lookupd_poll_interval=3)
def __init__(self): self.__code_obj = {} nsq.Reader( topic='mq_input', channel='mq', name="mq_input.mq", nsqd_tcp_addresses=['127.0.0.1:4150'], lookupd_http_addresses=['http://127.0.0.1:4161'], message_handler=self._handle_monitor, heartbeat_interval=10, tls_options={'cert_reqs': ssl.CERT_NONE}, output_buffer_size=4096, output_buffer_timeout=100, max_tries=5, max_in_flight=9, lookupd_poll_interval=60, low_rdy_idle_timeout=10, max_backoff_duration=128, lookupd_poll_jitter=0.3, lookupd_connect_timeout=1, lookupd_request_timeout=2, ) self.writer = nsq.Writer(['127.0.0.1:4150']) self.redis_client = redis.Redis(host='127.0.0.1', port=6379)
def create_nsq_reader(topic, message_handler, channel='default'): return nsq.Reader( topic=topic, channel=channel, message_handler=message_handler, nsqd_tcp_addresses=config.NSQD_TCP_ADDRESSES, )
def __init__(self, topic, channel, max_in_flight=5, auto_confirm=True, task_cls=Task, **conf): self.task_cls = task_cls self._auto_confirm = auto_confirm conf['max_in_flight'] = max_in_flight self._inner_q = asyncio.Queue(0) self._nsq_reader = nsq.Reader(topic=topic, channel=channel, message_handler=self._message_handler, **conf)
def test_backoff_hard(mock_ioloop, mock_iostream): conn = _get_test_conn() msg = _get_test_message() instance = Mock() mock_ioloop.instance.return_value = instance r = nsq.Reader({"test": two_arg_fxn}, "test", "test", nsqd_tcp_addresses=[ "test:9999", ], max_in_flight=5) r.conns["test:9999:test"] = conn r.total_ready = 0 num_fails = 0 fail = True last_timeout_time = 0 for i in range(50): if fail: r._client_callback(nsq.REQ, task="test", conn=conn, message=msg) num_fails += 1 send_args, send_kwargs = conn.send.call_args assert send_args[0] == 'RDY 0\n' else: r._client_callback(nsq.FIN, task="test", conn=conn, message=msg) num_fails -= 1 assert r.backoff_block["test"] == True assert r.backoff_timer["test"].get_interval() > 0 assert instance.add_timeout.called timeout_args, timeout_kwargs = instance.add_timeout.call_args if timeout_args[0] != last_timeout_time: timeout_args[1]() last_timeout_time = timeout_args[0] assert r.backoff_block["test"] == False send_args, send_kwargs = conn.send.call_args assert send_args[0] == 'RDY 1\n' fail = True if random.random() < 0.1 and num_fails > 1: fail = False for i in range(num_fails + 1): r._client_callback(nsq.FIN, task="test", conn=conn, message=msg) timeout_args, timeout_kwargs = instance.add_timeout.call_args if timeout_args[0] != last_timeout_time: timeout_args[1]() last_timeout_time = timeout_args[0] r._client_callback(nsq.FIN, task="test", conn=conn, message=msg) assert r.backoff_block["test"] == False assert r.backoff_timer["test"].get_interval() == 0 send_args, send_kwargs = conn.send.call_args_list[-3] assert send_args[0] == 'RDY 5\n'
def subscribe(conversation, client_id, callback): reader = nsq.Reader(message_handler=callback, lookupd_http_addresses=['http://nsqlookupd:4161'], topic='conversations.{}'.format(conversation['id']), channel='web-{}#ephemeral'.format(client_id), max_in_flight=10) return reader
def register_message_readers(): '''为nsq消息系统建立消费者''' for topic in MessageTopic.all_topic: handler = None if topic == MessageTopic.CHAT_MESSAGE_NEW: handler = chat_message_handler elif topic == MessageTopic.SEND_ACTIVATION_EMAIL: handler = send_activation_email_handler elif topic == MessageTopic.SEND_RESET_PASSWORD_EMAIL: handler = send_reset_password_email_handler elif topic == MessageTopic.SEND_HAS_UNREAD_MESSAGE_EMAIL: handler = send_has_unread_message_email_handler else: handler = message_handler if handler is not None: reader = { 'message_handler': handler, "nsqd_tcp_addresses": ['127.0.0.1:4150'], 'topic': topic, 'channel': topic, 'lookupd_poll_interval': 15 } nsq.Reader(**reader)
def consume(): r = nsq.Reader(message_handler=handler, nsqd_tcp_addresses=['127.0.0.1:4150'], topic='tag', channel='a', lookupd_poll_interval=15) nsq.run() # tornado.ioloop.IOLoop.instance().start()
def get_reader(max_in_flight=5): return nsq.Reader( "test", "test", message_handler=_message_handler, lookupd_http_addresses=["http://localhost:4161"], max_in_flight=max_in_flight, max_backoff_duration=2.0, )
def nsq_run(topic, channel, tcp_address, handler): nsq.Reader(message_handler=handler, topic=topic, channel=channel, lookupd_poll_interval=5, nsqd_tcp_addresses=tcp_address, lookupd_request_timeout=30, max_in_flight=3) nsq.run()
def run(self): self.reader = nsq.Reader( self.tasks, topic=self.config['topic'], channel=self.config['channel'], nsqd_tcp_addresses=self.config.get('nsqd_tcp_addresses', ['127.0.0.1:4150']), max_in_flight=self.config.get('max_in_flight', 1)) nsq.run()
def listen_abuses(lookupd_addresses): """ Listens for abuse messages. """ nsq.Reader(message_handler=abuses_handler, lookupd_http_addresses=lookupd_addresses, topic=ABUSES_TOPIC_NAME, channel='send-email', lookupd_poll_interval=15)
def listen_registrations(lookupd_addresses): """ Listens for registration messages. """ nsq.Reader(message_handler=registrations_handler, lookupd_http_addresses=lookupd_addresses, topic=REGISTRATIONS_TOPIC_NAME, channel='send-email', lookupd_poll_interval=15)
def listen_feedbacks(lookupd_addresses): """ Listens for feedback messages. """ nsq.Reader(message_handler=feedbacks_handler, lookupd_http_addresses=lookupd_addresses, topic=FEEDBACKS_TOPIC_NAME, channel='send-email', lookupd_poll_interval=15)
def __init__(self, nsqlookupd_addresses: List, topic: str, channel: str): self.reader = nsq.Reader(topic, channel, message_handler=self._handle_message, lookupd_poll_interval=5, lookupd_http_addresses=nsqlookupd_addresses) self.mediator = Queue() # Or client will have to wait `lookupd_pool_interval` # seconds after the service start. IOLoop.current().add_callback(self.reader.query_lookupd)
def run(workers_module="nsqworker.workers", **kw): worker = load_worker(workers_module) logging.debug("Starting worker: %s" % worker) topic = "%s%s" % (options.topic_prefix, options.topic) legacy = False try: from nsq import LegacyReader except ImportError: legacy = True if not legacy: # nsq 0.5+ for name, handler in worker.handlers.iteritems(): r = nsq.Reader( topic, "%s_%s" % (options.channel, name[0:-len("_handler")]), message_handler=handler, lookupd_http_addresses=options.nsqlookupd_http_addresses, **kw) # override default preprocess and validate method with worker's # method r.validate_message = worker.validate_message r.preprocess_message = worker.preprocess_message else: # nsq 0.4 r = nsq.Reader( all_tasks=worker.handlers, topic=topic, channel=options.channel, lookupd_http_addresses=options.nsqlookupd_http_addresses, **kw) r.validate_message = worker.validate_message r.preprocess_message = worker.preprocess_message if options.debug: autoreload.start() nsq.run()
def run(self): tornado.platform.asyncio.AsyncIOMainLoop().install() nsq.Reader(message_handler=self._handle, nsqd_tcp_addresses=self.nsqd, lookupd_http_addresses=self.nsqlookupd, topic=self.topic, channel=self.channel, max_in_flight=self.workers) executor = ThreadPoolExecutor(max_workers=self.workers) loop = asyncio.get_event_loop() loop.set_default_executor(executor) loop.run_forever()
def create_subscriber(nsqd_config, topic, channel, on_received_message): def _message_handler(message, callback): message.enable_async() data = message.body message.finish() callback(data) return nsq.Reader( topic=topic, channel=channel, message_handler=functools.partial(_message_handler, callback=on_received_message), nsqd_tcp_addresses=nsqd_config.tcp_addresses, )
def subscribe_worker(self): kwargs = {k: v for k, v in self.kwargs.items()} kwargs["message_handler"] = self._message_handler kwargs["max_in_flight"] = self.max_in_flight self.reader = nsq.Reader(**kwargs) self.logger.info( "Added an handler for NSQD messages on [service_name={}] [topic={}], [channel={}]." .format(self.service_name, self.kwargs["topic"], self.kwargs["channel"])) self.logger.info( "Handling messages with {} threads and {} max_in_flight.".format( self.concurrency, self.max_in_flight))
def _start(self): import nsq for topic, channel, func in self.handlers: func = alert_exceptions(func) reader = nsq.Reader( topic=topic, channel=channel, message_handler=func, nsqd_tcp_addresses=self.nsqd_tcp_addresses, lookupd_http_addresses=self.lookupd_http_addresses, **self.nsq_kwargs ) self.readers.append(reader) nsq.run()
def register_message_readers(): '''为nsq消息系统建立消费者''' handlers = { MessageTopic.CHAT_MESSAGE_NEW: chat_message_handler, MessageTopic.SEND_ACTIVATION_EMAIL: send_activation_email_handler, MessageTopic.SEND_RESET_PASSWORD_EMAIL: send_reset_password_email_handler, MessageTopic.SEND_HAS_UNREAD_MESSAGE_EMAIL: send_has_unread_message_email_handler, } for topic in MessageTopic.all_topic: nsq.Reader(message_handler=handlers.get(topic, message_handler), nsqd_tcp_addresses=['127.0.0.1:4150'], topic=topic, channel=topic, lookupd_poll_interval=15)
def test_backoff_easy(mock_ioloop, mock_iostream): conn = _get_test_conn() msg = _get_test_message() instance = Mock() mock_ioloop.instance.return_value = instance r = nsq.Reader({"test": two_arg_fxn}, "test", "test", nsqd_tcp_addresses=[ "test:9999", ], max_in_flight=5) r.conns["test:9999:test"] = conn r.total_ready = 0 r._client_callback(nsq.REQ, task="test", conn=conn, message=msg) assert r.backoff_block["test"] == True assert r.backoff_timer["test"].get_interval() > 0 assert instance.add_timeout.called send_args, send_kwargs = conn.send.call_args assert send_args[0] == 'RDY 0\n' timeout_args, timeout_kwargs = instance.add_timeout.call_args timeout_args[1]() assert r.backoff_block["test"] == False send_args, send_kwargs = conn.send.call_args assert send_args[0] == 'RDY 1\n' r._client_callback(nsq.FIN, task="test", conn=conn, message=msg) assert r.backoff_block["test"] == False assert r.backoff_timer["test"].get_interval() == 0 send_args, send_kwargs = conn.send.call_args assert send_args[0] == 'RDY 5\n'