def consume(self,
exchange,
routing_key,
callback,
durable=False,
ack=False,
queue=None):
'''
Inicia o consumo de mensagens do servidor. A mensagem será recebida por <code>callback</code>.
'''
nsq.Reader(message_handler=functools.partial(self._handler,
routing_key=routing_key,
callback=callback),
max_in_flight=1000,
lookupd_http_addresses=[
'http://{}:4161'.format(self.connection.server)
],
topic='{}{}'.format(queue, self.connection.path),
channel='proc',
lookupd_poll_interval=15)
#if hasattr(self.connection, 'time_in_secs') and self.connection.time_in_secs:
#nsq.tornado.ioloop.add_timeout(self.connection.time_in_secs * 1000, self.connection.callback) # @UndefinedVariable
#t.start()
nsq.run()
def run(workers_module="nsqworker.workers", **kw):
worker = load_worker(workers_module)
logging.debug("Starting worker: %s" % worker)
legacy = False
try:
from nsq import LegacyReader
except ImportError:
legacy = True
if not legacy:
# nsq 0.5
for name, handler in worker.handlers.iteritems():
r = nsq.Reader(options.topic, options.channel,
message_handler=handler,
lookupd_http_addresses=options.nsqlookupd_http_addresses,
**kw)
# override default preprocess and validate method with worker's method
r.validate_message = worker.validate_message
r.preprocess_message = worker.preprocess_message
else:
# nsq 0.4
r = nsq.Reader(all_tasks=worker.handlers,
topic=options.topic,
channel=options.channel,
lookupd_http_addresses=options.nsqlookupd_http_addresses,
**kw)
r.validate_message = worker.validate_message
r.preprocess_message = worker.preprocess_message
nsq.run()
def pull(self, topic, channel, runFunc, **args):
try:
def process_message(message):
global buf
message.enable_async()
buf.append(message)
if len(buf) >= 3:
for msg in buf:
msg.finish()
buf = []
else:
# print 'deferring processing'
return
result = json.loads(message.body)
response = runFunc(result, **args)
r = nsq.Reader(message_handler=process_message,
nsqd_tcp_addresses=["%s:%s" % (nsq_ip, nsq_port)],
topic=topic,
channel=channel,
max_in_flight=9)
nsq.run()
except Exception as e:
print e
return
def publish(src, dest, msgType, content):
data = {}
data['MacAddress'] = content['MacAddress']
data['longitude'] = content['longitude']
data['latitude'] = content['latitude']
data['distance'] = content['distance']
#should any filtering take place?
json_location = json.loads(json.dumps(data))
def publish_destination():
writer.pub(dest, json.dumps(json_location), finish_pub)
def finish_pub(conn, data):
print(data)
tornado.ioloop.IOLoop.current().stop()
writer = nsq.Writer(['127.0.0.1:4150'])
tornado.ioloop.PeriodicCallback(publish_destination,1000).start()
nsq.run()
def run(self):
self.r = nsq.Reader(message_handler=self.handler,
nsqd_tcp_addresses=[self.nsqd_addr],
topic=self.topic,
channel="default",
lookupd_poll_interval=15)
nsq.run()
def start(self):
nsq.Reader(message_handler=self.nsq_kwargs["handler"],
lookupd_http_addresses=[self.nsq_kwargs["nsq_url"]],
topic=self.nsq_kwargs["topic"],
channel=self.nsq_kwargs["channel"],
max_in_flight=9)
nsq.run()
def consume(self, callback):
reader = nsq.Reader(message_handler=self.create_message,
lookupd_http_addresses=['http://127.0.0.1:4161'],
topic=self.args.topic,
channel=self.args.channel,
lookupd_poll_interval=5
)
nsq.run()
def consume():
r = nsq.Reader(message_handler=handler,
nsqd_tcp_addresses=['127.0.0.1:4150'],
topic='tag',
channel='a',
lookupd_poll_interval=15)
nsq.run() # tornado.ioloop.IOLoop.instance().start()
def run(self):
self.reader = nsq.Reader(
self.tasks,
topic=self.config['topic'],
channel=self.config['channel'],
nsqd_tcp_addresses=self.config.get('nsqd_tcp_addresses',
['127.0.0.1:4150']),
max_in_flight=self.config.get('max_in_flight', 1))
nsq.run()
def nsq_run(topic, channel, tcp_address, handler):
nsq.Reader(message_handler=handler,
topic=topic,
channel=channel,
lookupd_poll_interval=5,
nsqd_tcp_addresses=tcp_address,
lookupd_request_timeout=30,
max_in_flight=3)
nsq.run()
def run(self):
self.reader = nsq.Reader(
self.tasks,
topic=self.config['topic'],
channel=self.config['channel'],
nsqd_tcp_addresses=self.config.get('nsqd_tcp_addresses',
['127.0.0.1:4150']),
max_in_flight=self.config.get('max_in_flight', 1))
nsq.run()
def consume(self, topics, channel):
self.logger.info("NSQ worker started, topics: {}, channel:{}, addresses:{}".format(','.join(topics), channel, ','.join(self.http_addresses)))
for t in topics:
nsq.Reader(message_handler=self.handle,
lookupd_http_addresses=self.http_addresses,
topic=t, channel=channel,
lookupd_poll_interval=15)
nsq.run()
def main(args):
query_server = client_query_server.ClientQueryRequestServer(12345)
query_server.start()
pool = client_pool.ClientPool(query_server)
pool.start_accepting_clients()
accessor = client_accessor.ClientAccessor(query_server, pool)
accessor.start_accepting_query()
nsq.run()
def main():
"""main"""
tornado.options.parse_command_line()
reader = nsq.Reader(topic='test', channel='pynsq',
nsqd_tcp_addresses=['127.0.0.1:4150'],
message_handler=handler, heartbeat_interval=10,
tls_v1=True, tls_options={'cert_reqs': ssl.CERT_NONE},
snappy=True,
output_buffer_size=4096, output_buffer_timeout=100,
user_agent='test')
nsq.run()
def listen(lookupd_addresses=None):
"""
Define listeners for incoming messages.
"""
# get lookupd addresses
if lookupd_addresses:
lookupd_addresses = [lookupd_addresses]
lookupd_addresses = lookupd_addresses or LOOKUPD_ADDRESSES
# register listeners
register_listeners(lookupd_addresses)
# start all listeners
nsq.run()
def listen(lookupd_addresses=None):
"""
Define listeners for incoming messages.
"""
# get lookupd addresses
if lookupd_addresses:
lookupd_addresses = [lookupd_addresses]
lookupd_addresses = lookupd_addresses or LOOKUPD_ADDRESSES
# register listeners
register_listeners(lookupd_addresses)
# start all listeners
nsq.run()
def _start(self):
import nsq
for topic, channel, func in self.handlers:
func = alert_exceptions(func)
reader = nsq.Reader(
topic=topic,
channel=channel,
message_handler=func,
nsqd_tcp_addresses=self.nsqd_tcp_addresses,
lookupd_http_addresses=self.lookupd_http_addresses,
**self.nsq_kwargs
)
self.readers.append(reader)
nsq.run()
def demo():
@NSQPipeline(listen_topic='input', send_topic='topic1', source=True)
def start_chain(message):
print 'in the start...'
return 'foo'
@NSQPipeline(listen_topic='topic1', send_topic='topic2')
def foobar1(message):
print 'in function foobar1 with message %s...' % (message)
return 'bar'
@NSQPipeline(listen_topic='topic2')
def foobar2(message):
print 'in function foobar2 with message %s...' % (message)
return 'baz'
nsq.run()
def start_writer(q):
writer = Writer([nsq_url])
def pub_worker():
try:
while True:
event = q.get(block=False) # will raise Queue.Empty exception
event_json = {}
# parse payload
if 'desired' in event['object']['status']['twins'][0].keys():
desired_value = event['object']['status']['twins'][0][
'desired']['value']
decoded_value = str(
base64.b64decode(desired_value.encode('utf-8')),
'utf-8')
event_json['desired'] = json.loads(decoded_value)
# event['object']['status']['twins'][0]['desired']['value'] = json.loads(decoded_value)
if 'reported' in event['object']['status']['twins'][0].keys():
reported_value = event['object']['status']['twins'][0][
'reported']['value']
decoded_value = str(
base64.b64decode(reported_value.encode('utf-8')),
'utf-8')
event_json['reported'] = json.loads(decoded_value)
# event['object']['status']['twins'][0]['reported']['value'] = json.loads(decoded_value)
event_json['resourceVersion'] = event['object']['metadata'][
'resourceVersion']
event_json['device'] = event['object']['metadata']['name']
event_json['node'] = event['object']['spec']['nodeSelector'][
'nodeSelectorTerms'][0]['matchExpressions'][0]['values'][0]
writer.pub(topic, json.dumps(event_json).encode())
except mp.queues.Empty:
pass
except:
traceback.print_exc()
tornado.ioloop.PeriodicCallback(pub_worker, 10).start()
nsq.run()
def run(workers_module="nsqworker.workers", **kw):
worker = load_worker(workers_module)
logging.debug("Starting worker: %s" % worker)
topic = "%s%s" % (options.topic_prefix, options.topic)
legacy = False
try:
from nsq import LegacyReader
except ImportError:
legacy = True
if not legacy:
# nsq 0.5+
for name, handler in worker.handlers.iteritems():
r = nsq.Reader(
topic,
"%s_%s" % (options.channel, name[0:-len("_handler")]),
message_handler=handler,
lookupd_http_addresses=options.nsqlookupd_http_addresses,
**kw)
# override default preprocess and validate method with worker's
# method
r.validate_message = worker.validate_message
r.preprocess_message = worker.preprocess_message
else:
# nsq 0.4
r = nsq.Reader(
all_tasks=worker.handlers,
topic=topic,
channel=options.channel,
lookupd_http_addresses=options.nsqlookupd_http_addresses,
**kw)
r.validate_message = worker.validate_message
r.preprocess_message = worker.preprocess_message
if options.debug:
autoreload.start()
nsq.run()
def run(self, workers_module="nsqworker.workers", debug=False, **kw):
worker = load_worker(self._topic, self._channel, workers_module)
logger.debug("Starting worker: %s" % worker)
topic = "%s%s" % (self._topic_prefix, self._topic)
legacy = False
try:
from nsq import LegacyReader
except ImportError:
legacy = True
if not legacy:
# nsq 0.5+
for name, handler in iteritems(worker.handlers):
r = nsq.Reader(
topic,
"%s_%s" % (self._channel, name[0:-len("_handler")]),
message_handler=handler,
lookupd_http_addresses=self._nsqlookupd_http_addresses,
**kw)
# override default preprocess and validate method with worker's
# method
r.validate_message = worker.validate_message
r.preprocess_message = worker.preprocess_message
else:
# nsq 0.4
r = nsq.Reader(
all_tasks=worker.handlers,
topic=topic,
channel=self._channel,
lookupd_http_addresses=self._nsqlookupd_http_addresses,
**kw)
r.validate_message = worker.validate_message
r.preprocess_message = worker.preprocess_message
if debug:
autoreload.start()
nsq.run()
def run():
"""Run standalone sockjs server.
"""
# Bind to mysql, and generate mapping.
db.bind("mysql", host="localhost", port=3306, user="******", passwd="123456", db="threads")
db.generate_mapping(check_tables=True, create_tables=False)
reader = nsq.Reader(
lookupd_http_addresses=[NsqConfig.NSQ_LOOKUP_ADDRESS],
topic=NsqTopic.NSQ_INBOX_TOPIC,
channel="chat",
lookupd_poll_interval=15,
)
reader.set_message_handler(handle_boardcast_message)
app = tornado.web.Application(urls)
port = os.environ.get("PORT", 9999)
port = int(port)
app.listen(port)
nsq.run()
tornado.ioloop.IOLoop.instance().start()
def run():
"""Run standalone sockjs server.
"""
# Bind to mysql, and generate mapping.
db.bind("mysql",
host="localhost",
port=3306,
user="******",
passwd="123456",
db="threads")
db.generate_mapping(check_tables=True, create_tables=False)
reader = nsq.Reader(lookupd_http_addresses=[NsqConfig.NSQ_LOOKUP_ADDRESS],
topic=NsqTopic.NSQ_INBOX_TOPIC,
channel="chat",
lookupd_poll_interval=15)
reader.set_message_handler(handle_boardcast_message)
app = tornado.web.Application(urls)
port = os.environ.get("PORT", 9999)
port = int(port)
app.listen(port)
nsq.run()
tornado.ioloop.IOLoop.instance().start()
def main():
consumer = SleepConsumer()
r = nsq.Reader(message_handler=consumer.handler,
nsqd_tcp_addresses=['127.0.0.1:4150'],
topic='test-naysync', channel='asdf')
nsq.run()
tornado.options.define('destination_topic', type=str)
tornado.options.define('topic', type=str)
tornado.options.define('nsqd_tcp_address', type=str, multiple=True)
tornado.options.define('destination_nsqd_tcp_address', type=str, multiple=True)
tornado.options.define('lookupd_http_address', type=str, multiple=True)
tornado.options.define('channel', type=str)
tornado.options.define('max_in_flight', type=int, default=500)
tornado.options.parse_command_line()
assert tornado.options.options.topic
assert tornado.options.options.destination_nsqd_tcp_address
assert tornado.options.options.channel
destination_topic = str(tornado.options.options.destination_topic or # noqa: W504
tornado.options.options.topic)
lookupd_http_addresses = map(lambda addr: 'http://' + addr,
tornado.options.options.lookupd_http_address)
proxy = NSQProxy(destination_topic, tornado.options.options.destination_nsqd_tcp_address)
Reader(
topic=tornado.options.options.topic,
channel=tornado.options.options.channel,
message_handler=proxy.relay,
max_in_flight=tornado.options.options.max_in_flight,
lookupd_http_addresses=lookupd_http_addresses,
nsqd_tcp_addresses=tornado.options.options.nsqd_tcp_address,
)
run()
def start_running(self):
print(json.dumps({"address": self.__http_input}))
print(self.__outputfiledir)
self.__intervalFunction.start()
nsq.run()
def main():
nsq.run()
def start(cls):
nsq.run()
def send(self):
tornado.ioloop.PeriodicCallback(
self._pub_message,500).start()
nsq.run()
def test():
reader = nsq.Reader(topic=TOPIC,
channel="c",
nsqd_tcp_addresses=["%s:%d" % (NSQ_HOST, NSQ_PORT)],
message_handler=handler)
nsq.run()
logging.warn(msg_txt)
response = post_to_hipchat(msg_txt, args, user)
if args.verbose:
logging.warn(response.read())
return True
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--hipchat-auth-token', required=True)
parser.add_argument('--hipchat-room-id', required=True)
parser.add_argument('--nsq-topic', required=True)
parser.add_argument('--nsq-channel', default='nsqadmin2hipchat')
parser.add_argument('-v', '--verbose', action='store_true')
sources = parser.add_mutually_exclusive_group(required=True)
sources.add_argument('--nsqd-tcp-address', action='append')
sources.add_argument('--lookupd-http-address', action='append')
args = parser.parse_args()
kwargs = {
'topic': args.nsq_topic,
'channel': args.nsq_channel,
'message_handler' : functools.partial(process_message, args=args)
}
if args.lookupd_http_address:
addresses = [a if a.startswith("http") else "http://%s" % a for a in args.lookupd_http_address]
kwargs['lookupd_http_addresses'] = addresses
else:
kwargs['nsqd_tcp_addresses'] = args.nsqd_tcp_address
r = nsq.Reader(**kwargs)
nsq.run()
def run():
nsq.run()
def run(self):
nsq.run()
def run(self):
self._set_reader()
nsq.run()
def run(self):
self._get_ioloop().start()
nsq.run()
type=str,
multiple=True)
tornado.options.define('lookupd_http_address', type=str, multiple=True)
tornado.options.define('channel', type=str)
tornado.options.define('max_in_flight', type=int, default=500)
tornado.options.parse_command_line()
assert tornado.options.options.topic
assert tornado.options.options.destination_nsqd_tcp_address
assert tornado.options.options.channel
destination_topic = str(tornado.options.options.destination_topic
or # noqa: W504
tornado.options.options.topic)
lookupd_http_addresses = map(lambda addr: 'http://' + addr,
tornado.options.options.lookupd_http_address)
proxy = NSQProxy(destination_topic,
tornado.options.options.destination_nsqd_tcp_address)
Reader(
topic=tornado.options.options.topic,
channel=tornado.options.options.channel,
message_handler=proxy.relay,
max_in_flight=tornado.options.options.max_in_flight,
lookupd_http_addresses=lookupd_http_addresses,
nsqd_tcp_addresses=tornado.options.options.nsqd_tcp_address,
)
run()
def main():
""" main """
tornado.ioloop.PeriodicCallback(check, 2000).start()
nsq.run()
def open_spider(self, spider):
self.writer = nsq.Writer([self.nsq_url])
nsq.run()
def send_msg(self,doc_id,msg):
self.nsq.pub(self.topic,msg)
nsq.run()
instrumentOriginPicProcess = nsq.Reader(
message_handler=instrument_pic.origin_process,
nsqd_tcp_addresses=[nsq_conf['host'] + ":" + nsq_conf['port']],
topic='instrumentOriginPicProcess',
channel='channel',
lookupd_poll_interval=15,
max_tries=max_tries,
heartbeat_interval=heartbeat)
instrumentProPicProcess = nsq.Reader(
message_handler=instrument_pic.pro_process,
nsqd_tcp_addresses=[nsq_conf['host'] + ":" + nsq_conf['port']],
topic='instrumentProPicProcess',
channel='channel',
lookupd_poll_interval=15,
max_tries=max_tries,
heartbeat_interval=heartbeat)
instrumentPack = nsq.Reader(
message_handler=instrument_pack.pack,
nsqd_tcp_addresses=[nsq_conf['host'] + ":" + nsq_conf['port']],
topic='instrumentPack',
channel='channel',
lookupd_poll_interval=15,
max_tries=max_tries,
heartbeat_interval=heartbeat)
print("pic process 0.2")
nsq.run()
def start_nsq(self):
def parse_json_data(body):
d = json.loads(body.decode())
data_type = d["type"]
http_data = base64.standard_b64decode(d["content"])
http_decode_split = http_data.decode().split("\r\n")
http_first = http_decode_split[0]
http_body = http_decode_split[-1]
url = http_first.split()[1]
params = url.split("?")[1].split("&")
params = {p.split("=")[0]: p.split("=")[1] for p in params}
return data_type, params[SUPERTOPIC], params[
TOPIC], http_data, http_body.encode()
def send_msgq(msgq, data, topic):
if msgq:
if msgq.full():
logging.critical("topic {} msg queue full".format(topic))
else:
msgq.put(data)
else:
logging.error("topic {} msgq not founed".format(topic))
def auth_data_confirm(reg_element, topic, http_data):
auth_func = self._get_reg_attr(reg_element.auth_name)
if reg_element.auth_stage < 0:
return
reg_element.auth_stage += AUTH_STATE_UPDATA_STEP
try:
confirm_data, status = auth_func(http_data,
reg_element.auth_stage)
except Exception as e:
logging.error(e)
self._process_exception(e)
else:
self._process_auth(reg_element, topic, confirm_data, status)
def nsq_handle(msg):
data_type, super_topic, topic, http_data, http_body = parse_json_data(
msg.body)
logging.debug("nsq recieve type {} supertopic {} topic {}".format(
data_type, super_topic, topic))
if super_topic != self.cfg[NSQ][NSQ_SUPER_TOPIC]:
logging.error(
"nsq receive supertopic {} unmatch with {}".format(
super_topic, self.cfg[NSQ][NSQ_SUPER_TOPIC]))
return True
reg_element = Registry.registries.get(topic, None)
if reg_element == None:
logging.error("topic {} is not registry".format(topic))
return True
if data_type == DATA_TYPE_AUTH:
auth_data_confirm(reg_element, topic, http_data)
else:
if reg_element.auth_name != None:
send_msgq(reg_element.msgq_resp, http_data, topic)
else:
send_msgq(reg_element.msgq_resp, http_body, topic)
return True
nsq_cfg = self.cfg[NSQ]
nsq.Reader(message_handler=nsq_handle,
lookupd_http_addresses=[nsq_cfg[NSQ_HTTP_ADDR]],
topic=nsq_cfg[NSQ_SUPER_TOPIC],
channel=nsq_cfg[NSQ_SUPER_CHNEL],
max_in_flight=9)
logging.info("nsq agent start")
nsq.run()
