def __init__(self, topic, host="localhost", port="4151", log=DUMMY_LOG): self.topic = topic self.log = log self.host = host self.http_port = port self.connection = gnsq.Nsqd(address=self.host, http_port=self.http_port)
def producer(port=4151, sleep=1, loop=10): conn = gnsq.Nsqd(address='localhost', http_port=port) for n in range(loop): msg = "gevent test {0} sent to {1}!".format(n, port) print("(P)>>> {0}: {1}".format(producer, msg)) conn.publish('topic', json.dumps({"msg": msg, "id": n})) gevent.sleep(sleep)
def __init__(self, name, event, queue, config, channel="default"): threading.Thread.__init__(self) self.name = name self.event = event self.config = config self.queue = queue self.timeout = int(self.config["connection"]["timeout"]) self.max_tries = int(self.config["connection"]["max_tries"]) nsqlookup_url = "{}:{}".format(self.config["nsqlookupd"]["ip"], self.config["nsqlookupd"]["port"]) self.reader = gnsq.Reader( message_handler=self.message_handler, lookupd_http_addresses=nsqlookup_url, lookupd_poll_interval=self.config["nsqlookupd"]["interval"], topic=self.config["topics"]["data_topic"], channel=channel) self.writer = gnsq.Nsqd(address=self.config["nsqd"]["ip"], http_port=self.config["nsqd"]["port"]) self.lookup = gnsq.Lookupd(address=nsqlookup_url) logger.info("{} initialized successfully".format(self.name))
def _(lookupd_server, nsqd_server): lookupd = gnsq.Lookupd(lookupd_server.http_address) conn = gnsq.Nsqd(nsqd_server.address, http_port=nsqd_server.http_port) assert len(lookupd.topics()['topics']) == 0 assert len(lookupd.channels('topic')['channels']) == 0 assert len(lookupd.nodes()['producers']) == 1 conn.create_topic('topic') gevent.sleep(0.1) info = lookupd.lookup('topic') assert len(info['channels']) == 0 assert len(info['producers']) == 1 assert len(lookupd.topics()['topics']) == 1 assert len(lookupd.channels('topic')['channels']) == 0 conn.create_channel('topic', 'channel') gevent.sleep(0.1) info = lookupd.lookup('topic') assert len(info['channels']) == 1 assert len(info['producers']) == 1 assert len(lookupd.topics()['topics']) == 1 assert len(lookupd.channels('topic')['channels']) == 1
def test_topics_channels(): with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) assert len(conn.stats()['topics']) == 0 with pytest.raises(gnsq.errors.NSQHttpError): conn.delete_topic('topic') conn.create_topic('topic') topics = conn.stats()['topics'] assert len(topics) == 1 assert topics[0]['topic_name'] == 'topic' conn.delete_topic('topic') assert len(conn.stats()['topics']) == 0 with pytest.raises(gnsq.errors.NSQHttpError): conn.create_channel('topic', 'channel') with pytest.raises(gnsq.errors.NSQHttpError): conn.delete_channel('topic', 'channel') conn.create_topic('topic') assert len(conn.stats()['topics'][0]['channels']) == 0 conn.create_channel('topic', 'channel') channels = conn.stats()['topics'][0]['channels'] assert len(channels) == 1 assert channels[0]['channel_name'] == 'channel' conn.delete_channel('topic', 'channel') assert len(conn.stats()['topics'][0]['channels']) == 0
def test_publish(): with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) conn.publish('topic', b'sup') assert conn.stats()['topics'][0]['depth'] == 1 conn.multipublish('topic', ['sup', 'sup']) assert conn.stats()['topics'][0]['depth'] == 3 conn.multipublish('topic', iter(['sup', 'sup', 'sup'])) assert conn.stats()['topics'][0]['depth'] == 6 conn.empty_topic('topic') assert conn.stats()['topics'][0]['depth'] == 0 conn.create_topic('topic') conn.create_channel('topic', 'channel') conn.publish('topic', b'sup') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1 conn.empty_channel('topic', 'channel') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0 if server.version < (0, 3, 6): return conn.publish('topic', b'sup', 60 * 1000) stats = conn.stats() assert stats['topics'][0]['channels'][0]['depth'] == 0 assert stats['topics'][0]['channels'][0]['deferred_count'] == 1
def test_lookup(): with LookupdIntegrationServer() as lookupd_server: nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) with NsqdIntegrationServer( lookupd=lookupd_server.tcp_address) as nsqd_server: lookupd = gnsq.Lookupd(lookupd_server.http_address) conn = gnsq.Nsqd(nsqd_server.address, http_port=nsqd_server.http_port) gevent.sleep(0.1) assert len(lookupd.topics()['topics']) == 0 assert len(lookupd.channels('topic')['channels']) == 0 assert len(lookupd.nodes()['producers']) == 1 conn.create_topic('topic') gevent.sleep(0.1) info = lookupd.lookup('topic') assert len(info['channels']) == 0 assert len(info['producers']) == 1 assert len(lookupd.topics()['topics']) == 1 assert len(lookupd.channels('topic')['channels']) == 0 conn.create_channel('topic', 'channel') gevent.sleep(0.1) info = lookupd.lookup('topic') assert len(info['channels']) == 1 assert len(info['producers']) == 1 assert len(lookupd.topics()['topics']) == 1 assert len(lookupd.channels('topic')['channels']) == 1
def __init__(self, nsqd_addr="127.0.0.1",port=4151, topic="default"): """ :param nsqd_addrs: nsqd地址 :param topic: topic """ self.nsq = gnsq.Nsqd(address=nsqd_addr, http_port=port) self.topic = topic
def execute(self): logger.debug('%s', { 'message': 'publishing_message_nsq', 'body': self.message, }) return ValuesContainer( gnsq.Nsqd(address=self.nsqd_address).publish( self.topic, json.dumps(self.message)))
def get_writer(self): """ Get a :py:class:`gnsq.Nsqd` instance configured to connect to the nsqd writer address configured in settings. The writer communicates over the nsq HTTP API and does not hold a connection open to the nsq instance. """ setting = self.settings.get('nsq.writer.address', 'localhost:4151') hostname, port = setting.split(':', 1) nsqd = gnsq.Nsqd(hostname, http_port=port) return nsqd
def init(self): self.poller = mq.Poller() conn = gnsq.Nsqd(address=config.zoofs.ip, http_port=4151) pub = mq.pub_socket('eventd') self.ksub = KernelEventSub(pub) self.usub = UserspaceEventSub(pub) self.uevent = UEventSub(pub, conn) self.poller.register(self.ksub) self.poller.register(self.usub) self.poller.register(self.uevent)
def main(args): print(args) message = { 'from': 'test-nsq-publish.py', 'message': 'hi-{}', } for i in range(args.num_messages): m = copy.copy(message) m['number'] = i m['message'] = m['message'].format(i) print('Sending message: {}'.format(m)) gnsq.Nsqd(address=args.nsqd_address).publish(args.topic, json.dumps(m))
def __init__(self, name, event, queue, config): threading.Thread.__init__(self) self.name = name self.config = config self.event = event self.queue = queue self.timeout = int(self.config["connection"]["timeout"]) self.max_tries = int(self.config["connection"]["max_tries"]) nsqlookup_url = "{}:{}".format(self.config["nsqlookupd"]["ip"], self.config["nsqlookupd"]["port"]) self.writer = gnsq.Nsqd(address=self.config["nsqd"]["ip"], http_port=self.config["nsqd"]["port"]) self.lookup = gnsq.Lookupd(address=nsqlookup_url) logger.info("{} initialized successfully".format(self.name))
def test_publish(): with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) conn.publish('topic', b'sup') assert conn.stats()['topics'][0]['depth'] == 1 conn.multipublish('topic', ['sup', 'sup']) assert conn.stats()['topics'][0]['depth'] == 3 conn.multipublish('topic', iter(['sup', 'sup', 'sup'])) assert conn.stats()['topics'][0]['depth'] == 6 conn.empty_topic('topic') assert conn.stats()['topics'][0]['depth'] == 0 conn.create_topic('topic') conn.create_channel('topic', 'channel') conn.publish('topic', b'sup') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1 conn.empty_channel('topic', 'channel') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0
def __init__(self, namespace, *args, **kwargs): self.client = gnsq.Nsqd(*args, **kwargs) self.namespace = namespace
def test_basic(): with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) assert conn.ping() == b'OK' assert 'topics' in conn.stats() assert 'version' in conn.info()
if len(sys.argv) < 2: print "Usage: ./links_benchmark.py [links file] [optional: results output name]" sys.exit(0) try: linksFile = open(sys.argv[1], "r") except: print "Error opening: ", sys.argv[1] sys.exit(0) if len(sys.argv) > 2: outExt = sys.argv[2] else: outExt = None writer = gnsq.Nsqd(address=NSQ_ADDR, tcp_port=NSQ_PORT) links = set() for line in linksFile: link = line.strip().strip('"') if len(link) > 0 and link != "null": links.add(link) benchmark = LinksBenchmark(outExt) reader = gnsq.Reader('links_out_', 'links_test', '%s:%d' % (NSQ_ADDR, NSQ_PORT)) @reader.on_message.connect def handler(reader, message): response = message.body
import gnsq from env import config conn = gnsq.Nsqd(address=config.rozofs.ip, http_port=4151) def pub_nsq(o): conn.publish('CloudEvent', o) def pub_nsq_info(o): conn.publish('CloudInfo', o)
def init(self): self._poller = mq.Poller() conn = gnsq.Nsqd(address=config.zoofs.ip, http_port=config.zoofs.port) self._rsp = Response(conn) self._poller.register(self._rsp)
def start(): currentCall = None logging.basicConfig( format='[%(levelname)s %(name)s] %(asctime)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG) logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) logger.debug('Loading blacklist...') load_blacklist() logger.debug('Loading whitelist...') load_whitelist() logger.debug('Restoring call history...') restore_history() logger.debug('Restoring wildcard rules...') restore_wildcards() logger.debug('Wildcard rule: ' + wildcard_rule) logger.debug('Loading settings...') settings.load_settings() logger.debug('Establishing pub connection to NSQ...') pub = gnsq.Nsqd(address='localhost', http_port=4151) logging.getLogger('urllib3').setLevel( logging.WARNING) # disable annoying urllib3 debugging logger.debug('Spinning up NSQ handler thread...') handler_pipe, handler_child_pipe = Pipe() handler_proc = Process(target=handlers.handler_process, args=(handler_child_pipe, )) handler_proc.start() logger.debug('Spinning up modem thread...') modem_pipe, modem_child_pipe = Pipe() modem_proc = Process(target=modem.modem_process, args=(modem_child_pipe, )) modem_proc.start() relay.init_relay_gpio() pub.publish('heartbeat', 'no') last_heartbeat_time = datetime.now() while True: # heartbeat if (datetime.now() - last_heartbeat_time).total_seconds() > 60: pub.publish('heartbeat', 'no') last_heartbeat_time = datetime.now() if handler_pipe.poll(): # message from NSQ msg = handler_pipe.recv() logger.debug('Message from NSQ: ' + str(msg)) if msg[0] == 'blacklist': # append number to blacklist if msg[1] not in blacklist: # don't add duplicates blacklist.add(msg[1]) if currentCall is not None: # blacklist currently incoming call modem_pipe.send('hangup') append_blacklist(msg[1]) elif msg[0] == 'blacklist_remove': if msg[1] in blacklist: # don't do expensive file rewrite if number wasn't in blacklist blacklist.remove(msg[1]) write_blacklist( ) # rewrite blacklist file to reflect removed number elif msg[0] == 'whitelist': # append number to whitelist whitelist.add(msg[1]) elif msg[0] == 'history': resp = history_to_str(msg[1]) pub.publish('history_give', resp) elif msg[0] == 'blacklist_get': resp = blacklist_to_str(msg[1]) pub.publish('blacklist_give', resp) elif msg[ 0] == 'settings_request': # respond to all settings request setList = sorted(settings.registry.keys()) pub.publish('settings_all', ':'.join(setList)) elif msg[0] == 'setting_get': # respond to single setting request setObj = settings.registry[msg[1]] # single setting requested allStates = ';'.join( setObj['states']) # str for all possible states respList = [ msg[1], setObj['help_text'], setObj['current_state'], allStates ] resp = ':'.join(respList) pub.publish('setting_give', resp) elif msg[0] == 'setting_set': # change the setting # special treatment of off-device programming setting if msg[1] == 'Off-device programming': partition = off_device.mount_device() if partition != None: if msg[2] == 'Append': off_device.append_lists(partition) elif msg[2] == 'Replace': off_device.replace_lists(partition) elif msg[2] == 'Export': off_device.copy_lists(partition) # special treatment of filter and wildcard settings elif msg[1] == 'Filter Disable': settings.filter_disable = True if msg[ 2] == 'Enabled' else False elif msg[1] == 'Wildcards': settings.wildcard_disable = True if msg[ 2] == 'Enabled' else False else: settings.registry[msg[1]]['current_state'] = msg[2] settings.save_settings() if modem_pipe.poll(): # incoming call from modem currentCall = modem_pipe.recv() relay.set_ans_machine_relay_pin(False) relay.set_telephone_out_relay_pin(False) if settings.filter_disable: append_history(currentCall) pub.publish('call_received', currentCall.number + ':' + currentCall.name) modem_pipe.send('pass') currentCall = None else: mode = settings.registry['Filtering mode']['current_state'] if (mode == 'Blacklist'): if (currentCall.number in blacklist) or ( (not settings.wildcard_disable) and matches_wildcard(currentCall.number)): relay.set_telephone_out_relay_pin(True) currentCall.wasBlocked = '1' append_history(currentCall) modem_pipe.send('hangup') currentCall = None else: append_history(currentCall) pub.publish( 'call_received', currentCall.number + ':' + currentCall.name) modem_pipe.send('pass') currentCall = None elif (mode == 'Whitelist'): if (currentCall.number in whitelist): append_history(currentCall) pub.publish( 'call_received', currentCall.number + ':' + currentCall.name) modem_pipe.send('pass') currentCall = None else: relay.set_telephone_out_relay_pin(True) currentCall.wasBlocked = '1' append_history(currentCall) modem_pipe.send('hangup') currentCall = None else: # is in Greylist mode; can check, but just assuming for now if (currentCall.number in blacklist): relay.set_telephone_out_relay_pin(True) currentCall.wasBlocked = '1' append_history(currentCall) modem_pipe.send('hangup') currentCall = None elif (currentCall.number in whitelist): append_history(currentCall) pub.publish( 'call_received', currentCall.number + ':' + currentCall.name) modem_pipe.send('pass') currentCall = None elif ((not settings.wildcard_disable) and matches_wildcard(currentCall.number)): relay.set_ans_machine_relay_pin(True) relay.set_telephone_out_relay_pin(True) append_history(currentCall) modem_pipe.send('ans_machine') currentCall = None else: append_history(currentCall) pub.publish( 'call_received', currentCall.number + ':' + currentCall.name) modem_pipe.send('pass') currentCall = None pub.publish('history_give', history_to_str([10, 0])) time.sleep( 0.05 ) # keep from using all of the CPU handling messages from threads
def __init__(self, topic, nsqd_hosts=['localhost']): if not nsqd_hosts: raise ValueError('at least one nsqd host is required') self.topic = topic self.nsqds = itertools.cycle([gnsq.Nsqd(host) for host in nsqd_hosts])