def __init__(self, topic, host="localhost", port="4151", log=DUMMY_LOG):
self.topic = topic
self.log = log
self.host = host
self.http_port = port
self.connection = gnsq.Nsqd(address=self.host,
http_port=self.http_port)
def producer(port=4151, sleep=1, loop=10):
conn = gnsq.Nsqd(address='localhost', http_port=port)
for n in range(loop):
msg = "gevent test {0} sent to {1}!".format(n, port)
print("(P)>>> {0}: {1}".format(producer, msg))
conn.publish('topic', json.dumps({"msg": msg, "id": n}))
gevent.sleep(sleep)
def __init__(self, name, event, queue, config, channel="default"):
threading.Thread.__init__(self)
self.name = name
self.event = event
self.config = config
self.queue = queue
self.timeout = int(self.config["connection"]["timeout"])
self.max_tries = int(self.config["connection"]["max_tries"])
nsqlookup_url = "{}:{}".format(self.config["nsqlookupd"]["ip"],
self.config["nsqlookupd"]["port"])
self.reader = gnsq.Reader(
message_handler=self.message_handler,
lookupd_http_addresses=nsqlookup_url,
lookupd_poll_interval=self.config["nsqlookupd"]["interval"],
topic=self.config["topics"]["data_topic"],
channel=channel)
self.writer = gnsq.Nsqd(address=self.config["nsqd"]["ip"],
http_port=self.config["nsqd"]["port"])
self.lookup = gnsq.Lookupd(address=nsqlookup_url)
logger.info("{} initialized successfully".format(self.name))
def _(lookupd_server, nsqd_server):
lookupd = gnsq.Lookupd(lookupd_server.http_address)
conn = gnsq.Nsqd(nsqd_server.address, http_port=nsqd_server.http_port)
assert len(lookupd.topics()['topics']) == 0
assert len(lookupd.channels('topic')['channels']) == 0
assert len(lookupd.nodes()['producers']) == 1
conn.create_topic('topic')
gevent.sleep(0.1)
info = lookupd.lookup('topic')
assert len(info['channels']) == 0
assert len(info['producers']) == 1
assert len(lookupd.topics()['topics']) == 1
assert len(lookupd.channels('topic')['channels']) == 0
conn.create_channel('topic', 'channel')
gevent.sleep(0.1)
info = lookupd.lookup('topic')
assert len(info['channels']) == 1
assert len(info['producers']) == 1
assert len(lookupd.topics()['topics']) == 1
assert len(lookupd.channels('topic')['channels']) == 1
def test_topics_channels():
with NsqdIntegrationServer() as server:
conn = gnsq.Nsqd(server.address, http_port=server.http_port)
assert len(conn.stats()['topics']) == 0
with pytest.raises(gnsq.errors.NSQHttpError):
conn.delete_topic('topic')
conn.create_topic('topic')
topics = conn.stats()['topics']
assert len(topics) == 1
assert topics[0]['topic_name'] == 'topic'
conn.delete_topic('topic')
assert len(conn.stats()['topics']) == 0
with pytest.raises(gnsq.errors.NSQHttpError):
conn.create_channel('topic', 'channel')
with pytest.raises(gnsq.errors.NSQHttpError):
conn.delete_channel('topic', 'channel')
conn.create_topic('topic')
assert len(conn.stats()['topics'][0]['channels']) == 0
conn.create_channel('topic', 'channel')
channels = conn.stats()['topics'][0]['channels']
assert len(channels) == 1
assert channels[0]['channel_name'] == 'channel'
conn.delete_channel('topic', 'channel')
assert len(conn.stats()['topics'][0]['channels']) == 0
def test_publish():
with NsqdIntegrationServer() as server:
conn = gnsq.Nsqd(server.address, http_port=server.http_port)
conn.publish('topic', b'sup')
assert conn.stats()['topics'][0]['depth'] == 1
conn.multipublish('topic', ['sup', 'sup'])
assert conn.stats()['topics'][0]['depth'] == 3
conn.multipublish('topic', iter(['sup', 'sup', 'sup']))
assert conn.stats()['topics'][0]['depth'] == 6
conn.empty_topic('topic')
assert conn.stats()['topics'][0]['depth'] == 0
conn.create_topic('topic')
conn.create_channel('topic', 'channel')
conn.publish('topic', b'sup')
assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1
conn.empty_channel('topic', 'channel')
assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0
if server.version < (0, 3, 6):
return
conn.publish('topic', b'sup', 60 * 1000)
stats = conn.stats()
assert stats['topics'][0]['channels'][0]['depth'] == 0
assert stats['topics'][0]['channels'][0]['deferred_count'] == 1
def test_lookup():
with LookupdIntegrationServer() as lookupd_server:
nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
with NsqdIntegrationServer(
lookupd=lookupd_server.tcp_address) as nsqd_server:
lookupd = gnsq.Lookupd(lookupd_server.http_address)
conn = gnsq.Nsqd(nsqd_server.address,
http_port=nsqd_server.http_port)
gevent.sleep(0.1)
assert len(lookupd.topics()['topics']) == 0
assert len(lookupd.channels('topic')['channels']) == 0
assert len(lookupd.nodes()['producers']) == 1
conn.create_topic('topic')
gevent.sleep(0.1)
info = lookupd.lookup('topic')
assert len(info['channels']) == 0
assert len(info['producers']) == 1
assert len(lookupd.topics()['topics']) == 1
assert len(lookupd.channels('topic')['channels']) == 0
conn.create_channel('topic', 'channel')
gevent.sleep(0.1)
info = lookupd.lookup('topic')
assert len(info['channels']) == 1
assert len(info['producers']) == 1
assert len(lookupd.topics()['topics']) == 1
assert len(lookupd.channels('topic')['channels']) == 1
def __init__(self, nsqd_addr="127.0.0.1",port=4151, topic="default"):
"""
:param nsqd_addrs: nsqd地址
:param topic: topic
"""
self.nsq = gnsq.Nsqd(address=nsqd_addr, http_port=port)
self.topic = topic
def execute(self):
logger.debug('%s', {
'message': 'publishing_message_nsq',
'body': self.message,
})
return ValuesContainer(
gnsq.Nsqd(address=self.nsqd_address).publish(
self.topic, json.dumps(self.message)))
def get_writer(self):
"""
Get a :py:class:`gnsq.Nsqd` instance configured to connect to the nsqd
writer address configured in settings. The writer communicates over the
nsq HTTP API and does not hold a connection open to the nsq instance.
"""
setting = self.settings.get('nsq.writer.address', 'localhost:4151')
hostname, port = setting.split(':', 1)
nsqd = gnsq.Nsqd(hostname, http_port=port)
return nsqd
def init(self):
self.poller = mq.Poller()
conn = gnsq.Nsqd(address=config.zoofs.ip, http_port=4151)
pub = mq.pub_socket('eventd')
self.ksub = KernelEventSub(pub)
self.usub = UserspaceEventSub(pub)
self.uevent = UEventSub(pub, conn)
self.poller.register(self.ksub)
self.poller.register(self.usub)
self.poller.register(self.uevent)
def main(args):
print(args)
message = {
'from': 'test-nsq-publish.py',
'message': 'hi-{}',
}
for i in range(args.num_messages):
m = copy.copy(message)
m['number'] = i
m['message'] = m['message'].format(i)
print('Sending message: {}'.format(m))
gnsq.Nsqd(address=args.nsqd_address).publish(args.topic, json.dumps(m))
def __init__(self, name, event, queue, config):
threading.Thread.__init__(self)
self.name = name
self.config = config
self.event = event
self.queue = queue
self.timeout = int(self.config["connection"]["timeout"])
self.max_tries = int(self.config["connection"]["max_tries"])
nsqlookup_url = "{}:{}".format(self.config["nsqlookupd"]["ip"],
self.config["nsqlookupd"]["port"])
self.writer = gnsq.Nsqd(address=self.config["nsqd"]["ip"],
http_port=self.config["nsqd"]["port"])
self.lookup = gnsq.Lookupd(address=nsqlookup_url)
logger.info("{} initialized successfully".format(self.name))
def test_publish():
with NsqdIntegrationServer() as server:
conn = gnsq.Nsqd(server.address, http_port=server.http_port)
conn.publish('topic', b'sup')
assert conn.stats()['topics'][0]['depth'] == 1
conn.multipublish('topic', ['sup', 'sup'])
assert conn.stats()['topics'][0]['depth'] == 3
conn.multipublish('topic', iter(['sup', 'sup', 'sup']))
assert conn.stats()['topics'][0]['depth'] == 6
conn.empty_topic('topic')
assert conn.stats()['topics'][0]['depth'] == 0
conn.create_topic('topic')
conn.create_channel('topic', 'channel')
conn.publish('topic', b'sup')
assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1
conn.empty_channel('topic', 'channel')
assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0
def __init__(self, namespace, *args, **kwargs):
self.client = gnsq.Nsqd(*args, **kwargs)
self.namespace = namespace
def test_basic():
with NsqdIntegrationServer() as server:
conn = gnsq.Nsqd(server.address, http_port=server.http_port)
assert conn.ping() == b'OK'
assert 'topics' in conn.stats()
assert 'version' in conn.info()
if len(sys.argv) < 2:
print "Usage: ./links_benchmark.py [links file] [optional: results output name]"
sys.exit(0)
try:
linksFile = open(sys.argv[1], "r")
except:
print "Error opening: ", sys.argv[1]
sys.exit(0)
if len(sys.argv) > 2:
outExt = sys.argv[2]
else:
outExt = None
writer = gnsq.Nsqd(address=NSQ_ADDR, tcp_port=NSQ_PORT)
links = set()
for line in linksFile:
link = line.strip().strip('"')
if len(link) > 0 and link != "null":
links.add(link)
benchmark = LinksBenchmark(outExt)
reader = gnsq.Reader('links_out_', 'links_test',
'%s:%d' % (NSQ_ADDR, NSQ_PORT))
@reader.on_message.connect
def handler(reader, message):
response = message.body
import gnsq
from env import config
conn = gnsq.Nsqd(address=config.rozofs.ip, http_port=4151)
def pub_nsq(o):
conn.publish('CloudEvent', o)
def pub_nsq_info(o):
conn.publish('CloudInfo', o)
def init(self):
self._poller = mq.Poller()
conn = gnsq.Nsqd(address=config.zoofs.ip, http_port=config.zoofs.port)
self._rsp = Response(conn)
self._poller.register(self._rsp)
def start():
currentCall = None
logging.basicConfig(
format='[%(levelname)s %(name)s] %(asctime)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.debug('Loading blacklist...')
load_blacklist()
logger.debug('Loading whitelist...')
load_whitelist()
logger.debug('Restoring call history...')
restore_history()
logger.debug('Restoring wildcard rules...')
restore_wildcards()
logger.debug('Wildcard rule: ' + wildcard_rule)
logger.debug('Loading settings...')
settings.load_settings()
logger.debug('Establishing pub connection to NSQ...')
pub = gnsq.Nsqd(address='localhost', http_port=4151)
logging.getLogger('urllib3').setLevel(
logging.WARNING) # disable annoying urllib3 debugging
logger.debug('Spinning up NSQ handler thread...')
handler_pipe, handler_child_pipe = Pipe()
handler_proc = Process(target=handlers.handler_process,
args=(handler_child_pipe, ))
handler_proc.start()
logger.debug('Spinning up modem thread...')
modem_pipe, modem_child_pipe = Pipe()
modem_proc = Process(target=modem.modem_process, args=(modem_child_pipe, ))
modem_proc.start()
relay.init_relay_gpio()
pub.publish('heartbeat', 'no')
last_heartbeat_time = datetime.now()
while True:
# heartbeat
if (datetime.now() - last_heartbeat_time).total_seconds() > 60:
pub.publish('heartbeat', 'no')
last_heartbeat_time = datetime.now()
if handler_pipe.poll(): # message from NSQ
msg = handler_pipe.recv()
logger.debug('Message from NSQ: ' + str(msg))
if msg[0] == 'blacklist': # append number to blacklist
if msg[1] not in blacklist: # don't add duplicates
blacklist.add(msg[1])
if currentCall is not None: # blacklist currently incoming call
modem_pipe.send('hangup')
append_blacklist(msg[1])
elif msg[0] == 'blacklist_remove':
if msg[1] in blacklist: # don't do expensive file rewrite if number wasn't in blacklist
blacklist.remove(msg[1])
write_blacklist(
) # rewrite blacklist file to reflect removed number
elif msg[0] == 'whitelist': # append number to whitelist
whitelist.add(msg[1])
elif msg[0] == 'history':
resp = history_to_str(msg[1])
pub.publish('history_give', resp)
elif msg[0] == 'blacklist_get':
resp = blacklist_to_str(msg[1])
pub.publish('blacklist_give', resp)
elif msg[
0] == 'settings_request': # respond to all settings request
setList = sorted(settings.registry.keys())
pub.publish('settings_all', ':'.join(setList))
elif msg[0] == 'setting_get': # respond to single setting request
setObj = settings.registry[msg[1]] # single setting requested
allStates = ';'.join(
setObj['states']) # str for all possible states
respList = [
msg[1], setObj['help_text'], setObj['current_state'],
allStates
]
resp = ':'.join(respList)
pub.publish('setting_give', resp)
elif msg[0] == 'setting_set': # change the setting
# special treatment of off-device programming setting
if msg[1] == 'Off-device programming':
partition = off_device.mount_device()
if partition != None:
if msg[2] == 'Append':
off_device.append_lists(partition)
elif msg[2] == 'Replace':
off_device.replace_lists(partition)
elif msg[2] == 'Export':
off_device.copy_lists(partition)
# special treatment of filter and wildcard settings
elif msg[1] == 'Filter Disable':
settings.filter_disable = True if msg[
2] == 'Enabled' else False
elif msg[1] == 'Wildcards':
settings.wildcard_disable = True if msg[
2] == 'Enabled' else False
else:
settings.registry[msg[1]]['current_state'] = msg[2]
settings.save_settings()
if modem_pipe.poll(): # incoming call from modem
currentCall = modem_pipe.recv()
relay.set_ans_machine_relay_pin(False)
relay.set_telephone_out_relay_pin(False)
if settings.filter_disable:
append_history(currentCall)
pub.publish('call_received',
currentCall.number + ':' + currentCall.name)
modem_pipe.send('pass')
currentCall = None
else:
mode = settings.registry['Filtering mode']['current_state']
if (mode == 'Blacklist'):
if (currentCall.number in blacklist) or (
(not settings.wildcard_disable)
and matches_wildcard(currentCall.number)):
relay.set_telephone_out_relay_pin(True)
currentCall.wasBlocked = '1'
append_history(currentCall)
modem_pipe.send('hangup')
currentCall = None
else:
append_history(currentCall)
pub.publish(
'call_received',
currentCall.number + ':' + currentCall.name)
modem_pipe.send('pass')
currentCall = None
elif (mode == 'Whitelist'):
if (currentCall.number in whitelist):
append_history(currentCall)
pub.publish(
'call_received',
currentCall.number + ':' + currentCall.name)
modem_pipe.send('pass')
currentCall = None
else:
relay.set_telephone_out_relay_pin(True)
currentCall.wasBlocked = '1'
append_history(currentCall)
modem_pipe.send('hangup')
currentCall = None
else: # is in Greylist mode; can check, but just assuming for now
if (currentCall.number in blacklist):
relay.set_telephone_out_relay_pin(True)
currentCall.wasBlocked = '1'
append_history(currentCall)
modem_pipe.send('hangup')
currentCall = None
elif (currentCall.number in whitelist):
append_history(currentCall)
pub.publish(
'call_received',
currentCall.number + ':' + currentCall.name)
modem_pipe.send('pass')
currentCall = None
elif ((not settings.wildcard_disable)
and matches_wildcard(currentCall.number)):
relay.set_ans_machine_relay_pin(True)
relay.set_telephone_out_relay_pin(True)
append_history(currentCall)
modem_pipe.send('ans_machine')
currentCall = None
else:
append_history(currentCall)
pub.publish(
'call_received',
currentCall.number + ':' + currentCall.name)
modem_pipe.send('pass')
currentCall = None
pub.publish('history_give', history_to_str([10, 0]))
time.sleep(
0.05
) # keep from using all of the CPU handling messages from threads
def __init__(self, topic, nsqd_hosts=['localhost']):
if not nsqd_hosts:
raise ValueError('at least one nsqd host is required')
self.topic = topic
self.nsqds = itertools.cycle([gnsq.Nsqd(host) for host in nsqd_hosts])
