def run(self, initiator, conditions, **kwargs):
"""
Connects to NSQd instance specified by address, and evaluates
the conditions against every message received.
:param initiator:
:param conditions:
:return:
"""
# user can still initiate
initiator.execute()
reader = gnsq.Reader(self.topic, self.channel, self.address)
reader._funcy_messages = []
reader._funcy_take_n = self.take_n
@reader.on_message.connect
def handler(_r, message):
# each message finish and save it
message.finish()
_r._funcy_messages.append(message)
if len(_r._funcy_messages) == self.take_n:
_r.close()
reader.start()
logger.debug({
'class': self.__class__.__name__,
'num_messages': len(reader._funcy_messages),
})
return ValuesContainer(
reader._funcy_messages
)
def get_reader(settings, topic, channel, sentry_client=None):
"""
Get a :py:class:`gnsq.Reader` instance configured to connect to the
nsqd reader addresses specified in settings. The reader will read from
the specified topic and channel.
The caller is responsible for adding appropriate `on_message` hooks and
starting the reader.
"""
topic = resolve_topic(topic, settings=settings)
addrs = aslist(settings.get('nsq.reader.addresses', 'localhost:4150'))
reader = gnsq.Reader(topic, channel, nsqd_tcp_addresses=addrs)
if sentry_client is not None:
extra = {'topic': topic}
def _capture_exception(message, error):
if message is not None:
extra['message'] = message.body
sentry_client.captureException(exc_info=True, extra=extra)
def _capture_error(error):
sentry_client.captureException(exc_info=(type(error), error, None),
extra=extra)
def _capture_message(message):
if message is not None:
extra['message'] = message.body
sentry_client.captureMessage(extra=extra)
reader.on_exception.connect(_capture_exception, weak=False)
reader.on_giving_up.connect(_capture_message, weak=False)
reader.on_error.connect(_capture_error, weak=False)
return reader
def __init__(self, name, event, queue, config, channel="default"):
threading.Thread.__init__(self)
self.name = name
self.event = event
self.config = config
self.queue = queue
self.timeout = int(self.config["connection"]["timeout"])
self.max_tries = int(self.config["connection"]["max_tries"])
nsqlookup_url = "{}:{}".format(self.config["nsqlookupd"]["ip"],
self.config["nsqlookupd"]["port"])
self.reader = gnsq.Reader(
message_handler=self.message_handler,
lookupd_http_addresses=nsqlookup_url,
lookupd_poll_interval=self.config["nsqlookupd"]["interval"],
topic=self.config["topics"]["data_topic"],
channel=channel)
self.writer = gnsq.Nsqd(address=self.config["nsqd"]["ip"],
http_port=self.config["nsqd"]["port"])
self.lookup = gnsq.Lookupd(address=nsqlookup_url)
logger.info("{} initialized successfully".format(self.name))
def get_reader(self, topic, channel):
"""
Get a :py:class:`gnsq.Reader` instance configured to connect to the
nsqd reader addresses specified in settings. The reader will read from
the specified topic and channel.
The caller is responsible for adding appropriate `on_message` hooks and
starting the reader.
"""
setting = self.settings.get('nsq.reader.addresses', 'localhost:4150')
addrs = [line for line in setting.splitlines() if line]
reader = gnsq.Reader(topic, channel, nsqd_tcp_addresses=addrs)
return reader
def consumer():
reader = gnsq.Reader("topic",
"channel",
max_in_flight=10,
max_concurrency=-1,
lookupd_http_addresses=['127.0.0.1:4161'])
@reader.on_message.connect
def handler(server, message):
msg = "got message from {0}={1}".format(server,
message.body.decode("utf-8"))
print("(C)>>> {0}: {1}".format(server, msg))
reader.start()
def get_reader(settings, topic, channel):
"""
Get a :py:class:`gnsq.Reader` instance configured to connect to the
nsqd reader addresses specified in settings. The reader will read from
the specified topic and channel.
The caller is responsible for adding appropriate `on_message` hooks and
starting the reader.
"""
ns = settings.get('nsq.namespace')
addrs = aslist(settings.get('nsq.reader.addresses', 'localhost:4150'))
if ns is not None:
topic = '{0}-{1}'.format(ns, topic)
reader = gnsq.Reader(topic, channel, nsqd_tcp_addresses=addrs)
return reader
def get_reader(settings, topic, channel, sentry_client=None):
"""
Get a :py:class:`gnsq.Reader` instance configured to connect to the
nsqd reader addresses specified in settings. The reader will read from
the specified topic and channel.
The caller is responsible for adding appropriate `on_message` hooks and
starting the reader.
"""
topic = resolve_topic(topic, settings=settings)
addrs = aslist(settings.get('nsq.reader.addresses', 'localhost:4150'))
reader = gnsq.Reader(topic, channel, nsqd_tcp_addresses=addrs)
if sentry_client is not None:
_attach_error_handlers(reader, sentry_client)
return reader
if len(sys.argv) > 2:
outExt = sys.argv[2]
else:
outExt = None
writer = gnsq.Nsqd(address=NSQ_ADDR, tcp_port=NSQ_PORT)
links = set()
for line in linksFile:
link = line.strip().strip('"')
if len(link) > 0 and link != "null":
links.add(link)
benchmark = LinksBenchmark(outExt)
reader = gnsq.Reader('links_out_', 'links_test',
'%s:%d' % (NSQ_ADDR, NSQ_PORT))
@reader.on_message.connect
def handler(reader, message):
response = message.body
#print response
keyLoc = response.find('X-Bn-Event-Id:')
resultLoc = response.find('{')
if keyLoc > 0 and resultLoc > 0:
keyLoc = keyLoc + len('X-Bn-Event-Id: ')
key = response[keyLoc:keyLoc + 32]
#print 'Key: ', key
result = json.loads(response[resultLoc:])
from env import config
import json
logging.basicConfig()
logger = logging.getLogger('nsq')
fh = logging.FileHandler('/var/log/nsq.log')
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)
iface = [info.ipaddr for info in network.ifaces().values() if info.link]
print iface[0]
print config.rozofs.ip
reader = gnsq.Reader('storage', iface[0], config.rozofs.ip + ':4150')
def manage(events):
if events['ip'] in iface:
logger.info(events)
if events['event'] == 'storage.init':
result = delete_all()
elif events['event'] == 'storage.build':
loading()
result = quick_create()
else:
result = 'unknown event'
# pub_manage(events['ip'], result)
def handler_process(pipe):
global backend_conn, backend_lock
backend_conn = pipe
backend_lock = multiprocessing.Lock()
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.debug('Establishing subscriptions to NSQ topics...')
logging.getLogger('gnsq').setLevel(logging.INFO)
cb_reader = gnsq.Reader('call_blacklist', 'backend_py', '127.0.0.1:4150')
br_reader = gnsq.Reader('blacklist_remove', 'backend_py', '127.0.0.1:4150')
cw_reader = gnsq.Reader('call_whitelist', 'backend_py', '127.0.0.1:4150')
hg_reader = gnsq.Reader('history_get', 'backend_py', '127.0.0.1:4150')
bg_reader = gnsq.Reader('blacklist_get', 'backend_py', '127.0.0.1:4150')
sr_reader = gnsq.Reader('settings_request_all', 'backend_py',
'127.0.0.1:4150')
sg_reader = gnsq.Reader('setting_get', 'backend_py', '127.0.0.1:4150')
ss_reader = gnsq.Reader('setting_set', 'backend_py', '127.0.0.1:4150')
logger.debug('Wiring event handlers...')
@cb_reader.on_message.connect
def call_blacklist_handler(reader, message):
# received message: number:name
number = message.body.split(':')[0]
logger.info('Blacklisting ' + number + '...')
backend_lock.acquire()
backend_conn.send(['blacklist', number])
backend_lock.release()
@br_reader.on_message.connect
def blacklist_remove_handler(reader, message):
# received message: number
number = message.body
logger.info('Removing ' + number + ' from blacklist...')
backend_lock.acquire()
backend_conn.send(['blacklist_remove', number])
backend_lock.release()
@cw_reader.on_message.connect
def call_whitelist_handler(reader, message):
# received message: number:name
number = message.body.split(':')[0]
logger.info('Whitelisting ' + number + '...')
backend_lock.acquire()
backend_conn.send(['whitelist', number])
backend_lock.release()
@hg_reader.on_message.connect
def history_get_handler(reader, message):
params = message.body.split(':')
backend_lock.acquire()
backend_conn.send(['history', params])
backend_lock.release()
@bg_reader.on_message.connect
def blacklist_get_handler(reader, message):
params = message.body.split(':')
backend_lock.acquire()
backend_conn.send(['blacklist_get', params])
backend_lock.release()
# request for all settings
@sr_reader.on_message.connect
def settings_request_handler(reader, message):
backend_lock.acquire()
backend_conn.send(['settings_request'])
backend_lock.release()
# request to get one setting
@sg_reader.on_message.connect
def setting_get_handler(reader, message):
backend_lock.acquire()
backend_conn.send(['setting_get', message.body])
backend_lock.release()
# request to set one setting
@ss_reader.on_message.connect
def setting_set_handler(reader, message):
backend_lock.acquire()
alteredSetting = message.body.split(':')
backend_conn.send(
['setting_set', alteredSetting[0],
alteredSetting[1]]) # setting, new state
backend_lock.release()
logger.debug('Starting readers...')
cb_reader.start(block=False)
br_reader.start(block=False)
cw_reader.start(block=False)
hg_reader.start(block=False)
bg_reader.start(block=False)
sr_reader.start(block=False)
sg_reader.start(block=False)
ss_reader.start() # keep the process running by blocking