def __init__(self):
self.__code_obj = {}
nsq.Reader(
topic='mq_input',
channel='mq',
name="mq_input.mq",
nsqd_tcp_addresses=['127.0.0.1:4150'],
lookupd_http_addresses=['http://127.0.0.1:4161'],
message_handler=self._handle_monitor,
heartbeat_interval=10,
tls_options={'cert_reqs': ssl.CERT_NONE},
output_buffer_size=4096,
output_buffer_timeout=100,
max_tries=5,
max_in_flight=9,
lookupd_poll_interval=60,
low_rdy_idle_timeout=10,
max_backoff_duration=128,
lookupd_poll_jitter=0.3,
lookupd_connect_timeout=1,
lookupd_request_timeout=2,
)
self.writer = nsq.Writer(['127.0.0.1:4150'])
self.redis_client = redis.Redis(host='127.0.0.1', port=6379)
def publish(src, dest, msgType, content):
data = {}
data['MacAddress'] = content['MacAddress']
data['longitude'] = content['longitude']
data['latitude'] = content['latitude']
data['distance'] = content['distance']
#should any filtering take place?
json_location = json.loads(json.dumps(data))
def publish_destination():
writer.pub(dest, json.dumps(json_location), finish_pub)
def finish_pub(conn, data):
print(data)
tornado.ioloop.IOLoop.current().stop()
writer = nsq.Writer(['127.0.0.1:4150'])
tornado.ioloop.PeriodicCallback(publish_destination,1000).start()
nsq.run()
def __init__(self, req, _name, class_path, class_args):
self.__req = req
self.name = _name
self.class_path = class_path
self.class_args = class_args
self.writer = nsq.Writer(['127.0.0.1:4150'])
def get_writer(self):
if len(NSQD_TCP_ADDRESSES) == 0:
self.logger.warning(
"Writer functionality is DISABLED. To enable it please provide NSQD_TCP_ADDRESSES."
)
return None
return nsq.Writer(nsqd_tcp_addresses=NSQD_TCP_ADDRESSES)
def __init__(self, conn=None):
self.connection = conn
#self.write_session = requests.Session()
self.queue_name = None
self.mq = 'nsq'
self.buffered_messages = []
self.exception = None
self.counter = 0
self.nsq_running = False
self.writer = nsq.Writer(['{}:4150'.format(self.connection.server)])
def test_writer_await_pub(self):
topic = 'test_writer_mpub_%s' % time.time()
w = nsq.Writer(nsqd_tcp_addresses=['127.0.0.1:4150'],
**self.identify_options)
@tornado.gen.coroutine
def trypub():
yield w.pub(topic, b'{"one": 1}')
yield w.pub(topic, b'{"two": 2}')
self.stop("OK")
self.io_loop.call_later(0.1, trypub)
result = self.wait()
print(str(result))
assert result == "OK"
def test_writer_mpub_one(self):
topic = 'test_writer_mpub_%s' % time.time()
w = nsq.Writer(nsqd_tcp_addresses=['127.0.0.1:4150'],
**self.identify_options)
def trypub():
w.mpub(topic, b'{"one": 1}', callback=pubcb)
def pubcb(conn, data):
if isinstance(data, nsq.protocol.Error):
if 'no open connections' in str(data):
self.io_loop.call_later(0.1, trypub)
return
self.stop(data)
self.io_loop.call_later(0.1, trypub)
result = self.wait()
print(str(result))
assert not isinstance(result, Exception)
def crawl(board, page_limit=10, retry=10, retry_timeout=1, topic='ptt_crawler', nsqd='127.0.0.1:4150', verbose=False):
'''
options:
--page-limit=<int>
--retry=<int>
--retry-timeout=<float>
--topic=<str>
--nsqd=<str>
-v, --verbose
'''
if verbose:
setup_logger()
tornado.platform.asyncio.AsyncIOMainLoop().install()
fetcher = Fetcher(retry=retry, retry_timeout=retry_timeout)
board = Board(board, fetcher)
writer = nsq.Writer(nsqd.split(','))
producer = Producer(board, writer, topic)
time.sleep(1)
loop = asyncio.get_event_loop()
loop.run_until_complete(producer.run(limit=page_limit))
import nsq
import tornado.ioloop
import time
def pub_message():
writer.pub('test', (time.strftime('%H:%M:%S') + " Hello").encode(),
finish_pub)
def finish_pub(conn, data):
print(data)
writer = nsq.Writer(['172.31.118.243:4150'])
tornado.ioloop.PeriodicCallback(pub_message, 1000).start()
nsq.run()
i = 0
def pub_message():
global i
i += 1
writer.pub(
topic='test',
msg=bytes(json.dumps(
dict(
type="add",
uuid=str(uuid.uuid4()),
num_a=random.choice(range(100)),
num_b=random.choice(range(100)),
)),
encoding="utf-8"),
callback=finish_pub,
)
if i == 10:
sys.exit(0)
def finish_pub(conn, data):
print(conn, data)
writer = nsq.Writer(nsqd_tcp_addresses=['127.0.0.1:4162'])
tornado.ioloop.PeriodicCallback(pub_message, 1000).start()
nsq.run()
'", "msgType" : "response", "queryType" : "", "content" : { "success" : "true", "error" : ""}}'
)
except errors.HttpError as err:
writer.pub(
reqObj.src,
'{"src" : "Notification", "dest" : "' + reqObj.src +
'", "msgType" : "response", "queryType" : "", "content" : { "success" : "false", "error" : "Error: '
+ err + '"}}')
return
def logNotification(self, notificationRequest):
messageLog.append(notificationRequest)
print(messageLog[0].userId)
return
try:
writer = nsq.Writer('127.0.0.1', 4150)
requestServer = nsq.Reader(
message_handler=NotificationRequest.sendEmail(),
lookupd_http_addresses=['http://127.0.0.1:4161'],
topic='Notification',
channel='NavUP',
lookupd_poll_interval=15)
nsq.run()
except:
print(
"NSQ Error! Notifications Module generated a critical server error. Notificaitons module likely offline."
)
# -*- coding: utf8 -*-
"""
imdemo.common.mq.producer
~~~~~~~~~~~~~~~~~~~~~~~~~
Message Producer.
"""
import nsq
from imdemo.common.const import NsqConfig
# TODO Should be lasy.
nsq_producer = nsq.Writer([NsqConfig.NSQD_ADDREESS])
import nsq
import pickle
from celery.exceptions import TimeoutError
from models import db, Tasks, Results
from get_python_from_html import get_python_from_html
def run_queue():
for task in Tasks.query.filter_by(task_status='NOT_STARTED'):
queue = get_python_from_html.delay(task.address)
task.task_status = 'PENDING'
db.session.add(task)
db.session.commit()
try:
result = {'task_id': task.id, 'result': queue.get(timeout=10)}
writer.pub('python', pickle.dumps(result))
print(queue.status, task.address)
except:
pass
task.task_status = 'FINISHED'
db.session.add(task)
db.session.commit()
if __name__ == '__main__':
writer = nsq.Writer(['dz-e8-nsqd:4150'])
tornado.ioloop.PeriodicCallback(run_queue, 10000).start()
nsq.run()
import nsq
import tornado.ioloop
import time
def pub_message():
writer.pub('test', time.strftime('%H:%M:%S'), finish_pub)
def finish_pub(conn, data):
print(data)
writer = nsq.Writer(['127.0.0.1:4150'])
tornado.ioloop.PeriodicCallback(pub_message, 10).start()
nsq.run()
)
except errors.HttpError as err:
writer.pub(
reqObj['src'], '{"src" : "Notification", "dest" : '
"'+reqObj['src']+'"
', "msgType" : "response", "queryType" : "", "content" : { "success" : "false", "error" : "Error: '
+ err + '"}}')
return True
def logNotification(self, notificationRequest):
"""
append notification request received to array
Keyword arguments:
request -- JSON object with message, notification type and recepient's email address
"""
messageLog.append(notificationRequest)
print(messageLog[0].userId)
return
writer = nsq.Writer('127.0.0.1:4150')
nr = NotificationRequest()
requestServer = nsq.Reader(message_handler=nr.sendEmail,
lookupd_http_addresses=['http://127.0.0.1:4161'],
topic='notification',
channel='navup',
lookupd_poll_interval=5)
nsq.run()
import nsq
import os
import time
import tornado.ioloop
import json
import logging
# Write message every second to a topic
TOPIC = os.environ.get('topic', 'topic')
NSQD = os.environ.get('nsqd', '127.0.0.1:4150')
LOGGER_LEVEL = logging.getLevelName(os.environ.get('LOGGER_LEVEL', 'WARN'))
logging.getLogger('nsq').setLevel(LOGGER_LEVEL)
def pub_message():
msg = {'time': time.strftime('%H:%M:%S')}
logging.info('publishing a message')
writer.pub(TOPIC, json.dumps(msg), finish_pub)
def finish_pub(conn, data):
logging.info(data)
logging.info(conn)
writer = nsq.Writer([NSQD])
tornado.ioloop.PeriodicCallback(pub_message, 1000).start()
nsq.run()
return write_message(data, writer)
else:
print("Published Message: ", data)
def calculate_prime(message, writer):
data = json.loads(message.body)
prime = is_prime(data["number"])
data["prime"] = prime
if prime:
topic = "prime"
else:
topic = "non_prime"
output_message = json.dumps(data).encode("utf8")
write_message(topic, output_message, writer)
message.finish() # <1>
if __name__ == "__main__":
writer = nsq.Writer(["127.0.0.1:4150"])
handler = partial(calculate_prime, writer=writer)
reader = nsq.Reader(
message_handler=handler,
nsqd_tcp_addresses=["127.0.0.1:4150"],
topic="numbers",
channel="worker_group_a",
)
nsq.run()
from fastapi import APIRouter, Depends
from pydantic import BaseModel, validator
import nsq
from .util import enforce_authentication, generate_message_id
router = APIRouter()
nsqd_addresses = []
for key, value in environ.items():
if key.startswith('NSQD_ADDR'):
nsqd_addresses.append(value)
nsq_writer = nsq.Writer(nsqd_addresses)
class Message(BaseModel):
text: str
@validator('text')
def text_length(cls, text):
if len(text) < 1 or len(text) > 4096:
raise ValueError('Text length MUST vary in [1, 4096]')
return text
@router.put('/message')
async def put_message(message: Message,
username: str = Depends(enforce_authentication)):
import nsq
import tornado.ioloop
import time
def pub_message():
writer.pub('test', time.strftime('%H:%M:%S').encode(), finish_pub)
def finish_pub(conn, data):
print(data)
writer = nsq.Writer(['localhost:4150'])
tornado.ioloop.PeriodicCallback(pub_message, 1000).start()
nsq.run()
import nsq
import pickle
from celery.exceptions import TimeoutError
from models import db, Tasks, Results
from get_python_from_html import get_python_from_html
def run_queue():
for task in Tasks.query.filter_by(task_status='NOT_STARTED'):
queue = get_python_from_html.delay(task.address)
task.task_status = 'PENDING'
db.session.add(task)
db.session.commit()
try:
result = {'task_id': task.id, 'result': queue.get(timeout=10)}
writer.pub('python', pickle.dumps(result))
print(queue.status, task.address)
except:
pass
task.task_status = 'FINISHED'
db.session.add(task)
db.session.commit()
if __name__ == '__main__':
writer = nsq.Writer(['sf-e8-nsqd:4150'])
tornado.ioloop.PeriodicCallback(run_queue, 10000).start()
nsq.run()
def register_message_writers():
'''为nsq消息系统建立生产者'''
WriterManager.writer = nsq.Writer(['127.0.0.1:4150'])
def __init__(self, data):
self.data = pickle.dumps(data)
self.writer = nsq.Writer([f'{NSQ_HOST_IP_PORT}'])
import tornado.ioloop
import time
import threading
def pub_message():
writer.pub('test', b'test', finish_pub)
def finish_pub(conn, data):
print(data)
# writer = nsq.AsyncConn('127.0.0.1',4150)
writer = nsq.Writer(['192.168.10.251:6000'])
tornado.ioloop.PeriodicCallback(pub_message, 1000).start()
writer.connect()
# async def tt():
# nsq.run()
# def run(coroutine):
# try:
# coroutine.send(None)
# except StopIteration as e:
# return e.value
# run(tt())
def __inititalizeNsqdWriters(self):
# write locally since topic will be polled to nsqlookupd
self.__requestProducerWriter = nsq.Writer(self.requestProducerAddrList)
self.__fastLaneWriter = nsq.Writer(self.fastLaneAddrList)
self.__slowLaneWriter = nsq.Writer(self.slowLaneAddrList)
def create_nsq_writer():
return nsq.Writer(config.NSQD_TCP_ADDRESSES)