class __Server:
def __init__(self):
scheduler = IOLoopScheduler(IOLoop.current())
self._app = Application([
(r'/exchange', ExchangeHandler),
(r'/', MainHandler),
])
self.orders = PriorityQueue()
self.posted_orders = []
self.fulfilled_orders = []
self.messages = Subject()
only_messages = self.messages \
.filter(lambda msg: msg[0] == 'message') \
.map(lambda msg: msg[1].split(',')) \
.publish()
def queue_order(msg):
self.orders.put(Order.from_list(msg))
only_messages \
.filter(lambda msg: msg[0] == 'order') \
.map(lambda msg: msg[1:]) \
.subscribe(queue_order)
def process_order(time):
try:
order = self.orders.get_nowait()
print('processing order: {} [{}]'.format(
order, order.timestamp))
matching = None
for posted in self.posted_orders:
if posted.matches(order):
matching = posted
break
if matching is None:
self.posted_orders.append(order)
print('could not find match, posted order count is {}'.
format(len(self.posted_orders)))
else:
self.posted_orders.remove(posted)
self.fulfilled_orders.append(posted)
self.fulfilled_orders.append(order)
print('order fulfilled: {}'.format(order))
print('fulfilled by: {}'.format(posted))
except QueueEmpty:
pass
Observable.interval(100,
scheduler=scheduler).subscribe(process_order)
only_messages.connect()
def start(self):
self._app.listen(8888)
class Client(object):
def __init__(self, register, name=None):
self._queue = PriorityQueue(maxsize=options.queries_number)
self.queries_number = options.queries_number
self.stop_time = datetime.now() + timedelta(seconds=options.test_time)
self.register = register
self.__count = 0
self.__processed = 0
if register is not None:
register.add(self)
if name is None:
self.name = uuid4()
else:
self.name = name
io_loop = IOLoop.instance()
io_loop.spawn_callback(self.consume)
io_loop.spawn_callback(self.start)
@gen.coroutine
def start(self):
yield self.produce()
yield self._queue.join()
@gen.coroutine
def exit(self):
logger.debug('client {} exiting'.format(self.name))
self.register.remove(self)
if not self.register:
IOLoop.instance().stop()
logger.debug('all clients exited')
else:
logger.debug('{} clients left'.format(len(self.register)))
@gen.coroutine
def produce(self):
logger.debug('producer {} start'.format(self.name))
while True:
if self.__count >= self.queries_number or datetime.now() > self.stop_time:
break
p = randint(0, 100)
res = 'r'
if float(p) / 100. <= options.rw_ratio:
res = 'w'
yield self._queue.put((randint(0, 10), res))
self.__count += 1
logger.debug('producer {} finished'.format(self.name))
self.stop_time = datetime.now() + timedelta(seconds=options.test_time)
@gen.coroutine
def consume(self):
logger.debug('consumer {} start'.format(self.name))
while True:
if datetime.now() > self.stop_time or self.__processed >= self.queries_number:
logger.debug('{}'.format(datetime.now() > self.stop_time))
break
pr, tp = yield self._queue.get()
w = Worker(self.name)
# if tp == 'r':
# yield w.read_worker()
# elif tp == 'w':
# yield w.write_worker()
yield gen.sleep(0.00000) # hack
if tp == 'w':
yield w.write_worker()
elif tp == 'r':
yield w.read_worker()
self.__processed += 1
self._queue.task_done()
logger.debug('consumer {} finished'.format(self.name))
yield gen.sleep(0.00000) # hack
yield self.exit()
from tornado.queues import PriorityQueue q = PriorityQueue() q.put((1, 'medium-priority item')) q.put((0, 'high-priority item')) q.put((10, 'low-priority item')) print(q.get_nowait()) print(q.get_nowait()) print(q.get_nowait())
#!urs/bin/env python #coding:utf-8 # PriorityQueue：一个又优先级的Queue最小的最最优先。 # 写入条目通常是元组，类似（proority number, data）。 from tornado.queues import PriorityQueue q = PriorityQueue() q.put((1, 'medium-priority item')) q.put((0, 'high-priority item')) q.put((10, 'low-priority item')) print(q.get_nowait()) print(q.get_nowait()) print(q.get_nowait())
class Worker(object):
"""
Description
-----------
Worker for stream proccessing
"""
stream = None
address = None
is_authed = None
name = None
id = None
_queue = None
def __init__(self, stream, address, server=None):
self.stream = stream
self.address = address
self.is_authed = False
self.last_message = None
self.server = server
self.id = hash(datetime.now())
self._queue = PriorityQueue()
self._queue.join()
logger.debug('Worker for {} is initiated'.format(address))
def run(self):
self.stream.set_close_callback(lambda: self.on_close(hard=True))
self._read_line()
def on_close(self, hard=False):
self._queue.put((300, self.sockets_broadcast('closed', self.name)))
self.is_authed = False
self.name = None
if self.stream and \
self.stream.closed() and \
self in self.server._workers and \
hard:
self.server._workers.remove(self)
logger.debug("worker for {} is closed".format(self.address))
def on_auth(self, message):
self.is_authed = True
self.name = message.value
self._queue.put((1, self.sockets_broadcast('opened', self.name)))
@gen.coroutine
def sockets_broadcast(self, label, message):
if self.server and self.is_authed:
msg = {label: message, 'worker': self.name, 'id': self.id}
msg = unicode(json.dumps(msg))
for socket in self.server.sockets:
socket.write_message(msg)
@gen.coroutine
def _read_line(self):
self.stream.read_until(helpers.ENDING, self._handle_read)
@gen.coroutine
def _handle_read(self, data_):
if not self.stream.closed():
data = data_.rstrip()
message = helpers.Message(data)
logger.debug('[{}][{}][{}]'.format(self.address, unicode(message), data))
if message.type == helpers.Message.MESSAGE and self.is_authed:
self.last_message = message
self._queue.put((100, self.sockets_broadcast('message', unicode(message))))
if not self.is_authed:
if message.type == helpers.Message.AUTH:
self.on_auth(message)
else:
if message.type == helpers.Message.END:
self.on_close()
self._read_line()
class Client(object):
name = None
conn = None
max_count = None
count = None
dt = None
_event_queue = None
def __init__(self, name=None, max_count=10, dt=0.5, client_set=None):
global clients_set
self.name = name or hash(datetime.now())
self.max_count = max_count
self.dt = dt
self.authorized = False
self.count = 0
if client_set is None:
self.client_set = set()
self.client_set.add(self)
else:
self.client_set = client_set
self._event_queue = PriorityQueue()
self._event_queue.join()
logger.debug('client start {}'.format(self.name))
IOLoop.current().spawn_callback(self.init)
IOLoop.current().spawn_callback(self.start_produce)
@gen.coroutine
def init(self):
logger.debug('client init {}'.format(self.name))
self.conn = yield TCPClient().connect(
config.options.host,
config.options.port_tcp
)
s = self.create_message(helpers.Message.AUTH)
self.conn.write(s)
@gen.coroutine
def exit(self, force=False):
logger.debug('client exit {}'.format(self.name))
if self.conn and not self.conn.closed():
s = self.create_message(helpers.Message.END)
priority = 300 if not force else 0
self._event_queue.put((priority, self.conn.write(s)))
self._event_queue.put((priority, self.conn.close()))
if self in self.client_set:
self.client_set.remove(self)
if not self.client_set:
IOLoop.instance().stop()
print len(self.client_set)
@gen.coroutine
def start_produce(self):
logger.debug('client producing {}'.format(self.name))
while True:
yield gen.sleep(self.dt)
if not self.conn.closed():
value = self.count
key = randint(1, 1000)
s = self.create_message(key=key, value=value)
yield self._event_queue.put((100, self.conn.write(s)))
self.count += 1
if self.count >= self.max_count:
break
yield self.exit()
logger.debug('client stopped {}'.format(self.name))
def create_message(self, type_=helpers.Message.MESSAGE, key=None, value=None):
res = None
if type_ == helpers.Message.AUTH:
res = helpers.get_auth_str(self.name)
elif type_ == helpers.Message.END:
res = helpers.get_end_str()
elif type_ == helpers.Message.MESSAGE:
res = helpers.format_string(key, value)
return bytes(res)