class TestIOLoopRunSync(unittest.TestCase):
def setUp(self):
self.io_loop = IOLoop()
def tearDown(self):
self.io_loop.close()
def test_sync_result(self):
with self.assertRaises(gen.BadYieldError):
self.io_loop.run_sync(lambda: 42)
def test_sync_exception(self):
with self.assertRaises(ZeroDivisionError):
self.io_loop.run_sync(lambda: 1 / 0)
def test_async_result(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_callback)
raise gen.Return(42)
self.assertEqual(self.io_loop.run_sync(f), 42)
def test_async_exception(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_callback)
1 / 0
with self.assertRaises(ZeroDivisionError):
self.io_loop.run_sync(f)
def test_current(self):
def f():
self.assertIs(IOLoop.current(), self.io_loop)
self.io_loop.run_sync(f)
def test_timeout(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_timeout, self.io_loop.time() + 1)
self.assertRaises(TimeoutError, self.io_loop.run_sync, f, timeout=0.01)
@skipBefore35
def test_native_coroutine(self):
namespace = exec_test(
globals(), locals(), """
async def f():
await gen.Task(self.io_loop.add_callback)
""")
self.io_loop.run_sync(namespace['f'])
class TestIOLoopRunSync(unittest.TestCase):
def setUp(self):
self.io_loop = IOLoop()
def tearDown(self):
self.io_loop.close()
def test_sync_result(self):
with self.assertRaises(gen.BadYieldError):
self.io_loop.run_sync(lambda: 42)
def test_sync_exception(self):
with self.assertRaises(ZeroDivisionError):
self.io_loop.run_sync(lambda: 1 / 0)
def test_async_result(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_callback)
raise gen.Return(42)
self.assertEqual(self.io_loop.run_sync(f), 42)
def test_async_exception(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_callback)
1 / 0
with self.assertRaises(ZeroDivisionError):
self.io_loop.run_sync(f)
def test_current(self):
def f():
self.assertIs(IOLoop.current(), self.io_loop)
self.io_loop.run_sync(f)
def test_timeout(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_timeout, self.io_loop.time() + 1)
self.assertRaises(TimeoutError, self.io_loop.run_sync, f, timeout=0.01)
@skipBefore35
def test_native_coroutine(self):
namespace = exec_test(globals(), locals(), """
async def f():
await gen.Task(self.io_loop.add_callback)
""")
self.io_loop.run_sync(namespace['f'])
class IOLoop(object):
NONE = TornadoIOLoop.NONE
READ = TornadoIOLoop.READ
WRITE = TornadoIOLoop.WRITE
ERROR = TornadoIOLoop.ERROR
def __init__(self):
self._tornado_io_loop = TornadoIOLoop()
def inner(self):
return self._tornado_io_loop
def close(self, all_fds=False):
self._tornado_io_loop.close(all_fds)
def add_handler(self, fd, handler, events):
self._tornado_io_loop.add_handler(fd, handler, events)
def update_handler(self, fd, events):
self._tornado_io_loop.update_handler(fd, events)
def remove_handler(self, fd):
self._tornado_io_loop.remove_handler(fd)
def start(self):
self._tornado_io_loop.start()
def stop(self):
self._tornado_io_loop.stop()
def time(self):
return self._tornado_io_loop.time()
def add_timeout(self, deadline, callback):
return self._tornado_io_loop.add_timeout(deadline, callback)
def remove_timeout(self, timeout):
self._tornado_io_loop.remove_timeout(timeout)
def add_callback(self, callback, *args, **kwargs):
self._tornado_io_loop.add_callback(callback, *args, **kwargs)
def run(self):
try:
self.start()
except KeyboardInterrupt:
print ""
print "Ctrl-C recieved. Exiting."
class TestIOLoopRunSync(unittest.TestCase):
def setUp(self):
self.io_loop = IOLoop()
def tearDown(self):
self.io_loop.close()
def test_sync_result(self):
self.assertEqual(self.io_loop.run_sync(lambda: 42), 42)
def test_sync_exception(self):
with self.assertRaises(ZeroDivisionError):
self.io_loop.run_sync(lambda: 1 / 0)
def test_async_result(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_callback)
raise gen.Return(42)
self.assertEqual(self.io_loop.run_sync(f), 42)
def test_async_exception(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_callback)
1 / 0
with self.assertRaises(ZeroDivisionError):
self.io_loop.run_sync(f)
def test_current(self):
def f():
self.assertIs(IOLoop.current(), self.io_loop)
self.io_loop.run_sync(f)
def test_timeout(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_timeout, self.io_loop.time() + 1)
self.assertRaises(TimeoutError, self.io_loop.run_sync, f, timeout=0.01)
class TestIOLoopRunSync(unittest.TestCase):
def setUp(self):
self.io_loop = IOLoop()
def tearDown(self):
self.io_loop.close()
def test_sync_result(self):
self.assertEqual(self.io_loop.run_sync(lambda: 42), 42)
def test_sync_exception(self):
with self.assertRaises(ZeroDivisionError):
self.io_loop.run_sync(lambda: 1 / 0)
def test_async_result(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_callback)
raise gen.Return(42)
self.assertEqual(self.io_loop.run_sync(f), 42)
def test_async_exception(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_callback)
1 / 0
with self.assertRaises(ZeroDivisionError):
self.io_loop.run_sync(f)
def test_current(self):
def f():
self.assertIs(IOLoop.current(), self.io_loop)
self.io_loop.run_sync(f)
def test_timeout(self):
@gen.coroutine
def f():
yield gen.Task(self.io_loop.add_timeout, self.io_loop.time() + 1)
self.assertRaises(TimeoutError, self.io_loop.run_sync, f, timeout=0.01)
def _on_connect(self):
try:
while True:
refuse_connect_flag = False
data = yield self.read_message()
send_data = {}
# register
if "command" in data and data["command"] == Command.register:
self.info = data["data"]
if self.info["http_host"] == "0.0.0.0":
self.info["http_host"] = self._address[0]
# no node_id
if "node_id" not in self.info or self.info["node_id"] == None:
node_id = str(uuid4())
send_data = {
"command": Command.register,
"data": {
"status": Status.success,
"message": Status.success,
"node_id": node_id,
}
}
self.info["node_id"] = node_id
self._status = Status.registered
if self.info["node_id"] not in Connection.clients_dict:
Connection.clients_dict[self.info["node_id"]] = self
# register with node_id
else:
if self.info["node_id"] not in Connection.clients_dict:
Connection.clients_dict[self.info["node_id"]] = self
send_data = {
"command": Command.register,
"data": {
"status": Status.success,
"message": Status.success,
"node_id": self.info["node_id"],
}
}
self._status = Status.registered
if "action_slots" in self.info:
Connection.total_action_slots += self.info["action_slots"]
elif "command" in data and data["command"] == Command.heartbeat:
self.info = data["data"]
if self.info["http_host"] == "0.0.0.0":
self.info["http_host"] = self._address[0]
if self._status == Status.registered:
send_data = {
"command": Command.heartbeat,
"data": {
"status": Status.success,
"message": Status.success,
}
}
if self._heartbeat_timeout:
IOLoop.instance().remove_timeout(self._heartbeat_timeout)
self._heartbeat_timeout = IOLoop.instance().add_timeout(
IOLoop.time(IOLoop.instance()) + self.info["heartbeat_timeout"],
self._remove_connection
)
else:
send_data = {
"command": Command.heartbeat,
"data": {
"status": Status.failure,
"message": "invalid node_id: %s" % self.info["node_id"],
}
}
refuse_connect_flag = True
else:
LOG.error("Client(%s) invaild message error: %s", self._address, data)
send_data = {
"command": Command.error,
"data": {
"status": Status.failure,
"message": "Unknown Command!",
}
}
self.send_message(send_data, refuse_connect_flag = refuse_connect_flag)
except tornado.iostream.StreamClosedError:
LOG.info("Closed: %s", self._address)
except Exception as e:
LOG.exception(e)
class IOLoop(threading.Thread):
_futures = []
loop_interval = 100 # ms
loop_quit_wait = MAX_WAIT_SECONDS_BEFORE_SHUTDOWN # second
def __init__(self):
threading.Thread.__init__(self)
self.ioloop = TornadoIOLoop()
def run(self):
logger.debug('ioloop starting')
def add_features():
if not self._futures:
pass
else:
need_add = self._futures[:]
self._futures = []
for each in need_add:
self.ioloop.add_future(each[0], each[1])
PeriodicCallback(add_features, self.loop_interval, self.ioloop).start()
self.ioloop.start()
def add_future(self, future, callback=None):
def nothing(future, **kwargs):
pass
if callback is None:
callback = nothing
# else:
# feature.add_done_callback(callback)
self._futures.append((future, callback))
def add_periodic(self, feature, interval=1000):
if self.ioloop._timeouts is None:
self.ioloop._timeouts = []
PeriodicCallback(feature, interval, self.ioloop).start()
def add_timeout(self, deadline, callback, *args, **kwargs):
self.ioloop.add_timeout(deadline, callback, *args, **kwargs)
def time(self):
return self.ioloop.time()
def quit(self):
logger.info('begin to quit')
self.ioloop.add_callback(self._quit)
def _quit(self):
"""
:return:
"""
logger.info('Will shutdown in %s seconds ...',
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)
io_loop = self.ioloop
deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN
def stop_loop():
"""
:return:
"""
now = time.time()
step = 0.01
if now < deadline and (io_loop._callbacks
or len(io_loop._timeouts) > 1):
io_loop.add_timeout(max(now + step, deadline), stop_loop)
else:
io_loop.stop()
io_loop.close()
io_loop.clear_current()
io_loop.clear_instance()
logger.info('Shutdown')
stop_loop()
def getServerTime():
return IOLoop.time()
def getServerTime():
return IOLoop.time()
class SwimpyProcess(multiprocessing.Process):
def __init__(self, routes, node, pipe, *args, **kwargs):
super(self.__class__, self).__init__(*args, **kwargs)
self.routes = routes
self.node = node
self.pipe = pipe
self.bind_addr = node.addr
self.bind_port = node.port
# We don't want to initialize these until after we fork
self.ioloop = None
self.server = None
self.app = None
def _handle_pipe_messages(self, *args, **kwargs):
message = self.pipe.recv()
unpacked_message = msgpack.unpackb(message)
message_type = unpacked_message.pop('type')
if message_type == Sync.MESSAGE_TYPE:
self.pipe.send(self.app.nodes)
def run(self):
signal.signal(signal.SIGTERM, self.shutdown_sig_handler)
signal.signal(signal.SIGINT, self.shutdown_sig_handler)
from tornado.ioloop import IOLoop
self.ioloop = IOLoop().current() # get a reference to the IOLoop post-fork
LOGGER.info('Starting server on tcp://{}:{}'.format(self.bind_addr, self.bind_port))
self.app = Application(routes=self.routes, node=self.node, pipe=self.pipe)
self.server = Server(message_handler=self.app.route_stream_message)
self.server.listen(self.bind_port, address=self.bind_addr)
self.ioloop.add_handler(self.pipe, self._handle_pipe_messages, self.ioloop.READ)
self.ioloop.spawn_callback(self.app.send_buffered_gossip)
# Ping a random node every PING_INVERVAL seconds, +/- 10%
# This jitter should reduce the average aggregate peak network throughput
# when running with large cluster sized
PeriodicCallbackWithSplay(self.app.ping_random_node,
PING_INTERVAL * 1000,
splay_pct=10).start()
LOGGER.info('Starting ioloop')
self.ioloop.start()
def stop(self, shutdown_timeout=SHUTDOWN_TIMEOUT, graceful=True):
"""
Trigger a graceful stop of the server and the server's ioloop, allowing in-flight
connections to finish
Fall back to a "less graceful" stop when stop is reached
"""
def poll_stop():
# Tornado uses a "waker" handler internally that we'd like to ignore here
remaining_handlers = {
k: v
for k, v in self.ioloop._handlers.iteritems()
if k != self.ioloop._waker.fileno()
}
# Poll for any remaining connections
remaining_count = len(remaining_handlers)
# Once all handlers have been removed, we can stop safely
if remaining_count == 0:
LOGGER.info('All finished! Graceful stop complete')
self.ioloop.stop()
else:
LOGGER.info('Waiting on IO handlers ({} remaining). '
'Handlers: {!r}'.format(remaining_count,
remaining_handlers))
self.ioloop.remove_handler(self.pipe)
self.server.shutdown()
self.server.stop()
if graceful:
# Poll the IOLoop's handlers until they all shut down.
poller = PeriodicCallback(poll_stop, 500, io_loop=self.ioloop)
poller.start()
self.ioloop.add_timeout(self.ioloop.time() + shutdown_timeout,
self.ioloop.stop)
else:
self.ioloop.stop()
def shutdown_sig_handler(self, sig, frame):
"""
Signal handler for "stop" signals (SIGTERM, SIGINT)
"""
LOGGER.warning('{!r} caught signal {!r}! Shutting down'.format(self, sig))
try:
self.ioloop.add_callback_from_signal(self.stop)
except Exception as e:
LOGGER.error(
'Encountered exception while shutting down: {}'.format(e)
)
sys.exit(1)
