def test_close_file_object(self):
"""When a file object is used instead of a numeric file descriptor,
the object should be closed (by IOLoop.close(all_fds=True),
not just the fd.
"""
# Use a socket since they are supported by IOLoop on all platforms.
# Unfortunately, sockets don't support the .closed attribute for
# inspecting their close status, so we must use a wrapper.
class SocketWrapper(object):
def __init__(self, sockobj):
self.sockobj = sockobj
self.closed = False
def fileno(self):
return self.sockobj.fileno()
def close(self):
self.closed = True
self.sockobj.close()
sockobj, port = bind_unused_port()
socket_wrapper = SocketWrapper(sockobj)
io_loop = IOLoop()
io_loop.add_handler(socket_wrapper, lambda fd, events: None,
IOLoop.READ)
io_loop.close(all_fds=True)
self.assertTrue(socket_wrapper.closed)
def test_close_file_object(self):
"""When a file object is used instead of a numeric file descriptor,
the object should be closed (by IOLoop.close(all_fds=True),
not just the fd.
"""
# Use a socket since they are supported by IOLoop on all platforms.
# Unfortunately, sockets don't support the .closed attribute for
# inspecting their close status, so we must use a wrapper.
class SocketWrapper(object):
def __init__(self, sockobj):
self.sockobj = sockobj
self.closed = False
def fileno(self):
return self.sockobj.fileno()
def close(self):
self.closed = True
self.sockobj.close()
sockobj, port = bind_unused_port()
socket_wrapper = SocketWrapper(sockobj)
io_loop = IOLoop()
io_loop.add_handler(socket_wrapper, lambda fd, events: None,
IOLoop.READ)
io_loop.close(all_fds=True)
self.assertTrue(socket_wrapper.closed)
class IOLoop(object):
NONE = TornadoIOLoop.NONE
READ = TornadoIOLoop.READ
WRITE = TornadoIOLoop.WRITE
ERROR = TornadoIOLoop.ERROR
def __init__(self):
self._tornado_io_loop = TornadoIOLoop()
def inner(self):
return self._tornado_io_loop
def close(self, all_fds=False):
self._tornado_io_loop.close(all_fds)
def add_handler(self, fd, handler, events):
self._tornado_io_loop.add_handler(fd, handler, events)
def update_handler(self, fd, events):
self._tornado_io_loop.update_handler(fd, events)
def remove_handler(self, fd):
self._tornado_io_loop.remove_handler(fd)
def start(self):
self._tornado_io_loop.start()
def stop(self):
self._tornado_io_loop.stop()
def time(self):
return self._tornado_io_loop.time()
def add_timeout(self, deadline, callback):
return self._tornado_io_loop.add_timeout(deadline, callback)
def remove_timeout(self, timeout):
self._tornado_io_loop.remove_timeout(timeout)
def add_callback(self, callback, *args, **kwargs):
self._tornado_io_loop.add_callback(callback, *args, **kwargs)
def run(self):
try:
self.start()
except KeyboardInterrupt:
print ""
print "Ctrl-C recieved. Exiting."
def run_socket(datasource):
io_loop = IOLoop()
io_loop.install()
host = pop_clean()
if host is None:
log("[E]",
'''No clean host found. Use "--seed default.txt" to add more.''')
return
log("[I]", "Connecting to (%s:%d)" % host)
with SQLiteControlledExecution(DB_PATH) as c:
c.execute('INSERT INTO history VALUES (?,?,?,?,?,?,?,?)',
(None, "", "", "CONNECTING", None, None, host[0], host[1]))
c.execute('SELECT id FROM history WHERE ROWID = ?', (c.lastrowid, ))
row_id = c.fetchone()[0]
received = StringIO()
sent = StringIO()
def update_connected():
c.execute('''UPDATE history SET status = ?, connected_at = ?
WHERE id = ?''',
("CONNECTED", datetime.utcnow(), row_id, ))
def update_status(status):
c.execute('UPDATE history SET status = ? WHERE id = ?',
(status, row_id, ))
def update_final_status(status):
c.execute('''UPDATE history SET disconnected_at = ?, status = ?,
sent = ?, received = ?
WHERE id = ?''',
(datetime.utcnow(), status, sent.getvalue(),
received.getvalue(), row_id, ))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(host)
except socket.error as e:
if e.errno == errno.ECONNRESET:
update_final_status("RESET")
else:
raise
s.setblocking(0)
set_nonblocking(datasource)
log("[I]", "Connected.")
update_connected()
def read_till_block(fd, step=1):
data = ""
while True:
try:
tmp = os.read(fd, step)
if not tmp:
return data if data else None
data += tmp
except OSError as e:
if e.args[0] == errno.EWOULDBLOCK:
break
else:
raise
return data
@engine
def stop():
yield Task(io_loop.add_timeout, timedelta(milliseconds=STOP_DELAY))
io_loop.stop()
def process_socket_errno(no):
log("[E]", errno.errorcode[no] + ":", os.strerror(no))
if no == errno.ECONNRESET:
update_final_status("RESET")
else:
update_final_status(errno.errorcode[no])
io_loop.remove_handler(s.fileno())
io_loop.remove_handler(datasource.fileno())
stop()
def on_socket_events(fd, events):
if events & io_loop.READ:
try:
data = read_till_block(fd)
except OSError as e:
process_socket_errno(e.args[0])
return
if data is not None:
with ExceptionIgnored():
sys.stdout.write(data)
received.write(data)
else:
log("[I]", "Socket closed.")
update_final_status("DISCONNECTED")
io_loop.remove_handler(s.fileno())
io_loop.remove_handler(datasource.fileno())
stop()
if events & io_loop.ERROR:
no = s.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
process_socket_errno(no)
return
def on_file_events(fd, events):
if events & io_loop.READ:
data = read_till_block(fd)
if data is not None:
s.send(data)
sent.write(data)
else:
log("[I]", "Input file closed.")
io_loop.remove_handler(fd)
io_loop.add_handler(s.fileno(), on_socket_events, io_loop.READ)
io_loop.add_handler(datasource.fileno(), on_file_events, io_loop.READ)
try:
io_loop.start()
except KeyboardInterrupt:
s.close()
log("[I]", "Disconnected.")
update_final_status("DISCONNECTED")
stop()
class SwimpyProcess(multiprocessing.Process):
def __init__(self, routes, node, pipe, *args, **kwargs):
super(self.__class__, self).__init__(*args, **kwargs)
self.routes = routes
self.node = node
self.pipe = pipe
self.bind_addr = node.addr
self.bind_port = node.port
# We don't want to initialize these until after we fork
self.ioloop = None
self.server = None
self.app = None
def _handle_pipe_messages(self, *args, **kwargs):
message = self.pipe.recv()
unpacked_message = msgpack.unpackb(message)
message_type = unpacked_message.pop('type')
if message_type == Sync.MESSAGE_TYPE:
self.pipe.send(self.app.nodes)
def run(self):
signal.signal(signal.SIGTERM, self.shutdown_sig_handler)
signal.signal(signal.SIGINT, self.shutdown_sig_handler)
from tornado.ioloop import IOLoop
self.ioloop = IOLoop().current() # get a reference to the IOLoop post-fork
LOGGER.info('Starting server on tcp://{}:{}'.format(self.bind_addr, self.bind_port))
self.app = Application(routes=self.routes, node=self.node, pipe=self.pipe)
self.server = Server(message_handler=self.app.route_stream_message)
self.server.listen(self.bind_port, address=self.bind_addr)
self.ioloop.add_handler(self.pipe, self._handle_pipe_messages, self.ioloop.READ)
self.ioloop.spawn_callback(self.app.send_buffered_gossip)
# Ping a random node every PING_INVERVAL seconds, +/- 10%
# This jitter should reduce the average aggregate peak network throughput
# when running with large cluster sized
PeriodicCallbackWithSplay(self.app.ping_random_node,
PING_INTERVAL * 1000,
splay_pct=10).start()
LOGGER.info('Starting ioloop')
self.ioloop.start()
def stop(self, shutdown_timeout=SHUTDOWN_TIMEOUT, graceful=True):
"""
Trigger a graceful stop of the server and the server's ioloop, allowing in-flight
connections to finish
Fall back to a "less graceful" stop when stop is reached
"""
def poll_stop():
# Tornado uses a "waker" handler internally that we'd like to ignore here
remaining_handlers = {
k: v
for k, v in self.ioloop._handlers.iteritems()
if k != self.ioloop._waker.fileno()
}
# Poll for any remaining connections
remaining_count = len(remaining_handlers)
# Once all handlers have been removed, we can stop safely
if remaining_count == 0:
LOGGER.info('All finished! Graceful stop complete')
self.ioloop.stop()
else:
LOGGER.info('Waiting on IO handlers ({} remaining). '
'Handlers: {!r}'.format(remaining_count,
remaining_handlers))
self.ioloop.remove_handler(self.pipe)
self.server.shutdown()
self.server.stop()
if graceful:
# Poll the IOLoop's handlers until they all shut down.
poller = PeriodicCallback(poll_stop, 500, io_loop=self.ioloop)
poller.start()
self.ioloop.add_timeout(self.ioloop.time() + shutdown_timeout,
self.ioloop.stop)
else:
self.ioloop.stop()
def shutdown_sig_handler(self, sig, frame):
"""
Signal handler for "stop" signals (SIGTERM, SIGINT)
"""
LOGGER.warning('{!r} caught signal {!r}! Shutting down'.format(self, sig))
try:
self.ioloop.add_callback_from_signal(self.stop)
except Exception as e:
LOGGER.error(
'Encountered exception while shutting down: {}'.format(e)
)
sys.exit(1)
