def test_close_file_object(self): """When a file object is used instead of a numeric file descriptor, the object should be closed (by IOLoop.close(all_fds=True), not just the fd. """ # Use a socket since they are supported by IOLoop on all platforms. # Unfortunately, sockets don't support the .closed attribute for # inspecting their close status, so we must use a wrapper. class SocketWrapper(object): def __init__(self, sockobj): self.sockobj = sockobj self.closed = False def fileno(self): return self.sockobj.fileno() def close(self): self.closed = True self.sockobj.close() sockobj, port = bind_unused_port() socket_wrapper = SocketWrapper(sockobj) io_loop = IOLoop() io_loop.add_handler(socket_wrapper, lambda fd, events: None, IOLoop.READ) io_loop.close(all_fds=True) self.assertTrue(socket_wrapper.closed)
class IOLoop(object): NONE = TornadoIOLoop.NONE READ = TornadoIOLoop.READ WRITE = TornadoIOLoop.WRITE ERROR = TornadoIOLoop.ERROR def __init__(self): self._tornado_io_loop = TornadoIOLoop() def inner(self): return self._tornado_io_loop def close(self, all_fds=False): self._tornado_io_loop.close(all_fds) def add_handler(self, fd, handler, events): self._tornado_io_loop.add_handler(fd, handler, events) def update_handler(self, fd, events): self._tornado_io_loop.update_handler(fd, events) def remove_handler(self, fd): self._tornado_io_loop.remove_handler(fd) def start(self): self._tornado_io_loop.start() def stop(self): self._tornado_io_loop.stop() def time(self): return self._tornado_io_loop.time() def add_timeout(self, deadline, callback): return self._tornado_io_loop.add_timeout(deadline, callback) def remove_timeout(self, timeout): self._tornado_io_loop.remove_timeout(timeout) def add_callback(self, callback, *args, **kwargs): self._tornado_io_loop.add_callback(callback, *args, **kwargs) def run(self): try: self.start() except KeyboardInterrupt: print "" print "Ctrl-C recieved. Exiting."
def run_socket(datasource): io_loop = IOLoop() io_loop.install() host = pop_clean() if host is None: log("[E]", '''No clean host found. Use "--seed default.txt" to add more.''') return log("[I]", "Connecting to (%s:%d)" % host) with SQLiteControlledExecution(DB_PATH) as c: c.execute('INSERT INTO history VALUES (?,?,?,?,?,?,?,?)', (None, "", "", "CONNECTING", None, None, host[0], host[1])) c.execute('SELECT id FROM history WHERE ROWID = ?', (c.lastrowid, )) row_id = c.fetchone()[0] received = StringIO() sent = StringIO() def update_connected(): c.execute('''UPDATE history SET status = ?, connected_at = ? WHERE id = ?''', ("CONNECTED", datetime.utcnow(), row_id, )) def update_status(status): c.execute('UPDATE history SET status = ? WHERE id = ?', (status, row_id, )) def update_final_status(status): c.execute('''UPDATE history SET disconnected_at = ?, status = ?, sent = ?, received = ? WHERE id = ?''', (datetime.utcnow(), status, sent.getvalue(), received.getvalue(), row_id, )) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect(host) except socket.error as e: if e.errno == errno.ECONNRESET: update_final_status("RESET") else: raise s.setblocking(0) set_nonblocking(datasource) log("[I]", "Connected.") update_connected() def read_till_block(fd, step=1): data = "" while True: try: tmp = os.read(fd, step) if not tmp: return data if data else None data += tmp except OSError as e: if e.args[0] == errno.EWOULDBLOCK: break else: raise return data @engine def stop(): yield Task(io_loop.add_timeout, timedelta(milliseconds=STOP_DELAY)) io_loop.stop() def process_socket_errno(no): log("[E]", errno.errorcode[no] + ":", os.strerror(no)) if no == errno.ECONNRESET: update_final_status("RESET") else: update_final_status(errno.errorcode[no]) io_loop.remove_handler(s.fileno()) io_loop.remove_handler(datasource.fileno()) stop() def on_socket_events(fd, events): if events & io_loop.READ: try: data = read_till_block(fd) except OSError as e: process_socket_errno(e.args[0]) return if data is not None: with ExceptionIgnored(): sys.stdout.write(data) received.write(data) else: log("[I]", "Socket closed.") update_final_status("DISCONNECTED") io_loop.remove_handler(s.fileno()) io_loop.remove_handler(datasource.fileno()) stop() if events & io_loop.ERROR: no = s.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) process_socket_errno(no) return def on_file_events(fd, events): if events & io_loop.READ: data = read_till_block(fd) if data is not None: s.send(data) sent.write(data) else: log("[I]", "Input file closed.") io_loop.remove_handler(fd) io_loop.add_handler(s.fileno(), on_socket_events, io_loop.READ) io_loop.add_handler(datasource.fileno(), on_file_events, io_loop.READ) try: io_loop.start() except KeyboardInterrupt: s.close() log("[I]", "Disconnected.") update_final_status("DISCONNECTED") stop()
class SwimpyProcess(multiprocessing.Process): def __init__(self, routes, node, pipe, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) self.routes = routes self.node = node self.pipe = pipe self.bind_addr = node.addr self.bind_port = node.port # We don't want to initialize these until after we fork self.ioloop = None self.server = None self.app = None def _handle_pipe_messages(self, *args, **kwargs): message = self.pipe.recv() unpacked_message = msgpack.unpackb(message) message_type = unpacked_message.pop('type') if message_type == Sync.MESSAGE_TYPE: self.pipe.send(self.app.nodes) def run(self): signal.signal(signal.SIGTERM, self.shutdown_sig_handler) signal.signal(signal.SIGINT, self.shutdown_sig_handler) from tornado.ioloop import IOLoop self.ioloop = IOLoop().current() # get a reference to the IOLoop post-fork LOGGER.info('Starting server on tcp://{}:{}'.format(self.bind_addr, self.bind_port)) self.app = Application(routes=self.routes, node=self.node, pipe=self.pipe) self.server = Server(message_handler=self.app.route_stream_message) self.server.listen(self.bind_port, address=self.bind_addr) self.ioloop.add_handler(self.pipe, self._handle_pipe_messages, self.ioloop.READ) self.ioloop.spawn_callback(self.app.send_buffered_gossip) # Ping a random node every PING_INVERVAL seconds, +/- 10% # This jitter should reduce the average aggregate peak network throughput # when running with large cluster sized PeriodicCallbackWithSplay(self.app.ping_random_node, PING_INTERVAL * 1000, splay_pct=10).start() LOGGER.info('Starting ioloop') self.ioloop.start() def stop(self, shutdown_timeout=SHUTDOWN_TIMEOUT, graceful=True): """ Trigger a graceful stop of the server and the server's ioloop, allowing in-flight connections to finish Fall back to a "less graceful" stop when stop is reached """ def poll_stop(): # Tornado uses a "waker" handler internally that we'd like to ignore here remaining_handlers = { k: v for k, v in self.ioloop._handlers.iteritems() if k != self.ioloop._waker.fileno() } # Poll for any remaining connections remaining_count = len(remaining_handlers) # Once all handlers have been removed, we can stop safely if remaining_count == 0: LOGGER.info('All finished! Graceful stop complete') self.ioloop.stop() else: LOGGER.info('Waiting on IO handlers ({} remaining). ' 'Handlers: {!r}'.format(remaining_count, remaining_handlers)) self.ioloop.remove_handler(self.pipe) self.server.shutdown() self.server.stop() if graceful: # Poll the IOLoop's handlers until they all shut down. poller = PeriodicCallback(poll_stop, 500, io_loop=self.ioloop) poller.start() self.ioloop.add_timeout(self.ioloop.time() + shutdown_timeout, self.ioloop.stop) else: self.ioloop.stop() def shutdown_sig_handler(self, sig, frame): """ Signal handler for "stop" signals (SIGTERM, SIGINT) """ LOGGER.warning('{!r} caught signal {!r}! Shutting down'.format(self, sig)) try: self.ioloop.add_callback_from_signal(self.stop) except Exception as e: LOGGER.error( 'Encountered exception while shutting down: {}'.format(e) ) sys.exit(1)