def __run_epoll(self): """ run the application use epoll """ server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind(('0.0.0.0', self.port)) server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM ) # the number of client that connect to server server_socket.setblocking(0) # set 0 not block other block server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) epoll = select.epoll() epoll.register(server_socket.fileno(), select.EPOLLIN) try: http_connection = HttpConnection() http_io_buffer = HttpIOBuffer() http_io_routine_pool = HttpIORoutinePool() events_buf = [] while True: events = epoll.poll(1) + events_buf events_buf = [] for fileno, event in events: try: if fileno == server_socket.fileno(): # if request come connection, address = server_socket.accept( ) # waiting income connection connection.setblocking(0) # none block epoll.register( connection.fileno(), select.EPOLLIN ) # register socket read event to epoll http_connection.add_connection( connection.fileno(), connection) http_io_buffer.add_request(connection.fileno(), b'') http_io_buffer.add_response( connection.fileno(), self.response) elif event & select.EPOLLIN: # when data in os's read buffer area http_parser = http_io_routine_pool.get( file_no=fileno) if http_parser: data = http_parser.parse() if isinstance(data, str) or isinstance( data, unicode): http_io_buffer.add_response( fileno, http_io_buffer.get_response(fileno) + data) epoll.modify( fileno, select.EPOLLOUT ) # change file number to epoll out mode http_io_routine_pool.remove(fileno) else: # if coroutine http_io_routine_pool.add( fileno, http_parser) events_buf.append((fileno, event)) else: http_request_buffer = http_connection.get_connection( fileno).recv(SOCKET_RECEIVE_SIZE) http_io_buffer.add_request( fileno, http_io_buffer.get_request(fileno) + http_request_buffer) if self.EOL1 in http_io_buffer.get_request(fileno) \ or self.EOL2 in http_io_buffer.get_request(fileno): request_data = http_io_buffer.get_request(fileno)[:-2] \ if http_io_buffer.get_request(fileno).endswith("\r\n") \ else http_io_buffer.get_request(fileno) http_parser = HttpParser( request_data, self.handlers, settings=self.settings) data = http_parser.parse() if isinstance(data, str) or isinstance( data, unicode): http_io_buffer.add_response( fileno, http_io_buffer.get_response(fileno) + data) epoll.modify( fileno, select.EPOLLOUT ) # change file number to epoll out mode http_io_routine_pool.remove(fileno) else: # if coroutine http_io_routine_pool.add( fileno, http_parser) events_buf.append((fileno, event)) else: self.logger.error( "connection error in __run_epoll: %s", str(e)) http_connection.remove_connection(fileno) http_io_buffer.remove_request(fileno) http_io_buffer.remove_response(fileno) http_io_routine_pool.remove(fileno) epoll.unregister(fileno) elif event & select.EPOLLOUT: # if out mode bytes_written = http_connection.get_connection( fileno).send( http_io_buffer.get_response(fileno)) http_io_buffer.add_response( fileno, http_io_buffer.get_response(fileno) [bytes_written:]) if len(http_io_buffer.get_response( fileno)) == 0: # if file sent http_connection.get_connection( fileno).shutdown(socket.SHUT_RDWR) epoll.modify(fileno, select.EPOLLHUP) elif event & select.EPOLLHUP: # if message sent and file number in epoll is hup epoll.unregister( fileno) # remove file number from epoll http_connection.get_connection( fileno).close() # close connection http_connection.remove_connection( fileno ) # delete connection from connections dict except Exception as e: self.logger.info("error in __run_epoll: %s", str(e)) http_connection.remove_connection(fileno) http_io_buffer.remove_request(fileno) http_io_buffer.remove_response(fileno) http_io_routine_pool.remove(fileno) self.logger.info("fileno is: %s", str(fileno)) epoll.close() epoll = select.epoll() epoll.register(server_socket.fileno(), select.EPOLLIN) finally: epoll.unregister(server_socket.fileno()) epoll.close() server_socket.close()
def __run_epoll(self): """ run the application use epoll """ server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind(('0.0.0.0', self.port)) server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM) # the number of client that connect to server server_socket.setblocking(0) # set 0 not block other block server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) epoll = select.epoll() epoll.register(server_socket.fileno(), select.EPOLLIN) try: http_connection = HttpConnection() http_io_buffer = HttpIOBuffer() http_io_routine_pool = HttpIORoutinePool() events_buf = [] while True: events = epoll.poll(1) + events_buf events_buf = [] for fileno, event in events: try: if fileno == server_socket.fileno(): # if request come connection, address = server_socket.accept() # waiting income connection connection.setblocking(0) # none block epoll.register(connection.fileno(), select.EPOLLIN) # register socket read event to epoll http_connection.add_connection(connection.fileno(), connection) http_io_buffer.add_request(connection.fileno(), '') http_io_buffer.add_response(connection.fileno(), self.response) elif event & select.EPOLLIN: # when data in os's read buffer area http_parser = http_io_routine_pool.get(file_no=fileno) if http_parser: data = http_parser.parse() if isinstance(data, str) or isinstance(data, unicode): http_io_buffer.add_response( fileno, http_io_buffer.get_response(fileno) + data ) epoll.modify(fileno, select.EPOLLOUT) # change file number to epoll out mode http_io_routine_pool.remove(fileno) else: # if coroutine http_io_routine_pool.add(fileno, http_parser) events_buf.append((fileno, event)) else: if PY3: http_request_buffer = http_connection.get_connection(fileno).recv( SOCKET_RECEIVE_SIZE).decode("utf-8") else: http_request_buffer = http_connection.get_connection(fileno).recv( SOCKET_RECEIVE_SIZE) http_io_buffer.add_request( fileno, http_io_buffer.get_request(fileno) + http_request_buffer ) if self.EOL1 in http_io_buffer.get_request(fileno) \ or self.EOL2 in http_io_buffer.get_request(fileno): request_data = http_io_buffer.get_request(fileno)[:-2] \ if http_io_buffer.get_request(fileno).endswith("\r\n") \ else http_io_buffer.get_request(fileno) http_parser = HttpParser( request_data, self.handlers, settings=self.settings ) data = http_parser.parse() if isinstance(data, str) or isinstance(data, unicode): http_io_buffer.add_response( fileno, http_io_buffer.get_response(fileno) + data ) epoll.modify(fileno, select.EPOLLOUT) # change file number to epoll out mode http_io_routine_pool.remove(fileno) else: # if coroutine http_io_routine_pool.add(fileno, http_parser) events_buf.append((fileno, event)) else: http_connection.remove_connection(fileno) http_io_buffer.remove_request(fileno) http_io_buffer.remove_response(fileno) http_io_routine_pool.remove(fileno) epoll.unregister(fileno) elif event & select.EPOLLOUT: # if out mode http_response_message = http_io_buffer.get_response(fileno) if PY3: http_response_message = http_response_message.encode('utf-8') bytes_written = http_connection.get_connection(fileno).send( http_response_message ) http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[bytes_written:]) if len(http_io_buffer.get_response(fileno)) == 0: # if file sent http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR) epoll.modify(fileno, select.EPOLLHUP) elif event & select.EPOLLHUP: # if message sent and file number in epoll is hup epoll.unregister(fileno) # remove file number from epoll http_connection.get_connection(fileno).close() # close connection http_connection.remove_connection(fileno) # delete connection from connections dict except Exception as e: self.logger.info("error in __run_epoll: %s", str(e)) http_connection.remove_connection(fileno) http_io_buffer.remove_request(fileno) http_io_buffer.remove_response(fileno) http_io_routine_pool.remove(fileno) self.logger.info("fileno is: %s", str(fileno)) epoll.close() epoll = select.epoll() epoll.register(server_socket.fileno(), select.EPOLLIN) finally: epoll.unregister(server_socket.fileno()) epoll.close() server_socket.close()
def __run_kqueue(self): """ run server use kqueue """ server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.bind(('0.0.0.0', self.port)) server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM) kq = select.kqueue() http_connection = HttpConnection() http_io_buffer = HttpIOBuffer() http_io_routine_pool = HttpIORoutinePool() index = 1 events = [ select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD), select.kevent(server_socket.fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_ADD) ] events_buf = [] while True: try: event_list = kq.control(events, 128, 0.0001) + events_buf events_buf = [] except select.error as e: self.logger.error("error in __run_kqueue: %s", str(e)) break if event_list: for each in event_list: if each.ident == server_socket.fileno(): index += 1 conn, addr = server_socket.accept() http_connection.add_connection(index, conn) events.append( select.kevent( http_connection.get_connection(index).fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD, udata=index)) else: try: if each.udata >= 1 and each.filter == select.KQ_FILTER_READ: http_parser = http_io_routine_pool.get( file_no=each.udata) if http_parser: data = http_parser.parse() if isinstance(data, str) or isinstance( data, unicode): http_io_routine_pool.remove(each.udata) http_io_buffer.add_response( each.udata, data) events.append( select.kevent( http_connection.get_connection( each.udata).fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_ADD, udata=each.udata)) events.remove( select.kevent( http_connection.get_connection( each.udata).fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD, udata=each.udata)) else: # if coroutine http_io_routine_pool.add( each.udata, http_parser) events_buf.append(each) else: conn = http_connection.get_connection( each.udata) request_data = conn.recv( SOCKET_RECEIVE_SIZE) request_data = request_data[:-2] if request_data.endswith( "\r\n") else request_data http_parser = HttpParser( request_data, handlers=self.handlers, settings=self.settings) data = http_parser.parse() if isinstance(data, str) or isinstance( data, unicode): http_io_buffer.add_response( each.udata, data) events.append( select.kevent( http_connection.get_connection( each.udata).fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_ADD, udata=each.udata)) events.remove( select.kevent( http_connection.get_connection( each.udata).fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD, udata=each.udata)) else: # if coroutine http_io_routine_pool.add( each.udata, http_parser) events_buf.append(each) elif each.udata >= 1 and each.filter == select.KQ_FILTER_WRITE: conn = http_connection.get_connection( each.udata) data = http_io_buffer.get_response(each.udata) conn.send(data) events.remove( select.kevent( http_connection.get_connection( each.udata).fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_ADD, udata=each.udata)) conn.close() http_connection.remove_connection(each.udata) except Exception as e: self.logger.info( "error in __run_kqueue event list: %s", str(e)) self.logger.info("each filter: %s", each.filter) self.__remove_event(events, each) http_connection.remove_connection(each.udata) http_io_buffer.remove_request(each.udata) http_io_buffer.remove_response(each.udata) http_io_routine_pool.remove(each.udata) kq.close() kq = select.kqueue() server_socket.close()
def __run_kqueue(self): """ run server use kqueue """ server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.bind(('0.0.0.0', self.port)) server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM) kq = select.kqueue() http_connection = HttpConnection() http_io_buffer = HttpIOBuffer() http_io_routine_pool = HttpIORoutinePool() index = 1 events = [ select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD), # select.kevent(server_socket.fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_DELETE), # select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD), # select.kevent(server_socket.fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_DELETE), ] events_buf = [] while True: try: event_list = kq.control(events, 128, 0.0001) + events_buf events_buf = [] except select.error as e: self.logger.error("error in __run_kqueue: %s", str(e)) break if event_list: for each in event_list: if each.ident == server_socket.fileno(): index += 1 conn, addr = server_socket.accept() http_connection.add_connection(index, conn) events.append( select.kevent( http_connection.get_connection(index).fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD, udata=index ) ) else: try: if each.udata >= 1 and each.filter == select.KQ_FILTER_READ: http_parser = http_io_routine_pool.get(file_no=each.udata) if http_parser: data = http_parser.parse() if isinstance(data, str) or isinstance(data, unicode): http_io_routine_pool.remove(each.udata) http_io_buffer.add_response(each.udata, data) events.append( select.kevent( http_connection.get_connection(each.udata).fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_ADD, udata=each.udata ) ) events.remove(select.kevent( http_connection.get_connection(each.udata).fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD, udata=each.udata) ) else: # if coroutine http_io_routine_pool.add(each.udata, http_parser) events_buf.append(each) else: conn = http_connection.get_connection(each.udata) if PY3: request_data = conn.recv(SOCKET_RECEIVE_SIZE).decode("utf-8") else: request_data = conn.recv(SOCKET_RECEIVE_SIZE) request_data = request_data[:-2] if request_data.endswith("\r\n") else request_data http_parser = HttpParser( request_data, handlers=self.handlers, settings=self.settings ) data = http_parser.parse() if isinstance(data, str) or isinstance(data, unicode): http_io_buffer.add_response(each.udata, data) events.append( select.kevent( http_connection.get_connection(each.udata).fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_ADD, udata=each.udata ) ) events.remove(select.kevent( http_connection.get_connection(each.udata).fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD, udata=each.udata) ) else: # if coroutine http_io_routine_pool.add(each.udata, http_parser) events_buf.append(each) elif each.udata >= 1 and each.filter == select.KQ_FILTER_WRITE: conn = http_connection.get_connection(each.udata) data = http_io_buffer.get_response(each.udata) if PY3: data = data.encode('utf-8') conn.send(data) events.remove(select.kevent( http_connection.get_connection(each.udata).fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_ADD, udata=each.udata) ) conn.close() http_connection.remove_connection(each.udata) except Exception as e: self.logger.info("error in __run_kqueue event list: %s", str(e)) self.logger.info("each filter: %s", each.filter) self.__remove_event(events, each) http_connection.remove_connection(each.udata) http_io_buffer.remove_request(each.udata) http_io_buffer.remove_response(each.udata) http_io_routine_pool.remove(each.udata) kq.close() kq = select.kqueue() server_socket.close()