Beispiel #1
0
 def __run_epoll(self):
     """
     run the application use epoll
     """
     server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     server_socket.bind(('0.0.0.0', self.port))
     server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM
                          )  # the number of client that connect to server
     server_socket.setblocking(0)  # set 0 not block other block
     server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
     epoll = select.epoll()
     epoll.register(server_socket.fileno(), select.EPOLLIN)
     try:
         http_connection = HttpConnection()
         http_io_buffer = HttpIOBuffer()
         http_io_routine_pool = HttpIORoutinePool()
         events_buf = []
         while True:
             events = epoll.poll(1) + events_buf
             events_buf = []
             for fileno, event in events:
                 try:
                     if fileno == server_socket.fileno():  # if request come
                         connection, address = server_socket.accept(
                         )  # waiting income connection
                         connection.setblocking(0)  # none block
                         epoll.register(
                             connection.fileno(), select.EPOLLIN
                         )  # register socket read event to epoll
                         http_connection.add_connection(
                             connection.fileno(), connection)
                         http_io_buffer.add_request(connection.fileno(),
                                                    b'')
                         http_io_buffer.add_response(
                             connection.fileno(), self.response)
                     elif event & select.EPOLLIN:  # when data in os's read buffer area
                         http_parser = http_io_routine_pool.get(
                             file_no=fileno)
                         if http_parser:
                             data = http_parser.parse()
                             if isinstance(data, str) or isinstance(
                                     data, unicode):
                                 http_io_buffer.add_response(
                                     fileno,
                                     http_io_buffer.get_response(fileno) +
                                     data)
                                 epoll.modify(
                                     fileno, select.EPOLLOUT
                                 )  # change file number to epoll out mode
                                 http_io_routine_pool.remove(fileno)
                             else:  # if coroutine
                                 http_io_routine_pool.add(
                                     fileno, http_parser)
                                 events_buf.append((fileno, event))
                         else:
                             http_request_buffer = http_connection.get_connection(
                                 fileno).recv(SOCKET_RECEIVE_SIZE)
                             http_io_buffer.add_request(
                                 fileno,
                                 http_io_buffer.get_request(fileno) +
                                 http_request_buffer)
                             if self.EOL1 in http_io_buffer.get_request(fileno) \
                                     or self.EOL2 in http_io_buffer.get_request(fileno):
                                 request_data = http_io_buffer.get_request(fileno)[:-2] \
                                     if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                     else http_io_buffer.get_request(fileno)
                                 http_parser = HttpParser(
                                     request_data,
                                     self.handlers,
                                     settings=self.settings)
                                 data = http_parser.parse()
                                 if isinstance(data, str) or isinstance(
                                         data, unicode):
                                     http_io_buffer.add_response(
                                         fileno,
                                         http_io_buffer.get_response(fileno)
                                         + data)
                                     epoll.modify(
                                         fileno, select.EPOLLOUT
                                     )  # change file number to epoll out mode
                                     http_io_routine_pool.remove(fileno)
                                 else:  # if coroutine
                                     http_io_routine_pool.add(
                                         fileno, http_parser)
                                     events_buf.append((fileno, event))
                             else:
                                 self.logger.error(
                                     "connection error in __run_epoll: %s",
                                     str(e))
                                 http_connection.remove_connection(fileno)
                                 http_io_buffer.remove_request(fileno)
                                 http_io_buffer.remove_response(fileno)
                                 http_io_routine_pool.remove(fileno)
                                 epoll.unregister(fileno)
                     elif event & select.EPOLLOUT:  # if out mode
                         bytes_written = http_connection.get_connection(
                             fileno).send(
                                 http_io_buffer.get_response(fileno))
                         http_io_buffer.add_response(
                             fileno,
                             http_io_buffer.get_response(fileno)
                             [bytes_written:])
                         if len(http_io_buffer.get_response(
                                 fileno)) == 0:  # if file sent
                             http_connection.get_connection(
                                 fileno).shutdown(socket.SHUT_RDWR)
                             epoll.modify(fileno, select.EPOLLHUP)
                     elif event & select.EPOLLHUP:  # if message sent and file number in epoll is hup
                         epoll.unregister(
                             fileno)  # remove file number from epoll
                         http_connection.get_connection(
                             fileno).close()  # close connection
                         http_connection.remove_connection(
                             fileno
                         )  # delete connection from connections dict
                 except Exception as e:
                     self.logger.info("error in __run_epoll: %s", str(e))
                     http_connection.remove_connection(fileno)
                     http_io_buffer.remove_request(fileno)
                     http_io_buffer.remove_response(fileno)
                     http_io_routine_pool.remove(fileno)
                     self.logger.info("fileno is: %s", str(fileno))
                     epoll.close()
                     epoll = select.epoll()
                     epoll.register(server_socket.fileno(), select.EPOLLIN)
     finally:
         epoll.unregister(server_socket.fileno())
         epoll.close()
         server_socket.close()
Beispiel #2
0
 def __run_epoll(self):
     """
     run the application use epoll
     """
     server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     server_socket.bind(('0.0.0.0', self.port))
     server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
     server_socket.setblocking(0)  # set 0 not block other block
     server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
     epoll = select.epoll()
     epoll.register(server_socket.fileno(), select.EPOLLIN)
     try:
         http_connection = HttpConnection()
         http_io_buffer = HttpIOBuffer()
         http_io_routine_pool = HttpIORoutinePool()
         events_buf = []
         while True:
             events = epoll.poll(1) + events_buf
             events_buf = []
             for fileno, event in events:
                 try:
                     if fileno == server_socket.fileno():  # if request come
                         connection, address = server_socket.accept()  # waiting income connection
                         connection.setblocking(0)  # none block
                         epoll.register(connection.fileno(), select.EPOLLIN)  # register socket read event to epoll
                         http_connection.add_connection(connection.fileno(), connection)
                         http_io_buffer.add_request(connection.fileno(), '')
                         http_io_buffer.add_response(connection.fileno(), self.response)
                     elif event & select.EPOLLIN:  # when data in os's read buffer area
                         http_parser = http_io_routine_pool.get(file_no=fileno)
                         if http_parser:
                             data = http_parser.parse()
                             if isinstance(data, str) or isinstance(data, unicode):
                                 http_io_buffer.add_response(
                                     fileno,
                                     http_io_buffer.get_response(fileno) + data
                                 )
                                 epoll.modify(fileno, select.EPOLLOUT)  # change file number to epoll out mode
                                 http_io_routine_pool.remove(fileno)
                             else:  # if coroutine
                                 http_io_routine_pool.add(fileno, http_parser)
                                 events_buf.append((fileno, event))
                         else:
                             if PY3:
                                 http_request_buffer = http_connection.get_connection(fileno).recv(
                                     SOCKET_RECEIVE_SIZE).decode("utf-8")
                             else:
                                 http_request_buffer = http_connection.get_connection(fileno).recv(
                                     SOCKET_RECEIVE_SIZE)
                             http_io_buffer.add_request(
                                 fileno,
                                 http_io_buffer.get_request(fileno) + http_request_buffer
                             )
                             if self.EOL1 in http_io_buffer.get_request(fileno) \
                                     or self.EOL2 in http_io_buffer.get_request(fileno):
                                 request_data = http_io_buffer.get_request(fileno)[:-2] \
                                     if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                     else http_io_buffer.get_request(fileno)
                                 http_parser = HttpParser(
                                     request_data,
                                     self.handlers,
                                     settings=self.settings
                                 )
                                 data = http_parser.parse()
                                 if isinstance(data, str) or isinstance(data, unicode):
                                     http_io_buffer.add_response(
                                         fileno,
                                         http_io_buffer.get_response(fileno) + data
                                     )
                                     epoll.modify(fileno, select.EPOLLOUT)  # change file number to epoll out mode
                                     http_io_routine_pool.remove(fileno)
                                 else:  # if coroutine
                                     http_io_routine_pool.add(fileno, http_parser)
                                     events_buf.append((fileno, event))
                             else:
                                 http_connection.remove_connection(fileno)
                                 http_io_buffer.remove_request(fileno)
                                 http_io_buffer.remove_response(fileno)
                                 http_io_routine_pool.remove(fileno)
                                 epoll.unregister(fileno)
                     elif event & select.EPOLLOUT:  # if out mode
                         http_response_message = http_io_buffer.get_response(fileno)
                         if PY3:
                             http_response_message = http_response_message.encode('utf-8')
                         bytes_written = http_connection.get_connection(fileno).send(
                             http_response_message
                         )
                         http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[bytes_written:])
                         if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                             http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                             epoll.modify(fileno, select.EPOLLHUP)
                     elif event & select.EPOLLHUP:  # if message sent and file number in epoll is hup
                         epoll.unregister(fileno)  # remove file number from epoll
                         http_connection.get_connection(fileno).close()  # close connection
                         http_connection.remove_connection(fileno)  # delete connection from connections dict
                 except Exception as e:
                     self.logger.info("error in __run_epoll: %s", str(e))
                     http_connection.remove_connection(fileno)
                     http_io_buffer.remove_request(fileno)
                     http_io_buffer.remove_response(fileno)
                     http_io_routine_pool.remove(fileno)
                     self.logger.info("fileno is: %s", str(fileno))
                     epoll.close()
                     epoll = select.epoll()
                     epoll.register(server_socket.fileno(), select.EPOLLIN)
     finally:
         epoll.unregister(server_socket.fileno())
         epoll.close()
         server_socket.close()
Beispiel #3
0
 def __run_kqueue(self):
     """
     run server use kqueue
     """
     server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     server_socket.bind(('0.0.0.0', self.port))
     server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)
     kq = select.kqueue()
     http_connection = HttpConnection()
     http_io_buffer = HttpIOBuffer()
     http_io_routine_pool = HttpIORoutinePool()
     index = 1
     events = [
         select.kevent(server_socket.fileno(), select.KQ_FILTER_READ,
                       select.KQ_EV_ADD),
         select.kevent(server_socket.fileno(), select.KQ_FILTER_WRITE,
                       select.KQ_EV_ADD)
     ]
     events_buf = []
     while True:
         try:
             event_list = kq.control(events, 128, 0.0001) + events_buf
             events_buf = []
         except select.error as e:
             self.logger.error("error in __run_kqueue: %s", str(e))
             break
         if event_list:
             for each in event_list:
                 if each.ident == server_socket.fileno():
                     index += 1
                     conn, addr = server_socket.accept()
                     http_connection.add_connection(index, conn)
                     events.append(
                         select.kevent(
                             http_connection.get_connection(index).fileno(),
                             select.KQ_FILTER_READ,
                             select.KQ_EV_ADD,
                             udata=index))
                 else:
                     try:
                         if each.udata >= 1 and each.filter == select.KQ_FILTER_READ:
                             http_parser = http_io_routine_pool.get(
                                 file_no=each.udata)
                             if http_parser:
                                 data = http_parser.parse()
                                 if isinstance(data, str) or isinstance(
                                         data, unicode):
                                     http_io_routine_pool.remove(each.udata)
                                     http_io_buffer.add_response(
                                         each.udata, data)
                                     events.append(
                                         select.kevent(
                                             http_connection.get_connection(
                                                 each.udata).fileno(),
                                             select.KQ_FILTER_WRITE,
                                             select.KQ_EV_ADD,
                                             udata=each.udata))
                                     events.remove(
                                         select.kevent(
                                             http_connection.get_connection(
                                                 each.udata).fileno(),
                                             select.KQ_FILTER_READ,
                                             select.KQ_EV_ADD,
                                             udata=each.udata))
                                 else:  # if coroutine
                                     http_io_routine_pool.add(
                                         each.udata, http_parser)
                                     events_buf.append(each)
                             else:
                                 conn = http_connection.get_connection(
                                     each.udata)
                                 request_data = conn.recv(
                                     SOCKET_RECEIVE_SIZE)
                                 request_data = request_data[:-2] if request_data.endswith(
                                     "\r\n") else request_data
                                 http_parser = HttpParser(
                                     request_data,
                                     handlers=self.handlers,
                                     settings=self.settings)
                                 data = http_parser.parse()
                                 if isinstance(data, str) or isinstance(
                                         data, unicode):
                                     http_io_buffer.add_response(
                                         each.udata, data)
                                     events.append(
                                         select.kevent(
                                             http_connection.get_connection(
                                                 each.udata).fileno(),
                                             select.KQ_FILTER_WRITE,
                                             select.KQ_EV_ADD,
                                             udata=each.udata))
                                     events.remove(
                                         select.kevent(
                                             http_connection.get_connection(
                                                 each.udata).fileno(),
                                             select.KQ_FILTER_READ,
                                             select.KQ_EV_ADD,
                                             udata=each.udata))
                                 else:  # if coroutine
                                     http_io_routine_pool.add(
                                         each.udata, http_parser)
                                     events_buf.append(each)
                         elif each.udata >= 1 and each.filter == select.KQ_FILTER_WRITE:
                             conn = http_connection.get_connection(
                                 each.udata)
                             data = http_io_buffer.get_response(each.udata)
                             conn.send(data)
                             events.remove(
                                 select.kevent(
                                     http_connection.get_connection(
                                         each.udata).fileno(),
                                     select.KQ_FILTER_WRITE,
                                     select.KQ_EV_ADD,
                                     udata=each.udata))
                             conn.close()
                             http_connection.remove_connection(each.udata)
                     except Exception as e:
                         self.logger.info(
                             "error in __run_kqueue event list: %s", str(e))
                         self.logger.info("each filter: %s", each.filter)
                         self.__remove_event(events, each)
                         http_connection.remove_connection(each.udata)
                         http_io_buffer.remove_request(each.udata)
                         http_io_buffer.remove_response(each.udata)
                         http_io_routine_pool.remove(each.udata)
                         kq.close()
                         kq = select.kqueue()
     server_socket.close()
Beispiel #4
0
 def __run_kqueue(self):
     """
     run server use kqueue
     """
     server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     server_socket.bind(('0.0.0.0', self.port))
     server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)
     kq = select.kqueue()
     http_connection = HttpConnection()
     http_io_buffer = HttpIOBuffer()
     http_io_routine_pool = HttpIORoutinePool()
     index = 1
     events = [
         select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD),
         # select.kevent(server_socket.fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_DELETE),
         # select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD),
         # select.kevent(server_socket.fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_DELETE),
     ]
     events_buf = []
     while True:
         try:
             event_list = kq.control(events, 128, 0.0001) + events_buf
             events_buf = []
         except select.error as e:
             self.logger.error("error in __run_kqueue: %s", str(e))
             break
         if event_list:
             for each in event_list:
                 if each.ident == server_socket.fileno():
                     index += 1
                     conn, addr = server_socket.accept()
                     http_connection.add_connection(index, conn)
                     events.append(
                         select.kevent(
                             http_connection.get_connection(index).fileno(),
                             select.KQ_FILTER_READ,
                             select.KQ_EV_ADD,
                             udata=index
                         )
                     )
                 else:
                     try:
                         if each.udata >= 1 and each.filter == select.KQ_FILTER_READ:
                             http_parser = http_io_routine_pool.get(file_no=each.udata)
                             if http_parser:
                                 data = http_parser.parse()
                                 if isinstance(data, str) or isinstance(data, unicode):
                                     http_io_routine_pool.remove(each.udata)
                                     http_io_buffer.add_response(each.udata, data)
                                     events.append(
                                         select.kevent(
                                             http_connection.get_connection(each.udata).fileno(),
                                             select.KQ_FILTER_WRITE,
                                             select.KQ_EV_ADD,
                                             udata=each.udata
                                         )
                                     )
                                     events.remove(select.kevent(
                                         http_connection.get_connection(each.udata).fileno(),
                                         select.KQ_FILTER_READ,
                                         select.KQ_EV_ADD,
                                         udata=each.udata)
                                     )
                                 else:  # if coroutine
                                     http_io_routine_pool.add(each.udata, http_parser)
                                     events_buf.append(each)
                             else:
                                 conn = http_connection.get_connection(each.udata)
                                 if PY3:
                                     request_data = conn.recv(SOCKET_RECEIVE_SIZE).decode("utf-8")
                                 else:
                                     request_data = conn.recv(SOCKET_RECEIVE_SIZE)
                                 request_data = request_data[:-2] if request_data.endswith("\r\n") else request_data
                                 http_parser = HttpParser(
                                     request_data,
                                     handlers=self.handlers,
                                     settings=self.settings
                                 )
                                 data = http_parser.parse()
                                 if isinstance(data, str) or isinstance(data, unicode):
                                     http_io_buffer.add_response(each.udata, data)
                                     events.append(
                                         select.kevent(
                                             http_connection.get_connection(each.udata).fileno(),
                                             select.KQ_FILTER_WRITE,
                                             select.KQ_EV_ADD,
                                             udata=each.udata
                                         )
                                     )
                                     events.remove(select.kevent(
                                         http_connection.get_connection(each.udata).fileno(),
                                         select.KQ_FILTER_READ,
                                         select.KQ_EV_ADD,
                                         udata=each.udata)
                                     )
                                 else:  # if coroutine
                                     http_io_routine_pool.add(each.udata, http_parser)
                                     events_buf.append(each)
                         elif each.udata >= 1 and each.filter == select.KQ_FILTER_WRITE:
                             conn = http_connection.get_connection(each.udata)
                             data = http_io_buffer.get_response(each.udata)
                             if PY3:
                                 data = data.encode('utf-8')
                             conn.send(data)
                             events.remove(select.kevent(
                                 http_connection.get_connection(each.udata).fileno(),
                                 select.KQ_FILTER_WRITE,
                                 select.KQ_EV_ADD,
                                 udata=each.udata)
                             )
                             conn.close()
                             http_connection.remove_connection(each.udata)
                     except Exception as e:
                         self.logger.info("error in __run_kqueue event list: %s", str(e))
                         self.logger.info("each filter: %s", each.filter)
                         self.__remove_event(events, each)
                         http_connection.remove_connection(each.udata)
                         http_io_buffer.remove_request(each.udata)
                         http_io_buffer.remove_response(each.udata)
                         http_io_routine_pool.remove(each.udata)
                         kq.close()
                         kq = select.kqueue()
     server_socket.close()