def _corou_timer(self, now_milli): temp_ntf_list = [] while not self._corou_notify_queue.empty(): ntf_obj = self._corou_notify_queue.get_nowait() temp_ntf_list.append(ntf_obj) for ntf_obj in temp_ntf_list: if not type(ntf_obj) is CoroutineNotify: Logger().error('234223 ntf_obj type error.{}'.format(ntf_obj)) continue if ntf_obj.ope == CoroutineOpe.COROUTINE_ADD.value: cid = ntf_obj.cid gene_obj = ntf_obj.obj self.__corou_dict[cid] = gene_obj try: gene_obj.send(None) gene_obj.send(cid) except StopIteration: self._safe_del_corou_item(cid) elif ntf_obj.ope == CoroutineOpe.COROUTINE_PUSH_MSG.value: cid = ntf_obj.cid msg = ntf_obj.obj it_obj = self.__corou_dict.get(cid, None) if it_obj is None: Logger().warning( '434679 cid not in __corou_dict keys .{}'.format(cid)) continue try: it_obj.send(msg) except StopIteration: self._safe_del_corou_item(cid) else: Logger().error('531238 ntf_obj ope error.{}'.format( ntf_obj.ope)) continue
def __send_ws_msg(self, sock_id, buf): if sock_id in self.__sid_conn_dict.keys(): msgLen = len(buf) backMsgList = [] backMsgList.append(struct.pack('B', 0x82)) # 0x82是二进制流,0x81是text if msgLen <= 125: backMsgList.append(struct.pack('b', msgLen)) elif msgLen <= 65535: backMsgList.append(struct.pack('b', 126)) backMsgList.append(struct.pack('!h', msgLen)) elif msgLen <= (2 ^ 64 - 1): backMsgList.append(struct.pack('b', 127)) backMsgList.append(struct.pack('!q', msgLen)) else: Logger().info("the message is too long to send in a time") return message_byte = bytes() for c in backMsgList: message_byte += c message_byte += buf try: self.__sid_conn_dict[sock_id].send(message_byte) except Exception as e: Logger().warning(e) else: Logger().warning( 'ws server __send_ws_msg. sid not exist. {}'.format(sock_id))
def __on_read(self, sock, mask): sid = self.__sock_sid_dict[sock] try: if self.__handshakes_dict[sock] == False: # 首先判断websocket握手 msg = sock.recv(PacketDef.PER_RECV_SIZE.value) if (not msg) or (len(msg) == 0): self.__close_socket(sid, False) # 还未连接上,不发给逻辑层 nf = NetNotify(sid, NeTDef.CLIENT_CONNECT_FAIL.value) self._out_queue.put(nf) return handshake_recv = str(msg) if handshake_recv.lower().find('connection: upgrade') != -1: self.__handshakes_dict[sock] = True nf = NetNotify(sid, NeTDef.CONNECT.value) self._out_queue.put(nf) return # receive = sock.recv(2) # 130是二进制流,129是text if (not receive) or (len(receive) == 0): self.__close_socket(sid, True) return len_tag = receive[1] & 0x7f content_len = 0 #实际应该接收的内容长度 if len_tag <= 125: content_len = len_tag elif len_tag == 126: receive, recv_ok = self.__recv_msg(sock, 2) if recv_ok is False: Logger().warning('读取websocket长度出错:126') self.__close_socket(sid, True) return content_len, = struct.unpack_from('!h', receive, 0) elif len_tag == 127: receive, recv_ok = self.receive_msg(sock, 8) if recv_ok is False: Logger().warning('读取websocket长度出错:127') self.__close_socket(sid, True) return content_len, = struct.unpack_from('!q', receive, 0) else: Logger().warning('读取websocket长度出错:too large') self.__close_socket(sid, True) return receive, recv_ok = self.__recv_msg(sock, content_len) if recv_ok is False: Logger().warning('读取websocket包内容出错') self.__close_socket(sid, True) return ntif = NetNotify(sid, NeTDef.RECV.value, receive) # 告诉主进程收到网络消息 self._out_queue.put(ntif) except ConnectionResetError as err: #Logger().info('ConnectionResetError:{} {}'.format(sid, err)) self.__close_socket(sid, True) except Exception as e: if (type(e) is OSError) and (e.errno == 107): pass # linux下,没连接上就会recv else: Logger().info('{}:{}:{}'.format(sid, e, type(e))) self.__close_socket(sid, True)
def __send_msg(self, sock_id, buf): if sock_id in self.__sid_conn_dict.keys(): try: self.__sid_conn_dict[sock_id].send(buf) except Exception as e: Logger().warning(e) else: Logger().warning( 'server send_msg. sid not exist. {}'.format(sock_id))
def __recv_msg(self, conn, length): try: receive = conn.recv(length) except ConnectionResetError as err: Logger().info(err) return None, False except Exception as e: Logger().info(e) return None, False return receive, True
def run(self): self.thread_init() self.__event.set() now_milli = Helper.get_program_milli_second() last_milli = now_milli tick_milli = 0 cls_name = self.__class__.__name__ while True: b_quit = False #填充消息队列 self.fill_msg_queue() temp_msg_list = [] while not self._msg_queue.empty(): msg_obj = self._msg_queue.get_nowait() temp_msg_list.append(msg_obj) queue_size = len(temp_msg_list) for msg_obj in temp_msg_list: if type(msg_obj) is str and msg_obj.lower( ) == Helper.quit_signal(): # 退出信号 b_quit = True continue now_milli = Helper.get_program_milli_second() tick_milli = now_milli - last_milli if tick_milli > self.__frame_abort_time: Logger().warning( '{}.主循环超期严重.放弃部分消息:帧耗时:{}.消息队列大小:{}'.format( cls_name, tick_milli, queue_size)) break try: # 此处有了try,下级函数可以减少try self.process_msg(msg_obj) except Exception as e: traceback.print_exc() now_milli = Helper.get_program_milli_second() try: self.process_timer(now_milli) # 此处有了try,下级函数可以减少try except Exception as e: traceback.print_exc() # 时间处理 tick_milli = now_milli - last_milli last_milli = now_milli if tick_milli > self.__frame_warn_time: Logger().warning('{}.主循环线程超期:帧耗时:{}.消息队列大小:{}'.format( cls_name, tick_milli, queue_size)) else: time.sleep(self.__frame_min_time / 1000) if b_quit: # 退出 break self.thread_quit()
def on_client_quit(self, sock_id, user_id): if sock_id in self.sid_dict.keys(): del self.sid_dict[sock_id] else: Logger().warning( 'BaseClient:quit error, sock_id not in mgr dict.{}'.format( sock_id)) if not user_id is None: if user_id in self.uid_dict.keys(): del self.uid_dict[user_id] else: Logger().warning( 'BaseClient:quit error, user_id not in mgr dict.{}'.format( user_id)) if user_id in self.account_new_login_dict.keys(): # 在新登录队列中存在 new_client = self.account_new_login_dict[user_id] new_client.on_account_old_login_quit() del self.account_new_login_dict[user_id]
def __send_handshake_msg(self, sock): handshake_str = ( 'GET / HTTP/1.1\r\nHost: {}:{}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: EZFAWipfRYaGQ79BAMHd+A==' .format(self.__server_host, self.__server_port)) handshake_bytes = handshake_str.encode(encoding="utf8") try: sock.send(handshake_bytes) except Exception as e: Logger().warning(e)
def run_frame(self, now_milli): try: events = self.__selector.select(NeTDef.SELECT_TIME_OUT.value) except Exception as e: Logger().error('{}.{}'.format(self.__net_id, e)) return for key, mask in events: callback = key.data callback(key.fileobj, mask) # 以下处理外界的消息,发关消息或关闭socket,或者退出进程 notify_list = [] while not self._in_queue.empty(): nf = self._in_queue.get_nowait() notify_list.append(nf) send_count = 0 for nf in notify_list: if type(nf) is str and nf.lower() == Helper.quit_signal(): self.__close_all() self._run_flag = False break # 此处break, 因为已经close_all, 服务端类此处是continue sid = nf.sid if sid.net_id != self.__net_id: Logger().warning('client connect.net_id error.{}.{}.'.format( self.__net_id, sid)) continue if nf.ope == NeTDef.CLIENT_CREATE.value: if platform.system().lower() == 'windows': '''windows select模型超过509连接会报错''' conn_num = len(self.__sock_sid_dict) if conn_num >= NeTDef.WINDOWS_SELECT_MAX.value: Logger().error( 'client connect windows下select连接树超出数量.{}:{}'. format(self.__net_id, conn_num)) continue self.__create_socket() elif nf.ope == NeTDef.CLIENT_CLOSE.value: self.__close_socket(sid, True) elif nf.ope == NeTDef.SEND.value: self.__send_msg(sid, nf.buf) send_count += 1 if send_count >= NeTDef.MAX_TRANSMIT_ONE_FRAME.value: continue
def deal_with_bytes(self, sid, bys): if not sid in self.split_dict.keys(): Logger().warning('deal_with_bytes:sid not exist.{}'.format(sid)) return sid, [] ret_list = [] try: split = self.split_dict[sid] ret_list = split.deal_with_bytes(bys) except Exception as e: raise e #留给调用者处理 return sid, ret_list
def add_corou(self, gene_func): if not type(gene_func) is types.GeneratorType: # 判断类型 Logger().warning('128099. gene_func not GeneratorType.{}'.format( type(gene_func))) return None cur_id = self.__corou_tick self.__corou_tick += 1 it_obj = gene_func ntf = CoroutineNotify(CoroutineOpe.COROUTINE_ADD.value, cur_id, it_obj) self._corou_notify_queue.put(ntf) return cur_id
def run_timer(self, now_milli): if self.is_end: return if (now_milli - self.last_trigger_time) >= self.ticker: try: self.func() except Exception as e: Logger().error('%s:%s:%s' % (self.event_name, repr(self.func), e)) self.last_trigger_time = now_milli self.exe_count += 1 if self.exe_num != 0 and self.exe_count >= self.exe_num: self.is_end = True
def push_data(self, dt_bys): dt_len = len(dt_bys) if (dt_len > PacketDef.MAX_PACKET_LEN.value): Logger().warning('PacketSplit.push_data.长度超长.{}'.format(dt_len)) return False for i in range(0, dt_len): if self._copy_pos >= PacketDef.PACKET_SPLIT_BUFFER_SIZE.value: self._copy_pos = 0 self._byte_arr[self._copy_pos] = dt_bys[i] self._copy_pos += 1 self._data_len += dt_len return True
def process_msg(self, msg_obj): # 重载父类 '''子类重载时,记得调用super().process_msg''' if type(msg_obj) is NetNotify: # 只处理网络消息 if msg_obj.ope == NeTDef.CONNECT.value: #新连接连上 try: self.create_packet_split(msg_obj.sid) # 创建粘包处理器 except NetSplitError as e: Logger().warning(e) try: client = self.create_client(msg_obj.sid) self._client_mgr.on_net_connect(msg_obj.sid) except ClientError as e: Logger().warning(e) elif msg_obj.ope == NeTDef.DISCONNECT.value: # 连接关闭 self._net_split_mgr.remove_split(msg_obj.sid) # 关闭粘包处理 try: self._client_mgr.on_net_disconnect(msg_obj.sid) # 关闭客户端 except ClientError as e: Logger().warning(e) elif msg_obj.ope == NeTDef.RECV.value: # 收到网络消息 if msg_obj.buf is None or len(msg_obj.buf) == 0: Logger().warning('收到空串,{}'.format(msg_obj.sid)) return try: sid, pack_list = self._net_split_mgr.deal_with_bytes(msg_obj.sid, msg_obj.buf) #粘包处理 for pack in pack_list: try: self._client_mgr.push_net_msg(sid, pack) except ClientError as e: Logger().warning(e) except PacketError as e: Logger().warning(e) elif msg_obj.ope == NeTDef.CLIENT_CONNECT_FAIL.value: # 作为客户端,连远程服务器失败 self.on_client_connect_fail(msg_obj.sid)
def run_frame(self, now_milli): if self.mysql_op.is_open is not True: # 中途数据库断开连接 if (now_milli - self.__last_reconn) > 60000: # 每60秒重连一次 self.mysql_op.open_mysql() self.__last_reconn = now_milli notify_list = [] while not self._in_queue.empty(): nf = self._in_queue.get_nowait() notify_list.append(nf) for nf in notify_list: if type(nf) is str and nf.lower() == Helper.quit_signal(): self._run_flag = False continue if not type(nf) is MySqlNotify: Logger().warning('{}:MySqlProcess.not mysqlnotify:{} pid:{}'.format(self.__alias, nf, os.getpid())) continue if nf.status != MNTDef.REQ.value: Logger().warning('{}:MySqlProcess.mysqlnotify status error:{} pid:{}'.format(self.__alias, nf, os.getpid())) continue if nf.ope == MNTDef.SELECT_ONE.value: sql = nf.sqls[0] nf.result = self.mysql_op.select_one(sql) nf.success = (nf.result != None) elif nf.ope == MNTDef.SELECT_LIST.value: sql = nf.sqls[0] nf.result = self.mysql_op.select_list(sql) nf.success = (nf.result != None) elif nf.ope == MNTDef.INSERT_LID.value: sql = nf.sqls[0] nf.success, nf.result = self.mysql_op.insert_last_id(sql) elif nf.ope == MNTDef.EXE_SINGLE.value: sql = nf.sqls[0] nf.success, nf.result = self.mysql_op.execute_single_sql(sql) elif nf.ope == MNTDef.EXE_MULTI.value: sql_list = nf.sqls nf.success = self.mysql_op.execute_multi_sql(sql_list) nf.status = MNTDef.RET.value self._out_queue.put(nf) self.__time_group.run_timer(Helper.get_program_milli_second())
def __close_socket(self, sock_id, b_out): if not sock_id in self.__sid_sock_dict.keys(): Logger().warning('close sock id not exist.{}'.format(sock_id)) return sock = self.__sid_sock_dict[sock_id] sock.close() self.__selector.unregister(sock) del self.__sid_sock_dict[sock_id] del self.__sock_sid_dict[sock] if b_out: nf = NetNotify(sock_id, NeTDef.DISCONNECT.value) self._out_queue.put(nf)
def add_timer_event(self, event_name, ticker, func, exe_num=0): '''添加timer事件, 返回是否成功''' n_exist = False if event_name in self.__run_dict.keys(): n_exist = True if event_name in self.__add_dict.keys(): n_exist = True if n_exist: Logger().error('event_name exist. {}'.format(event_name)) return False timer_event = TimerEvent(event_name, ticker, func, exe_num) self.__add_dict[event_name] = timer_event return True
def __on_read(self, conn, mask): sid = self.__conn_sid_dict[conn] try: msg = conn.recv(PacketDef.PER_RECV_SIZE.value) if (not msg) or (len(msg) == 0): self.__close_conn(conn) return ntif = NetNotify(sid, NeTDef.RECV.value, msg) self._out_queue.put(ntif) except ConnectionResetError as err: # 目标方突然强行关闭程序会发生,如果是正常关闭socket,不会产生异常 # Logger().info('ConnectionResetError:{} {}'.format(sid, err)) self.__close_conn(conn) except Exception as e: Logger().info(sid, e) self.__close_conn(conn)
def run_frame(self, now_milli): try: events = self.__selector.select(NeTDef.SELECT_TIME_OUT.value) except Exception as e: Logger().error('{}.{}'.format(self.__net_id, e)) return for key, mask in events: callback = key.data callback(key.fileobj, mask) # 以下处理外界的消息,发关消息或关闭socket,或者退出进程 notify_list = [] while not self._in_queue.empty(): nf = self._in_queue.get_nowait() notify_list.append(nf) send_count = 0 for nf in notify_list: if type(nf) is str and nf.lower() == Helper.quit_signal(): self._run_flag = False continue sid = nf.sid if sid.net_id != self.__net_id: Logger().warning('server listen.net_id error.{}.{}'.format( self.__net_id, sid)) continue if nf.ope == NeTDef.SERVER_CLOSE.value: if not sid in self.__sid_conn_dict.keys(): Logger().warning( 'SERVER_CLOSE.sid not exist. {}'.format(sid)) continue conn_close = self.__sid_conn_dict[sid] self.__close_conn(conn_close) elif nf.ope == NeTDef.SEND.value: self.__send_msg(sid, nf.buf) send_count += 1 if send_count >= NeTDef.MAX_TRANSMIT_ONE_FRAME.value: continue
def __send_ws_msg(self, sock_id, msg): if not sock_id in self.__sid_sock_dict.keys(): Logger().warning( 'ws client send_ws_msg. sid not exist. {}'.format(sock_id)) return msgLen = len(msg) backMsgList = [] backMsgList.append(struct.pack('B', 0x82)) # 130是二进制流,129是text if msgLen <= 125: backMsgList.append(struct.pack('B', (msgLen | 0x80))) elif msgLen <= 65535: backMsgList.append(struct.pack('B', (126 | 0x80))) backMsgList.append(struct.pack('!h', msgLen)) elif msgLen <= (2 ^ 64 - 1): backMsgList.append(struct.pack('B', (127 | 0x80))) backMsgList.append(struct.pack('!q', msgLen)) else: Logger().info("the message is too long to send in a time") return backMsgList.append(struct.pack('bbbb', 0x01, 0x02, 0x03, 0x04)) #mask, 自定义 bye_arr = bytearray() mask = b'\x01\x02\x03\x04' i = 0 for d in msg: bye_arr.append(d ^ mask[i % 4]) i += 1 message_byte = bytes() for c in backMsgList: message_byte += c message_byte += bytes(bye_arr) sock = self.__sid_sock_dict[sock_id] try: sock.send(message_byte) except Exception as e: Logger().warning(e)
def __on_accept(self, sock, mask): conn, addr = sock.accept() conn.setblocking(False) if platform.system().lower() == 'windows': '''windows select模型超过509连接会报错''' conn_num = len(self.__conn_sid_dict) if conn_num >= NeTDef.WINDOWS_SELECT_MAX.value: Logger().error('wsserver, windows平台超过最大select数量.{}:{}'.format( self.__net_id, conn_num)) conn.close() return sid = SockId(self.__net_id) sid.set_data(addr) self.__conn_sid_dict[conn] = sid self.__sid_conn_dict[sid] = conn self.__selector.register(conn, selectors.EVENT_READ, self.__on_read) self.__handshakes_dict[conn] = False
def __on_read(self, sock, mask): sid = self.__sock_sid_dict[sock] try: msg = sock.recv(PacketDef.PER_RECV_SIZE.value) #linux下,创建后就会执行 if (not msg) or (len(msg) == 0): self.__close_socket(sid, True) return ntif = NetNotify(sid, NeTDef.RECV.value, msg) self._out_queue.put(ntif) except ConnectionResetError as err: #Logger().info('ConnectionResetError:{} {}'.format(sid, err)) self.__close_socket(sid, True) except Exception as e: if (type(e) is OSError) and (e.errno == 107): pass # linux下,没连接上就会recv else: Logger().info('{}:{}:{}'.format(sid, e, type(e))) self.__close_socket(sid, True)
def run(self): LogInit(self._log_name) # 不同进程,都要初始化一次 self.process_init() self.__event.set() now_milli = Helper.get_program_milli_second() last_milli = now_milli tick_milli = 0 self._run_flag = True while self._run_flag: now_milli = Helper.get_program_milli_second() self.run_frame(now_milli) tick_milli = now_milli - last_milli last_milli = now_milli if tick_milli > self.__frame_warn_time: Logger().warning('{}.Process主循环线程超期:帧耗时:{}.'.format( self._log_name, tick_milli)) else: time.sleep(self.__frame_min_time / 1000) self.process_end()
def __create_socket(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #sock.setblocking(False) 这里不能设,否则会出现BlockingIOError: [WinError 10035] far_address = (self.__server_host, self.__server_port) try: sock.connect(far_address) except Exception as e: Logger().info(e) sock_id = SockId(self.__net_id) nf = NetNotify(sock_id, NeTDef.CLIENT_CONNECT_FAIL.value) self._out_queue.put(nf) return sock_id = SockId(self.__net_id) sock_id.set_data(sock.getsockname()) self.__sid_sock_dict[sock_id] = sock self.__sock_sid_dict[sock] = sock_id self.__selector.register(sock, selectors.EVENT_READ, self.__on_read) nf = NetNotify(sock_id, NeTDef.CONNECT.value) self._out_queue.put(nf)
def process_init(self): self.__sock_listen.bind(('0.0.0.0', self.__port)) self.__sock_listen.listen(NeTDef.LISTEN_NUM.value) self.__selector.register(self.__sock_listen, selectors.EVENT_READ, self.__on_accept) Logger().info('%s 监听启动. 端口:%d' % (self._log_name, self.__port))
def remove_split(self, sid): if not sid in self.split_dict.keys(): Logger().warning('remove_split:sid not exist.{}'.format(sid)) return del self.split_dict[sid]
def __on_read(self, conn, mask): sid = self.__conn_sid_dict[conn] try: if self.__handshakes_dict[conn] == False: # 首先判断websocket握手 msg = conn.recv(PacketDef.PER_RECV_SIZE.value) if (not msg) or (len(msg) == 0): self.__close_conn(conn) return handshake_recv = str(msg) entities = handshake_recv.split("\\r\\n") sec_key_in = '' for str_sub in entities: if str_sub.split(":")[0].strip() == 'Sec-WebSocket-Key': sec_key_in = str_sub.split(":")[1].strip() break sec_websocket_key = sec_key_in + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11' response_key = base64.b64encode( hashlib.sha1(bytes(sec_websocket_key, encoding="utf8")).digest()) response_key_str = str(response_key) response_key_str = response_key_str[2:30] response_key_entity = "Sec-WebSocket-Accept: " + response_key_str + "\r\n" try: conn.send( bytes("HTTP/1.1 101 Web Socket Protocol Handshake\r\n", encoding="utf8")) conn.send(bytes("Upgrade: websocket\r\n", encoding="utf8")) conn.send(bytes(response_key_entity, encoding="utf8")) conn.send( bytes("Connection: Upgrade\r\n\r\n", encoding="utf8")) except Exception as e: Logger().warning(e) return self.__handshakes_dict[conn] = True ntif = NetNotify(sid, NeTDef.CONNECT.value) # 没能通过握手的连接不告诉主程序 self._out_queue.put(ntif) return # receive = conn.recv(2) # 130是二进制流,129是text if (not receive) or (len(receive) == 0): self.__close_conn(conn) return is_mask = receive[1] & 0x80 if is_mask == 0: Logger().warning('暂不支持不带mask的websocket.sid:{}'.format(sid)) self.__close_conn(conn) return len_tag = receive[1] & 0x7f if len_tag == 0: self.__close_conn(conn) return content_len = 0 #实际应该接收的内容长度 if len_tag <= 125: content_len = len_tag elif len_tag == 126: receive, recv_ok = self.__recv_msg(conn, 2) if recv_ok is False: Logger().warning('读取websocket长度出错:126.sid:{}'.format(sid)) self.__close_conn(conn) return content_len, = struct.unpack_from('!h', receive, 0) elif len_tag == 127: receive, recv_ok = self.receive_msg(conn, 8) if recv_ok is False: Logger().warning('读取websocket长度出错:127.sid:{}'.format(sid)) self.__close_conn(conn) return content_len, = struct.unpack_from('!q', receive, 0) else: Logger().warning( '读取websocket长度出错:too large.sid:{}'.format(sid)) self.__close_conn(conn) return mask, recv_ok = self.__recv_msg(conn, 4) if recv_ok is False: Logger().warning('读取websocketmask出错.sid:{}'.format(sid)) self.__close_conn(conn) return receive, recv_ok = self.__recv_msg(conn, content_len) if recv_ok is False: Logger().warning('读取websocket包内容出错.sid:{}'.format(sid)) self.__close_conn(conn) return byte_arr = bytearray() i = 0 for d in receive: byte_arr.append(d ^ mask[i % 4]) i += 1 data_bytes = bytes(byte_arr) if (len(data_bytes) == 0): self.__close_conn(conn) return if (len(data_bytes) == 2) and (data_bytes[0] == 0x03) and ( data_bytes[1] == 0xe9 or data_bytes[1] == 0xe8): self.__close_conn(conn) return ntif = NetNotify(sid, NeTDef.RECV.value, data_bytes) # 告诉主进程收到网络消息 self._out_queue.put(ntif) except ConnectionResetError as err: # 目标方突然强行关闭程序会发生,如果是正常关闭socket,不会产生异常 Logger().info('ConnectionResetError:{} {}'.format(sid, err)) self.__close_conn(conn) except Exception as e: Logger().info(sid, e) self.__close_conn(conn)