def doResponse(self): ''' @brief: 处理接收的数据 @return: 返回响应报文的列表,如果出错返回None @rtype: list: ResponsePacket ''' tarsLogger.debug('TcpTransceiver:doResponse') if not self.isValid(): return None bufs = [self._recvBuf] while True: buf = self.recv(8292) if not buf: break bufs.append(buf) self._recvBuf = ''.join(bufs) tarsLogger.info('tcp doResponse, fd: %d, recvbuf: %d', self.getFd(), len(self._recvBuf)) if not self._recvBuf: return None rsplist = None try: rsplist, bufsize = ReqMessage.unpackRspList(self._recvBuf) self._recvBuf = self._recvBuf[bufsize:] except Exception, msg: tarsLogger.error( 'tcp doResponse, fd: %d, %s, tcp recv unpack error: %s', self.getFd(), self.getEndPointInfo(), msg) self.close()
def sendRequest(self): ''' @brief: 把队列中的请求放到Transceiver的发送缓存里 @return: 放入缓存的数据长度 @rtype: int ''' tarsLogger.debug('AdapterProxy:sendRequest') if not self.__trans.hasConnected(): return False reqmsg = self.__object.popRequest() blen = 0 while reqmsg: reqmsg.adapter = self buf = reqmsg.packReq() self.__trans.writeToSendBuf(buf) tarsLogger.info('sendRequest, id: %d, len: %d', reqmsg.request.iRequestId, len(buf)) blen += len(buf) # 合并一次发送的包 最大合并至8k 提高异步时客户端效率? if (self.__trans.getEndPointInfo().getConnType() == EndPointInfo.SOCK_UDP or blen > 8192): break reqmsg = self.__object.popRequest() return blen
def send(self, buf, flag = 0): ''' @brief: 实现tcp的发送 @param buf: 发送的数据 @type buf: str @param flag: 发送标志 @param flag: int @return: 发送字节数 @rtype: int ''' tarsLogger.debug('TcpTransceiver:send') if not self.isValid(): return -1 nbytes = 0 try: nbytes = self.getSock().send(buf, flag) tarsLogger.info('tcp send, fd: %d, %s, len: %d', self.getFd(), self.getEndPointInfo(), nbytes) except socket.error, msg: if msg.errno != errno.EAGAIN: tarsLogger.error('tcp send, fd: %d, %s, fail!, %s, close', self.getFd(), self.getEndPointInfo(), msg) self.close() return 0
def recv(self, bufsize, flag = 0): ''' @brief: 实现tcp的recv @param bufsize: 接收大小 @type bufsize: int @param flag: 接收标志 @param flag: int @return: 接收的内容,接收出错返回None @rtype: str ''' tarsLogger.debug('TcpTransceiver:recv') assert(self.isValid()) buf = '' try: buf = self.getSock().recv(bufsize, flag) if len(buf) == 0: tarsLogger.info('tcp recv, fd: %d, %s, recv 0 bytes, close', self.getFd(), self.getEndPointInfo()) self.close() return None except socket.error, msg: if msg.errno != errno.EAGAIN: tarsLogger.info('tcp recv, fd: %d, %s, faild!, %s, close', self.getFd(), self.getEndPointInfo(), msg) self.close() return None
def doReconnect(self): ''' @brief: 重新发起连接 @return: None @rtype: None ''' tarsLogger.debug('AdapterProxy:doReconnect') assert (self.__trans) self.__trans.reInit() tarsLogger.info('doReconnect, connect: %s, fd:%d', self.__trans.getEndPointInfo(), self.__trans.getFd()) self.__reactor.registerAdapter(self, select.EPOLLIN | select.EPOLLOUT)
def doReconnect(self): ''' @brief: 重新发起连接 @return: None @rtype: None ''' tarsLogger.debug('AdapterProxy:doReconnect') assert(self.__trans) self.__trans.reInit() tarsLogger.info('doReconnect, connect: %s, fd:%d', self.__trans.getEndPointInfo(), self.__trans.getFd()) self.__reactor.registerAdapter(self, select.EPOLLIN | select.EPOLLOUT)
def close(self): ''' @brief: 关闭连接 @return: None @rtype: None @note: 多次调用不会有问题 ''' tarsLogger.debug('Transceiver:close') if not self.isValid(): return self.__sock.close() self.__sock = None self.__connStatus = Transceiver.UNCONNECTED self.__connFailed = False self._sendBuff = '' self._recvBuf = '' tarsLogger.info('trans close : %s' % self.__epi)
def checkActive(self, forceConnect=False): ''' @brief: 检测连接是否失效 @param forceConnect: 是否强制发起连接,为true时不对状态进行判断就发起连接 @type forceConnect: bool @return: 连接是否有效 @rtype: bool ''' tarsLogger.debug('AdapterProxy:checkActive') self.__lock.acquire() tarsLogger.info('checkActive, %s, forceConnect: %s', self.__trans.getEndPointInfo(), forceConnect) if not self.__trans.isConnecting() and not self.__trans.hasConnected(): self.doReconnect() self.__lock.release() return self.__trans.isConnecting() or self.__trans.hasConnected()
def checkActive(self, forceConnect = False): ''' @brief: 检测连接是否失效 @param forceConnect: 是否强制发起连接,为true时不对状态进行判断就发起连接 @type forceConnect: bool @return: 连接是否有效 @rtype: bool ''' tarsLogger.debug('AdapterProxy:checkActive') self.__lock.acquire() tarsLogger.info('checkActive, %s, forceConnect: %s', self.__trans.getEndPointInfo(), forceConnect) if not self.__trans.isConnecting() and not self.__trans.hasConnected(): self.doReconnect() self.__lock.release() return self.__trans.isConnecting() or self.__trans.hasConnected()
def reInit(self): ''' @brief: 初始化socket,并连接服务器 @return: 成功返回0,失败返回-1 @rtype: int ''' tarsLogger.debug('Transceiver:reInit') assert (self.isValid() == False) if self.__epi.getConnType() != EndPointInfo.SOCK_TCP: return -1 try: self.__sock = socket.socket() self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.__sock.setblocking(0) self.__sock.connect((self.__epi.getIp(), self.__epi.getPort())) self.__connStatus = Transceiver.CONNECTED except socket.error, msg: if msg.errno == errno.EINPROGRESS: self.__connStatus = Transceiver.CONNECTING else: tarsLogger.info('reInit, %s, faild!, %s', self.__epi, msg) self.__sock = None return -1
def reInit(self): ''' @brief: 初始化socket,并连接服务器 @return: 成功返回0,失败返回-1 @rtype: int ''' tarsLogger.debug('Transceiver:reInit') assert(self.isValid() == False) if self.__epi.getConnType() != EndPointInfo.SOCK_TCP: return -1 try: self.__sock = socket.socket() self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.__sock.setblocking(0) self.__sock.connect((self.__epi.getIp(), self.__epi.getPort())) self.__connStatus = Transceiver.CONNECTED except socket.error, msg: if msg.errno == errno.EINPROGRESS: self.__connStatus = Transceiver.CONNECTING else: tarsLogger.info('reInit, %s, faild!, %s', self.__epi, msg) self.__sock = None return -1
class AdapterProxy: ''' @brief: 每一个Adapter管理一个服务端端口的连接,数据收发 ''' def __init__(self): tarsLogger.debug('AdapterProxy:__init__') self.__closeTrans = False self.__trans = None self.__object = None self.__reactor = None self.__lock = None self.__asyncProc = None def __del__(self): tarsLogger.debug('AdapterProxy:__del__') def initialize(self, endPointInfo, objectProxy, reactor, asyncProc): ''' @brief: 初始化 @param endPointInfo: 连接对端信息 @type endPointInfo: EndPointInfo @type objectProxy: ObjectProxy @type reactor: FDReactor @type asyncProc: AsyncProcThread ''' tarsLogger.debug('AdapterProxy:initialize') self.__closeTrans = False self.__trans = TcpTransceiver(endPointInfo) self.__object = objectProxy self.__reactor = reactor self.__lock = threading.Lock() self.__asyncProc = asyncProc def terminate(self): ''' @brief: 关闭 ''' tarsLogger.debug('AdapterProxy:terminate') self.setCloseTrans(True) def trans(self): ''' @brief: 获取传输类 @return: 负责网络传输的trans @rtype: Transceiver ''' return self.__trans def invoke(self, reqmsg): ''' @brief: 远程过程调用处理方法 @param reqmsg: 请求响应报文 @type reqmsg: ReqMessage @return: 错误码:0表示成功,-1表示连接失败 @rtype: int ''' tarsLogger.debug('AdapterProxy:invoke') assert (self.__trans) if (not self.__trans.hasConnected() and not self.__trans.isConnecting): # -1表示连接失败 return -1 reqmsg.request.iRequestId = self.__object.getTimeoutQueue().generateId( ) self.__object.getTimeoutQueue().push(reqmsg, reqmsg.request.iRequestId) self.__reactor.notify(self) return 0 def finished(self, rsp): ''' @brief: 远程过程调用返回处理 @param rsp: 响应报文 @type rsp: ResponsePacket @return: 函数是否执行成功 @rtype: bool ''' tarsLogger.debug('AdapterProxy:finished') reqmsg = self.__object.getTimeoutQueue().pop(rsp.iRequestId) if not reqmsg: tarsLogger.error( 'finished, can not get ReqMessage, may be timeout, id: %d', rsp.iRequestId) return False reqmsg.response = rsp if reqmsg.type == ReqMessage.SYNC_CALL: return reqmsg.servant._finished(reqmsg) elif reqmsg.callback: self.__asyncProc.put(reqmsg) return True tarsLogger.error( 'finished, adapter proxy finish fail, id: %d, ret: %d', rsp.iRequestId, rsp.iRet) return False # 检测连接是否失败,失效时重连 def checkActive(self, forceConnect=False): ''' @brief: 检测连接是否失效 @param forceConnect: 是否强制发起连接,为true时不对状态进行判断就发起连接 @type forceConnect: bool @return: 连接是否有效 @rtype: bool ''' tarsLogger.debug('AdapterProxy:checkActive') self.__lock.acquire() tarsLogger.info('checkActive, %s, forceConnect: %s', self.__trans.getEndPointInfo(), forceConnect) if not self.__trans.isConnecting() and not self.__trans.hasConnected(): self.doReconnect() self.__lock.release() return self.__trans.isConnecting() or self.__trans.hasConnected() def doReconnect(self): ''' @brief: 重新发起连接 @return: None @rtype: None ''' tarsLogger.debug('AdapterProxy:doReconnect') assert (self.__trans) self.__trans.reInit() tarsLogger.info('doReconnect, connect: %s, fd:%d', self.__trans.getEndPointInfo(), self.__trans.getFd()) self.__reactor.registerAdapter(self, select.EPOLLIN | select.EPOLLOUT) def sendRequest(self): ''' @brief: 把队列中的请求放到Transceiver的发送缓存里 @return: 放入缓存的数据长度 @rtype: int ''' tarsLogger.debug('AdapterProxy:sendRequest') if not self.__trans.hasConnected(): return False reqmsg = self.__object.popRequest() blen = 0 while reqmsg: reqmsg.adapter = self buf = reqmsg.packReq() self.__trans.writeToSendBuf(buf) tarsLogger.info('sendRequest, id: %d, len: %d', reqmsg.request.iRequestId, len(buf)) blen += len(buf) # 合并一次发送的包 最大合并至8k 提高异步时客户端效率? if (self.__trans.getEndPointInfo().getConnType() == EndPointInfo.SOCK_UDP or blen > 8192): break reqmsg = self.__object.popRequest() return blen def finishConnect(self): ''' @brief: 使用的非阻塞socket连接不能立刻判断是否连接成功, 在epoll响应后调用此函数处理connect结束后的操作 @return: 是否连接成功 @rtype: bool ''' tarsLogger.debug('AdapterProxy:finishConnect') success = True errmsg = '' try: ret = self.__trans.getSock().getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if ret: success = False errmsg = os.strerror(ret) except Exception, msg: errmsg = msg success = False if not success: self.__reactor.unregisterAdapter(self, socket.EPOLLIN | socket.EPOLLOUT) self.__trans.close() self.__trans.setConnFailed() tarsLogger.error( 'AdapterProxy finishConnect, exception: %s, error: %s', self.__trans.getEndPointInfo(), errmsg) return False self.__trans.setConnected() self.__reactor.notify(self) tarsLogger.info('AdapterProxy finishConnect, connect %s success', self.__trans.getEndPointInfo()) return True
class Transceiver: ''' @brief: 网络传输基类,提供网络send/recv接口 ''' CONNECTED = 0 CONNECTING = 1 UNCONNECTED = 2 def __init__(self, endPointInfo): tarsLogger.debug('Transceiver:__init__, %s', endPointInfo) self.__epi = endPointInfo self.__sock = None self.__connStatus = Transceiver.UNCONNECTED self.__connFailed = False # 这两个变量要给子类用,不能用name mangling隐藏 self._sendBuff = '' self._recvBuf = '' def __del__(self): tarsLogger.debug('Transceiver:__del__') self.close() def getSock(self): ''' @return: socket对象 @rtype: socket.socket ''' return self.__sock def getFd(self): ''' @brief: 获取socket的文件描述符 @return: 如果self.__sock没有建立返回-1 @rtype: int ''' if self.__sock: return self.__sock.fileno() else: return -1 def getEndPointInfo(self): ''' @return: 端口信息 @rtype: EndPointInfo ''' return self.__epi def isValid(self): ''' @return: 是否创建了socket @rtype: bool ''' return self.__sock != None def hasConnected(self): ''' @return: 是否连接上了 @rtype: bool ''' return self.isValid() and self.__connStatus == Transceiver.CONNECTED def isConnFailed(self): ''' @return: 是否连接失败 @rtype: bool ''' return self.__connFailed def isConnecting(self): ''' @return: 是否正在连接 @rtype: bool ''' return self.isValid() and self.__connStatus == Transceiver.CONNECTING def setConnFailed(self): ''' @brief: 设置为连接失败 @return: None @rtype: None ''' self.__connFailed = True self.__connStatus = Transceiver.UNCONNECTED def setConnected(self): ''' @brief: 设置为连接完 @return: None @rtype: None ''' self.__connFailed = False self.__connStatus = Transceiver.CONNECTED def close(self): ''' @brief: 关闭连接 @return: None @rtype: None @note: 多次调用不会有问题 ''' tarsLogger.debug('Transceiver:close') if not self.isValid(): return self.__sock.close() self.__sock = None self.__connStatus = Transceiver.UNCONNECTED self.__connFailed = False self._sendBuff = '' self._recvBuf = '' tarsLogger.info('trans close : %s' % self.__epi) def writeToSendBuf(self, msg): ''' @brief: 把数据添加到send buffer里 @param msg: 发送的数据 @type msg: str @return: None @rtype: None @note: 没有加锁,多线程调用会有race conditions ''' self._sendBuff += msg def recv(self, bufsize, flag = 0): raise NotImplementedError() def send(self, buf, flag = 0): raise NotImplementedError() def doResponse(self): raise NotImplementedError() def doRequest(self): ''' @brief: 将请求数据发送出去 @return: 发送的字节数 @rtype: int ''' tarsLogger.debug('Transceiver:doRequest') if not self.isValid(): return -1 nbytes = 0 buf = buffer(self._sendBuff) while True: if not buf: break ret = self.send(buf[nbytes:]) if ret > 0: nbytes += ret else: break self._sendBuff = buf[nbytes:] return nbytes def reInit(self): ''' @brief: 初始化socket,并连接服务器 @return: 成功返回0,失败返回-1 @rtype: int ''' tarsLogger.debug('Transceiver:reInit') assert(self.isValid() == False) if self.__epi.getConnType() != EndPointInfo.SOCK_TCP: return -1 try: self.__sock = socket.socket() self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.__sock.setblocking(0) self.__sock.connect((self.__epi.getIp(), self.__epi.getPort())) self.__connStatus = Transceiver.CONNECTED except socket.error, msg: if msg.errno == errno.EINPROGRESS: self.__connStatus = Transceiver.CONNECTING else: tarsLogger.info('reInit, %s, faild!, %s', self.__epi, msg) self.__sock = None return -1 tarsLogger.info('reInit, connect: %s, fd: %d', self.__epi, self.getFd()) return 0
buf = '' try: buf = self.getSock().recv(bufsize, flag) if len(buf) == 0: tarsLogger.info('tcp recv, fd: %d, %s, recv 0 bytes, close', self.getFd(), self.getEndPointInfo()) self.close() return None except socket.error, msg: if msg.errno != errno.EAGAIN: tarsLogger.info('tcp recv, fd: %d, %s, faild!, %s, close', self.getFd(), self.getEndPointInfo(), msg) self.close() return None tarsLogger.info('tcp recv, fd: %d, %s, nbytes: %d', self.getFd(), self.getEndPointInfo(), len(buf)) return buf def doResponse(self): ''' @brief: 处理接收的数据 @return: 返回响应报文的列表,如果出错返回None @rtype: list: ResponsePacket ''' tarsLogger.debug('TcpTransceiver:doResponse') if not self.isValid(): return None bufs = [self._recvBuf] while True: buf = self.recv(8292)