def _init_stream(**kwargs): """ 根据传入参数初始化流对象(实现类自定义,也可以是标识),基类将保留该对象并供后续流处理函数调用 @param {string} str_obj - 需进行流处理的字符串对象 @returns {object} - 返回具有两个属性的object对象: obj : string 流处理对象 pos : int 流当前位置 """ _stream_obj = NullObj() _stream_obj.obj = kwargs['str_obj'] _stream_obj.pos = 0 return _stream_obj
def _start_server_self(self, tid): """ 自定义服务启动函数,实现类可重载该函数加入自己的处理逻辑 重载该函数,实现网络监听服务的启动,但不接收连接 @param {int} tid - 线程id @returns {CResult} - 启动结果: result.code :'00000'-成功,其他值为失败 result.server_info :启动成功后的服务对象,用于传递到后续的服务处理函数 """ _result = CResult(code='00000') # 成功 _result.server_info = NullObj() with ExceptionTool.ignored_cresult(_result): # 可在该部分实现自定义逻辑 self._logger.log( self._log_level, '[%s-STARTING][NAME:%s]%s:\n%s' % ( self._server_log_prefix, self._server_name, _('net start parameter'), StringTool.format_obj_property_str(self._server_opts, is_deal_subobj=True) ) ) # 启动服务,但不接受连接 _result = self._start_server_without_accept(self._server_opts) _result.server_info = _result.net_info # 返回处理结果 return _result
def _create_connection_self(self): """ 创建一个连接对象(具体类实现) @return {object} - 返回有效的连接对象 @throws {Exception} - 当创建失败或连接无效时应直接抛出异常 """ if not TEST_SWITCH['create']: raise InterruptedError('_create_connection_self Interrupted!') if self.connect_para.prop1 != 'a' or self.connect_para.prop2 != 'b': raise ValueError('_create_connection_self error!') connection = NullObj() connection.name = 'TestPoolClass' connection.id = str(uuid.uuid1()) connection.status = True print('create connection id: %s' % connection.id) return connection
def _accept_one(self, server_opts, net_info): """ 监听接受一个请求并返回 提供监听并获取到请求连接返回的方法;注意该该函数必须捕获并处理异常 @param {objcet} server_opts - 网络服务启动参数 @param {objcet} net_info - 网络连接信息对象,_start_server_without_accept中获取到的结果 @returns {CResult} - 获取网络连接结果: result.code :'00000'-成功,'20407'-获取客户端连接请求超时 result.net_info :客户端连接信息对象,该对象将传给后续单个连接处理的线程 """ # 子类必须定义该功能 _result = CResult('00000') _result.net_info = None with ExceptionTool.ignored_cresult( _result, logger=self._logger, expect=(BlockingIOError), expect_no_log=True, # 超时不记录日志 error_map={BlockingIOError: ('20407', None)}, self_log_msg='[LIS][NAME:%s]%s error: ' % ( self._server_name, _('accept client connect')), force_log_level=None ): # _sys_str = platform.system() _csocket, _addr = net_info.csocket.accept() # 接收客户端连接,返回客户端和地址 _csocket.setblocking(False) # 将socket设置为非阻塞. 在创建socket对象后就进行该操作. _result.net_info = NullObj() _result.net_info.csocket = _csocket _result.net_info.raddr = _addr _result.net_info.laddr = _csocket.getsockname() _result.net_info.send_timeout = server_opts.send_timeout _result.net_info.recv_timeout = server_opts.recv_timeout # 采用非阻塞模式处理数据,超时自行实现 """ if (_sys_str == 'Windows'): _csocket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, server_opts.recv_timeout) _csocket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDTIMEO, server_opts.send_timeout) else: # linux 设置超时时间不同,需重新测试 _csocket.settimeout(server_opts.recv_timeout / 1000) """ self._logger.log( self._log_level, '[LIS][NAME:%s]%s: %s - %s' % ( self._server_name, _('accept one client connection'), str(_addr), str(_csocket) ) ) if not _result.is_success(): # 出现异常,睡眠一段时间 RunTool.sleep(0.01) return _result
def _start_server_without_accept(self, server_opts): """ 启动服务但不接受请求服务,该方法只做到启动端口层面,轮询监听不在该方法中实现: @param {object} server_opts - 服务参数 @returns {CResult} - 启动结果: result.code :'00000'-成功,其他值为失败 result.net_info :启动后的服务端网络连接信息对象,该对象将传给后续的监听线程(_AcceptOne),定义为: result.net_info.csocket - socket对象 result.net_info.laddr 本地地址,地址对象,("IP地址",打开端口) result.net_info.raddr 远端地址,地址对象,("IP地址",打开端口) result.net_info.send_timeout 发送超时时间,单位为毫秒 result.net_info.recv_timeout 收取超时时间,单位为毫秒 """ # 子类必须定义该功能 _result = CResult('00000') _result.net_info = None with ExceptionTool.ignored_cresult( _result, logger=self._logger, self_log_msg='[LIS-STARTING][NAME:%s]%s - %s error: ' % ( self._server_name, _('net service starting'), _('listen without accept')), force_log_level=logging.ERROR ): # _sys_str = platform.system() _server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) _server_socket.setblocking(False) # 将socket设置为非阻塞. 在创建socket对象后就进行该操作. _server_socket.bind((server_opts.ip, server_opts.port)) _server_socket.listen(server_opts.max_connect) # 改为用非阻塞模式支持,因此超时自行实现 """ if (_sys_str == 'Windows'): _server_socket.setsockopt( socket.SOL_SOCKET, socket.SO_RCVTIMEO, server_opts.recv_timeout) _server_socket.setsockopt( socket.SOL_SOCKET, socket.SO_SNDTIMEO, server_opts.send_timeout) else: # linux 设置超时时间不同,需重新测试 _server_socket.settimeout(server_opts.recv_timeout/1000) """ _result.net_info = NullObj() _result.net_info.laddr = _server_socket.getsockname() # _result.net_info.raddr = _server_socket.getpeername() _result.net_info.raddr = ('', 0) _result.net_info.csocket = _server_socket _result.net_info.send_timeout = server_opts.send_timeout _result.net_info.recv_timeout = server_opts.recv_timeout return _result
def _start_server_self(self, tid): """ 自定义服务启动函数,实现类可重载该函数加入自己的处理逻辑 @param {int} tid - 线程id @returns {CResult} - 启动结果: result.code :'00000'-成功,其他值为失败 result.server_info :启动成功后的服务对象,用于传递到后续的服务处理函数 """ _result = CResult(code='00000') # 成功 _result.server_info = NullObj() with ExceptionTool.ignored_cresult(_result): # 可在该部分实现自定义逻辑 pass # 返回处理结果 return _result
def connect_server(cls, connect_para): """ 客户端通过该函数连接服务器端 @param {object} connect_para - 需要连接服务器的参数,与server_opts一致 @returns {CResult} - 连接结果: result.code :'00000'-成功,其他值为失败 result.net_info : 连接后的网络信息对象 """ # 子类必须定义该功能 _result = CResult('00000') _result.net_info = None with ExceptionTool.ignored_cresult( _result, logger=None ): # _sys_str = platform.system() _tcp_cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # 分配 TCP 客户端套接字 _tcp_cli_sock.connect((connect_para.ip, connect_para.port)) # 主动连接 _tcp_cli_sock.setblocking(False) # 将socket设置为非阻塞. 在创建socket对象后就进行该操作. # 转换为非阻塞模式,自行控制超时 """ if (_sys_str == 'Windows'): _tcp_cli_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, connect_para.recv_timeout) _tcp_cli_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDTIMEO, connect_para.send_timeout) else: # linux 设置超时时间不同,需重新测试 _tcp_cli_sock.settimeout(connect_para.recv_timeout / 1000) """ _result.net_info = NullObj() _result.net_info.csocket = _tcp_cli_sock _result.net_info.laddr = _tcp_cli_sock.getsockname() _result.net_info.raddr = _tcp_cli_sock.getpeername() _result.net_info.send_timeout = connect_para.send_timeout _result.net_info.recv_timeout = connect_para.recv_timeout return _result
def generate_server_opts(ip='', port=8080, max_connect=20, recv_timeout=10000, send_timeout=10000): """ 生成默认服务启动参数 @param {string} ip='' - 主机名或IP地址 @param {int} port=8080 - 监听端口 @param {int} max_connect=20 - 允许最大连接数 @param {int} recv_timeout=10000 - 数据接收的超时时间,单位为毫秒 @param {int} send_timeout=10000 - 数据发送的超时时间,单位为毫秒 @returns {object} - 返回带参数属性的对象,例如对象为ret: ret.ip = '' ... """ _server_opts = NullObj() _server_opts.ip = ip # 主机名或IP地址 _server_opts.port = port # 监听端口 _server_opts.max_connect = max_connect # 允许最大连接数 _server_opts.recv_timeout = recv_timeout # 数据接收的超时时间,单位为毫秒 _server_opts.send_timeout = send_timeout # 数据发送的超时时间,单位为毫秒 return _server_opts
def _create_connection(self): """ 创建一个新的连接对象 @return {string} - 返回连接对象的id @throws {Exception} - 当创建失败或连接无效时应直接抛出异常 """ # 调用实现类创建新的连接对象 _connection = self._create_connection_self() _cid = str(uuid.uuid1()) _connection.connection_pool_id = _cid # 生成内部的连接信息对象 _connection_info = NullObj() _connection_info.connection = _connection _connection_info.is_using = False _connection_info.last_free_time = datetime.datetime.now() _connection_info.last_test_time = datetime.datetime.now() # 放入连接池 self._connection_pool[_cid] = _connection_info # 返回连接id return _cid
def test_case1(self): print("测试案例1") print('测试初始化时异常-连接失败') _connect_para = NullObj() _connect_para.prop1 = 'a' _connect_para.prop2 = 'c' try: _pool = TestPoolClass( _connect_para, name='ConnectionPool', maxsize=5, minsize=2, realse_free_time=10, test_on_get=True, test_on_free=True, test_while_idle=False, test_idle_time=60, validation_query='', get_connection_timeout=1, logger=self._logger, init_break_if_connect_error=True ) self.assertTrue(False, '期望初始化异常,但未抛出') except Exception as e: print('测试抛出初始化异常成功: %s' % str(e)) print('测试正常获取连接') _connect_para = NullObj() _connect_para.prop1 = 'a' _connect_para.prop2 = 'b' _pool = TestPoolClass( _connect_para, name='ConnectionPool', maxsize=5, minsize=1, realse_free_time=10, test_on_get=True, test_on_free=True, test_while_idle=True, test_idle_time=5, validation_query='', get_connection_timeout=1, logger=self._logger, init_break_if_connect_error=True ) c_dict = dict() c_dict['1'] = _pool.get_connection() print('current_size: %d' % _pool.current_size) self.assertTrue( c_dict['1'].name == 'TestPoolClass', msg='测试正常获取连接失败 - 获取第1个连接信息错误!' ) print('前面获取的是已有连接,后面获取才创建新连接') _i = 2 while _i <= 5: c_dict[str(_i)] = _pool.get_connection() _i += 1 print('current_size: %d' % _pool.current_size) print('测试连接超过以后获取超时') try: c_dict['6'] = _pool.get_connection() self.assertTrue( False, msg='测试连接超过以后获取超时失败 - 期望抛出超时异常!' ) except TimeoutError: pass except: self.assertTrue( False, msg='测试连接超过以后获取超时失败,错误信息: %s' % str(sys.exc_info()) ) print('current_size: %d' % _pool.current_size) print('测试释放连接') _pool.free_connection(c_dict['1']) print('current_size: %d' % _pool.current_size) self.assertTrue( _pool.current_size == 5 and _pool.free_pool_size == 1, msg='测试释放连接失败1 - 当前连接池大小及空闲池大小错误:%d %d' % (_pool.current_size, _pool.free_pool_size) ) _pool.free_connection(c_dict['2']) self.assertTrue( _pool.current_size == 5 and _pool.free_pool_size == 2, msg='测试释放连接失败2 - 当前连接池大小及空闲池大小错误:%d %d' % (_pool.current_size, _pool.free_pool_size) ) c_dict['1'] = _pool.get_connection() c_dict['2'] = _pool.get_connection() try: c_dict['6'] = _pool.get_connection() self.assertTrue( False, msg='测试释放连接失败 - 释放后再获取当超过最大大小时,期望抛出超时异常!' ) except TimeoutError: pass except: self.assertTrue( False, msg='测试释放连接失败,释放后再获取当超过最大大小时应超时,错误信息: %s' % str(sys.exc_info()) ) print('测试自动释放空闲连接') _i = 1 while _i <= 5: _pool.free_connection(c_dict[str(_i)]) _i += 1 print('等待15秒释放') time.sleep(15) self.assertTrue( _pool.current_size == 1 and _pool.free_pool_size == 1, msg='测试自动释放空闲连接 - 当前连接池大小及空闲池大小错误:%d %d' % (_pool.current_size, _pool.free_pool_size) ) print('测试获取连接出现异常的情况') TEST_SWITCH['create'] = False # 生成连接异常 c_dict['1'] = _pool.get_connection() try: c_dict['2'] = _pool.get_connection() self.assertTrue( False, msg='测试获取连接出现异常的情况 - 期望抛出异常!' ) except: pass self.assertTrue( _pool.current_size == 1 and _pool.free_pool_size == 0, msg='测试自动释放空测试获取连接出现异常的情况,失败 - 当前连接池大小及空闲池大小错误:%d %d' % ( _pool.current_size, _pool.free_pool_size) ) TEST_SWITCH['create'] = True print('测试释放连接时检查失败的情况') TEST_SWITCH['test'] = False _pool.free_connection(c_dict['1']) self.assertTrue( _pool.current_size == 0 and _pool.free_pool_size == 0, msg='测试释放连接时检查失败的情况,失败 - 当前连接池大小及空闲池大小错误:%d %d' % ( _pool.current_size, _pool.free_pool_size) ) print('测试连接时检查失败的情况') try: c_dict['1'] = _pool.get_connection() self.assertTrue( False, msg='测试连接时检查失败的情况 - 期望抛出异常!' ) except: pass self.assertTrue( _pool.current_size == 0 and _pool.free_pool_size == 0, msg='测试连接时检查失败的情况,失败 - 当前连接池大小及空闲池大小错误:%d %d' % ( _pool.current_size, _pool.free_pool_size) ) TEST_SWITCH['test'] = True print('测试空闲时检查连接的情况') _i = 1 while _i <= 5: c_dict[str(_i)] = _pool.get_connection() _i += 1 _i = 1 while _i <= 5: _pool.free_connection(c_dict[str(_i)]) _i += 1 self.assertTrue( _pool.current_size == 5 and _pool.free_pool_size == 5, msg='测试空闲时检查连接的情况,失败 - 当前连接池大小及空闲池大小错误:%d %d' % ( _pool.current_size, _pool.free_pool_size) ) print('等待8秒,让空闲连接检查连接有效性') TEST_SWITCH['test'] = False time.sleep(8) self.assertTrue( _pool.current_size == 0 and _pool.free_pool_size == 0, msg='测试空闲时检查连接的情况,失败 - 当前连接池大小及空闲池大小错误:%d %d' % ( _pool.current_size, _pool.free_pool_size) )
def generate_connect_para(ip='', port=50051, conn_str=None, timeout=None, is_use_ssl=False, root_certificates=None, private_key=None, certificate_chain=None, options=None, compression=None, test_on_connect=False, test_use_health_check=False, servicer_name='', logger=None, log_level=logging.INFO, is_use_global_logger=True, idpool=None, get_id_overtime=0, send_logging_para={}, back_logging_para={}, **kwargs): """ 生成客户端连接参数 @param {string} ip='' - 要连接的服务器IP 注意:TSLSSL模式下,客户端是通过"服务名称:port"来获取服务的凭据,而不是"ip:port", 如果使用TSL/SSL的情况客户端连接失败,可从这个角度排查解决问题 @param {int} port=50051 - 要连接的服务器端口 @param {conn_str} conn_str=None - 连接字符串,如果传入该字符串则不再使用ip和端口方式连接 连接字符串的格式如下:'ip协议(ipv4|ipv6):///ip1:port1,ip2:port2,...' 例如"ipv4:///1.2.3.4:9999,1.2.3.5:9999,1.2.3.6:9999" "ipv6:///[1::2]:9999,[1::3]:9999,[1::4]:9999" @param {number} timeout=None - 超时时间,单位为秒 @param {bool} is_use_ssl=False - 是否使用SSL/TLS @param {bytes} root_certificates=None - 用于验证服务器证书的根证书,即服务器端的公钥证书 The PEM-encoded root certificates as a byte string with open('ca.crt', 'rb') as f: root_certificates = f.read() @param {bytes} private_key=None - 当反向认证时(服务器验证客户端证书),客户端的私钥文件 The PEM-encoded private key as a byte string with open('server.pem', 'rb') as f: private_key = f.read() @param {bytes} certificate_chain=None - 当反向认证时(服务器验证客户端证书),客户端的公钥证书文件 The PEM-encoded certificate chain as a byte string with open('server.crt', 'rb') as f: certificate_chain = f.read() @param {?} options=None - An optional list of key-value pairs (channel args in gRPC Core runtime) to configure the channel @param {?} compression=None - An optional value indicating the compression method to be used over the lifetime of the channel @param {bool} test_on_connect=False - 连接时进行有效性测试 @param {bool} test_use_health_check=False - 使用标准的health_check进行测试 @param {string} servicer_name='' - 使用health_check进行测试的对应服务名(由服务端定义) @param {Logger} logger=None - 日志对象,服务过程中通过该函数写日志: 可以为标准的logging日志库对象,也可以为simple_log对象,但要求对象实现: 标准的info、debug、warning、error、critical五个日志方法 @param {int} log_level=logging.INFO - 处理中正常日志的输出登记级别,默认为INFO,如果不想输出 过多日志可以设置为DEBUG @param {bool} is_use_global_logger=True - 当logger=None时,是否使用全局logger对象 注:通过RunTool.set_global_logger进行设置 @param {HiveNetLib.IdPool} idpool=None - 获取id的资源池,如果传入None代表直接通过uuid生成id @param {number} get_id_overtime=0 - 超时时间,单位为秒,如果需要一直不超时送入0 @param {dict} send_logging_para={} - 接收报文打印参数 @param {dict} back_logging_para={} - 返回报文打印参数 send_logging_para的参数格式一致,定义如下: 'msg_class' {class} - 继承MsgFW框架的报文解析类对象,如果为None代表不处理信息 'logging_head' {dict}- 定义打印的日志规范头信息 key {string} - 日志头信息项名,例如'IP' value {string} - 日志头信息值,None代表从报文对象msg或proto_msg中获取(从api_mapping获取定义) 跟当前服务相关的可选信息项包括: C-IP : 客户端的IP地址 C-PORT : 客户端的连接端口 S-IP : 服务端绑定服务 S-PORT : 服务端监听端口 SERVICE_NAME : 访问的服务名 PARA_BYTES : 转换为字符串显示的参数字节数组信息 PARA_BYTES_LEN : 字节数组长度 RETURN_BYTES : 转换为字符串显示的响应字节数组信息 RETURN_BYTES_LEN : 响应报文字节数组长度 'api_mapping' {dict}- 定义从报文中获取logging_head所需的信息 'key_para' {dict} - 要打印的关键业务参数 'print_in_para' {dict} - 要打印的指定接口字段 以上三项的定义都是一样 key {string} - 打印信息项名 value {list}- 映射信息,为三项的数组: value[0] {string} - 获取api对象类型,'msg'或'proto_msg' value[1] {string} - 搜索路径,具体规则参考对应的MsgFW实例 value[2] {dict} - 获取参数,具体规则参考对应的MsgFW实例 'is_print_msg' {bool} - 是否打印报文内容 'msg_print_kwargs' {dict} - MsgFW对象(例如MsgJSON)的msg.to_str()函数的传入参数 @param {kwargs} - 动态参数,已定义的参数如下: id的资源池的get_id传入参数 @returns {object} - 返回带参数属性的对象,例如对象为ret: ret.ip = '' ... """ _connect_para = NullObj() _connect_para.ip = ip _connect_para.port = port _connect_para.conn_str = conn_str if conn_str == '': _connect_para.conn_str = None _connect_para.timeout = timeout if timeout is not None and timeout <= 0: _connect_para.timeout = None _connect_para.is_use_ssl = is_use_ssl # 是否使用SSL/TLS _connect_para.private_key = private_key _connect_para.certificate_chain = certificate_chain _connect_para.root_certificates = root_certificates _connect_para.options = options _connect_para.compression = compression _connect_para.test_on_connect = test_on_connect _connect_para.test_use_health_check = test_use_health_check _connect_para.servicer_name = servicer_name _connect_para.logger = logger _connect_para.log_level = log_level _connect_para.is_use_global_logger = is_use_global_logger _connect_para.idpool = idpool _connect_para.get_id_overtime = get_id_overtime _connect_para.send_logging_para = send_logging_para _connect_para.back_logging_para = back_logging_para _connect_para.kwargs = kwargs return _connect_para
def _get_trace_info(self, **kwargs): """ 获取调用链信息 @param {**kwargs} kwargs - 动态参数,用于支持调用链信息 @return {NullObj} - 从kwargs获取信息处理后的调用链信息 """ _trace_info = NullObj() _trace_info.trace_id = '' _trace_info.parent_id = '' _trace_info.trace_level = 0 _trace_info.call_id = CallChainTool.generate_trace_id( idpool=self._connect_para.idpool, get_id_overtime=self._connect_para.get_id_overtime, **self._connect_para.kwargs) # 当前函数的执行id if 'trace_id' in kwargs.keys(): _trace_info.trace_id = kwargs['trace_id'] if 'parent_id' in kwargs.keys(): _trace_info.parent_id = kwargs['parent_id'] if 'trace_level' in kwargs.keys(): _trace_info.trace_level = kwargs['trace_level'] if _trace_info.trace_id == '': # 上送请求没有调用链,则链从自己开始 _trace_info.trace_id = _trace_info.call_id _trace_info.trace_level = 0 else: _trace_info.trace_level = _trace_info.trace_level + 1 # 返回信息 return _trace_info