def test_mutiple_thread(self): if not TEST_FLAG['test_mutiple_thread']: return print("测试同一个连接多线程并发") # 建立连接 _connect_para = SimpleGRpcConnection.generate_connect_para( conn_str='127.0.0.1:50051' ) _connection = SimpleGRpcConnection(_connect_para) # 创建多线程执行 _result = list() for _i in range(80): _result.append(False) _call_thread = threading.Thread( target=self._mutiple_thread_fun, args=(_i, _connection, _result), name='Thread-gRpc-call' ) _call_thread.setDaemon(True) _call_thread.start() # 等待线程执行完成 time.sleep(20) _connection.close() # 检查结果 for _res_i in _result: if not _res_i: self.assertTrue(False, '测试同一个连接多线程并发失败')
def open_writer(self, index: int = 0) -> dict: """ 打开写入对象并返回对象属性 @param {int} index=0 - 当前的数据处理线程索引 @returns {dict} - 打开的文件对象属性字典 {'handle': 写入对象, 'close_able': 是否可关闭, lock: 锁对象} """ self._writer_handles_lock.acquire() try: if index not in self._writer_handles.keys(): # 新增写入对象属性(远程连接对象) self._writer_handles[index] = { 'handle': SimpleGRpcConnection(self.connect_para), 'close_able': True, 'lock': threading.RLock(), 'req_queue': queue.Queue(), # 用于向远端请求文件数据的队列 'ret_queue': queue.Queue(), # 用于获取远端请求文件数据的队列 } # 启动后台线程监控数据读取请求 _write_data_thread = threading.Thread( target=self._write_data_thread_fun, args=(index, ), name='Thread-gRpc-Write-Data') _write_data_thread.setDaemon(True) _write_data_thread.start() # 返回打开的写入对象 return self._writer_handles[index] finally: self._writer_handles_lock.release()
def _grpc_call(self, service_name: str, para_list: list, is_support_bytes=False): """ 调用远程grpc服务 注:该函数不处理返回对象中的bytes信息 @param {str} service_name - 要调用的服务名 @param {list} para_list - 要转换的参数列表,list的每一项为一个参数项,每个参数项如下: [para_name, call_value] : para_name可以为'' @param {bool} is_support_bytes=False - 是否支持参数中的字节数组传递(只支持最后一个参数) @returns {object} - 返回远程调用返回的对象 @throws {RuntimeError} - 如果远程调用出现错误抛出异常 """ # 生成请求对象 _para_obj = SimpleGRpcTools.parameters_to_json( para_list, is_support_bytes=is_support_bytes) _req_obj = SimpleGRpcTools.generate_request_obj( service_name=service_name, para_json=_para_obj.para_json, has_para_bytes=_para_obj.has_para_bytes, para_bytes=_para_obj.para_bytes) # 连接并请求服务 _connection = SimpleGRpcConnection(self.connect_para) try: _cresult: CResult = _connection.call(_req_obj) if not _cresult.is_success(): # 执行失败,抛出异常 raise RuntimeError('call %s error: %s' % (service_name, str(_cresult))) # 处理返回值 if _cresult.return_json == '': return None else: return json.loads(_cresult.return_json) finally: _connection.close()
def server_bidirectional_stream_call_one_by_one(): """ 双向流模式,客户端处理,一个请求对应一个响应的情况 通过队列传递交互传递参数 """ # 清空队列 TEMP_QUEUE.queue.clear() TEMP_QUEUE.put(None) # 处理 _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.1', port=50051 ) _connection = SimpleGRpcConnection(_connect_para) _cresult_iterator = _connection.call( bidirectional_stream_one_by_one_generator(), call_mode=EnumCallMode.BidirectionalStream) for _cresult in _cresult_iterator: # 放入队列 s = _cresult.return_json TEMP_QUEUE.put(s) if not _cresult.is_success(): return False _connection.close() return True
def server_side_stream_call(a, b, *args, **kwargs): """ 服务端流模式,真正调用服务端的函数 """ # 转换参数 _para_values = RunTool.get_current_function_parameter_values(is_simple_mode=True) _para_obj = SimpleGRpcTools.parameters_to_json(_para_values) _req_obj = SimpleGRpcTools.generate_request_obj( service_name='service_server_side_stream', para_json=_para_obj.para_json, has_para_bytes=_para_obj.has_para_bytes, para_bytes=_para_obj.para_bytes ) # 发送请求, 默认使用全局的日志对象 _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.1', port=50051 ) _connection = SimpleGRpcConnection(_connect_para) _cresult_iterator = _connection.call(_req_obj, call_mode=EnumCallMode.ServerSideStream) _ret = True for _cresult in _cresult_iterator: if not _cresult.is_success(): _ret = False _connection.close() return _ret
def client_simple_call_para_server_tsl(a, b, *args, c=10, d={'d1': 'd1value'}, **kwargs): """ 测试简单调用,进行单向认证(客户端验证服务端证书) """ # 转换参数 _para_values = RunTool.get_current_function_parameter_values(is_simple_mode=True) _para_obj = SimpleGRpcTools.parameters_to_json(_para_values) _req_obj = SimpleGRpcTools.generate_request_obj( service_name='service_simple_call_para', para_json=_para_obj.para_json, has_para_bytes=_para_obj.has_para_bytes, para_bytes=_para_obj.para_bytes ) # 发送请求 with open(_TEMP_DIR + '/../../simple_grpc/server.crt', 'rb') as f: _root_certificates = f.read() # 发送请求, 默认使用全局的日志对象 _connect_para = SimpleGRpcConnection.generate_connect_para( ip='localhost', port=50053, is_use_ssl=True, root_certificates=_root_certificates ) _connection = SimpleGRpcConnection(_connect_para) _cresult = _connection.call(_req_obj) _connection.close() _cresult.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping( _cresult.return_json, 'client_simple_call_para' ) return _cresult
def open_file(self, index: int = 0): """ 打开文件并返回文件对象 @param {int} index=0 - 当前的数据处理线程索引 @returns {dict} - 打开的文件对象属性字典 {'handle': connection, 'close_able': 是否可关闭, lock: 锁对象, req_queue: 请求队列, ret_queue: 获取文件数据队列} """ self._mutiple_read = True self._file_handles_lock.acquire() _file_dict = self._file_handles.get(index, None) self._file_handles_lock.release() if _file_dict is None: # 获取不到,创建新文件访问对象 _id = self._grpc_call('pull_transfer_open_file_handle', [['', self.src_file]])['id'] _lock = threading.RLock() _file_dict = { 'handle': SimpleGRpcConnection(self.connect_para), 'close_able': True, 'lock': _lock, 'file_id': _id, # 已打开的文件访问id 'req_queue': queue.Queue(), # 用于向远端请求文件数据的队列 'ret_queue': queue.Queue(), # 用于获取远端请求文件数据的队列 } self._file_handles_lock.acquire() self._file_handles[index] = _file_dict self._file_handles_lock.release() # 启动后台线程监控数据读取请求 _read_file_data_thread = threading.Thread( target=self._read_file_data_thread_fun, args=(index, ), name='Thread-gRpc-Read-File-Data') _read_file_data_thread.setDaemon(True) _read_file_data_thread.start() # 返回结果 return _file_dict
def client_side_stream_call(): """ 客户端流模式,真正调用服务端的函数 """ # 发送请求, 默认使用全局的日志对象 _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.1', port=50051 ) _connection = SimpleGRpcConnection(_connect_para) _cresult = _connection.call(client_side_stream_generator(), call_mode=EnumCallMode.ClientSideStream) _connection.close() _cresult.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping( _cresult.return_json, 'service_client_side_stream' ) return _cresult
def server_bidirectional_stream_call_n_by_one(): """ 双向流模式,客户端处理,多个请求对应一个响应的情况 """ # 发送请求, 默认使用全局的日志对象 _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.1', port=50051 ) _connection = SimpleGRpcConnection(_connect_para) _cresult_iterator = _connection.call(bidirectional_stream_n_by_one_generator(), call_mode=EnumCallMode.BidirectionalStream) _ret = True for _cresult in _cresult_iterator: # 打印 print('bidirectional_stream_n_by_one client get: ' + _cresult.return_json) if not _cresult.is_success(): _ret = False _connection.close() return _ret
def client_simple_throw_excepiton(): """ 测试简单调用,抛出异常 """ # 转换参数 _para_values = RunTool.get_current_function_parameter_values( is_simple_mode=True) _para_obj = SimpleGRpcTools.parameters_to_json(_para_values) _req_obj = SimpleGRpcTools.generate_request_obj( service_name='service_simple_throw_excepiton', para_json=_para_obj.para_json, has_para_bytes=_para_obj.has_para_bytes, para_bytes=_para_obj.para_bytes) # 发送请求, 默认使用全局的日志对象 _connect_para = SimpleGRpcConnection.generate_connect_para( conn_str='127.0.0.1:50051') _connection = SimpleGRpcConnection(_connect_para) _cresult = _connection.call(_req_obj) _connection.close() _cresult.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping( _cresult.return_json, 'client_simple_throw_excepiton') return _cresult
def client_simple_call_para(a, b, *args, c=10, d={'d1': 'd1value'}, **kwargs): """ 测试简单调用,直接返回参数 """ # 转换参数 _para_values = RunTool.get_current_function_parameter_values(is_simple_mode=True) _para_obj = SimpleGRpcTools.parameters_to_json(_para_values) _req_obj = SimpleGRpcTools.generate_request_obj( service_name='service_simple_call_para', para_json=_para_obj.para_json, has_para_bytes=_para_obj.has_para_bytes, para_bytes=_para_obj.para_bytes ) # 发送请求, 默认使用全局的日志对象 _connect_para = SimpleGRpcConnection.generate_connect_para( conn_str='127.0.0.1:50051', servicer_name='servicer_simple_call', test_on_connect=True, test_use_health_check=False ) _connection = SimpleGRpcConnection(_connect_para) _cresult = _connection.call(_req_obj) _connection.close() _cresult.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping( _cresult.return_json, 'client_simple_call_para' ) return _cresult
def test_health_check(self): """ 测试服务健康状态 """ if not TEST_FLAG['test_health_check']: return print("测试服务健康状态") _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.1', port=50051, servicer_name='servicer_simple_call', test_use_health_check=True ) _connection = SimpleGRpcConnection(_connect_para) print("测试服务健康状态 - 服务中") _resp_obj = _connection.test() # _resp_obj = SimpleGRpcTools.health_check(_connect_para, 'servicer_simple_call') self.assertTrue(_resp_obj.status == health_pb2.HealthCheckResponse.SERVING, '测试服务健康状态失败 - 服务中') print("测试服务健康状态 - 停止服务") self.server_no_ssl_no_zoo.set_service_status('servicer_simple_call', health_pb2.HealthCheckResponse.NOT_SERVING) _resp_obj = _connection.test() # _resp_obj = SimpleGRpcTools.health_check(_connect_para, 'servicer_simple_call') self.assertTrue(_resp_obj.status == health_pb2.HealthCheckResponse.NOT_SERVING, '测试服务健康状态失败 - 停止服务') # 恢复服务 self.server_no_ssl_no_zoo.set_service_status('servicer_simple_call', health_pb2.HealthCheckResponse.SERVING) _connection.close() print("测试服务健康状态 - 服务不存在") _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.2', port=50051, servicer_name='servicer_simple_call' ) _connection = SimpleGRpcConnection(_connect_para) _resp_obj = _connection.test() # _resp_obj = SimpleGRpcTools.health_check(_connect_para, 'servicer_simple_call') self.assertTrue(_resp_obj.status == health_pb2.HealthCheckResponse.UNKNOWN, '测试服务健康状态失败 - 服务不存在')
def test_error(self): """ 测试错误信息 """ if not TEST_FLAG['test_error']: return print("测试错误信息") print("测试错误信息 - 连接失败") _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.1', port=60051 ) _connection = SimpleGRpcConnection(_connect_para) _cresult = _connection.call(SimpleGRpcTools.generate_request_obj('test')) _connection.close() self.assertTrue( _cresult.code == '20408' and grpc.StatusCode.UNAVAILABLE.name == _cresult.i18n_msg_paras[0], '测试错误信息失败 - 连接失败 ' ) print("测试错误信息 - 服务名不存在") _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.1', port=50051 ) _connection = SimpleGRpcConnection(_connect_para) _cresult = _connection.call(SimpleGRpcTools.generate_request_obj('test', '')) _connection.close() self.assertTrue( _cresult.code == '11403', '测试错误信息失败 - 服务名不存在 ' ) print("测试错误信息 - 超时") _connect_para = SimpleGRpcConnection.generate_connect_para( ip='127.0.0.1', port=50051, timeout=0.1 ) _connection = SimpleGRpcConnection(_connect_para) _cresult = _connection.call(SimpleGRpcTools.generate_request_obj('service_simple_overtime')) _connection.close() self.assertTrue( _cresult.code == '30403', '测试错误信息失败 - 超时' )
def test_grpc_to_local(self): if not TEST_FLAG['test_grpc_to_local']: return print('测试获取gRpc远程文件') # DebugTool.set_debug(set_on=True) _copy_file = os.path.join(_temp_path, 'grpc_to_local.bin') # 删除临时文件 if os.path.exists(_copy_file): FileTool.remove_file(_copy_file) for _ext in ('.lock', '.tmp', '.info'): if os.path.exists(_copy_file + _ext): FileTool.remove_file(_copy_file + _ext) # 连接参数 _connect_para = SimpleGRpcConnection.generate_connect_para( conn_str='127.0.0.1:50051') # 单线程-缓存大于文件大小 _tips = '单线程-缓存大于文件大小' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.0) _status = _reader.start(wait_finished=True) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status)) # 单线程-缓存小于文件大小 _tips = '单线程-缓存小于文件大小' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, cache_size=2, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.0) _status = _reader.start(wait_finished=True) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status)) # 单线程-暂停重复 _tips = '单线程-暂停重复' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, cache_size=2, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.1) _status = _reader.start() time.sleep(1) _reader.stop() self.assertTrue(_reader.status == 'stop', msg="本地gRpc推送-%s(暂停): %s" % (_tips, _reader.status)) _reader.thread_interval = 0.0 _status = _reader.start(wait_finished=True) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status)) # 单线程-停止后续传 _tips = '单线程-停止后续传' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, cache_size=2, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.1) _status = _reader.start() time.sleep(1) _reader.stop() self.assertTrue(_reader.status == 'stop', msg="本地gRpc推送-%s(暂停): %s" % (_tips, _reader.status)) # 续传处理 with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, cache_size=2, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.0) _status = _reader.start(wait_finished=True) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status)) # 多线程 _tips = '多线程' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, cache_size=2, thread_num=230, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.0) _status = _reader.start(wait_finished=True) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status)) # 多线程-缓存大于文件大小 _tips = '多线程-缓存大于文件大小' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, thread_num=5, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.0) _status = _reader.start(wait_finished=True) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status)) # 多线程-每次传输块大小大于文件 _tips = '多线程-每次传输块大小大于文件' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, thread_num=5, block_size=900000, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.0) _status = _reader.start(wait_finished=True) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status)) # 多线程-暂停重复 _tips = '多线程-暂停重复' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, cache_size=2, thread_num=5, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.3) _status = _reader.start() time.sleep(1) _reader.stop() self.assertTrue(_reader.status == 'stop', msg="本地gRpc推送-%s(暂停): %s" % (_tips, _reader.status)) _reader.thread_interval = 0.0 _status = _reader.start(wait_finished=True) if _status != 'finished': print(NetTool.get_file_md5(_temp_file)) print(NetTool.get_file_md5(_copy_file + '.tmp')) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status)) # 多线程-停止后续传 _tips = '多线程-停止后续传' with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, cache_size=2, thread_num=5, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.3) _status = _reader.start() time.sleep(1) _reader.stop() self.assertTrue(_reader.status == 'stop', msg="本地gRpc推送-%s(暂停): %s" % (_tips, _reader.status)) # 续传处理 with GRpcPullProtocol(_temp_file, _copy_file, is_resume=True, is_overwrite=True, cache_size=2, thread_num=5, block_size=40960, connect_para=_connect_para) as _protocol: _reader = Transfer( _protocol, show_process_bar_fun=ProgressRate.show_cmd_process_bar, process_bar_label=_tips, thread_interval=0.0) _status = _reader.start(wait_finished=True) if _status != 'finished': print(NetTool.get_file_md5(_temp_file)) print(NetTool.get_file_md5(_copy_file + '.tmp')) self.assertTrue(_status == 'finished', msg="本地gRpc推送-%s: %s" % (_tips, _status))
def test_connect_pool(self): """ 测试连接池 """ print("开始测试连接池") _connect_para = SimpleGRpcConnection.generate_connect_para( conn_str='127.0.0.1:50051', servicer_name='servicer_simple_call', test_on_connect=False, test_use_health_check=False) # 建立连接池 _pool = SimpleGRpcConnectionPool(_connect_para, name='ConnectionPool', maxsize=3, minsize=0, realse_free_time=5, test_on_get=True, test_on_free=True, test_while_idle=True, test_idle_time=5, validation_query='', get_connection_timeout=1, logger=self._asyn_logger, init_break_if_connect_error=True) print("测试连接池-获取连接并执行") # 尝试获取连接 _connection = _pool.get_connection() _back_server = service_simple_call_para('a1', 'b1') _req_obj = get_client_simple_call_para('a1', 'b1') _back_client = _connection.call(_req_obj) _back_client.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping( _back_client.return_json, 'client_simple_call_para') self.assertTrue( _back_client.is_success(), msg= '测试连接池-获取连接并执行失败,执行RPC失败: code=%s, msg=%s, error=%s, i18n_msg_paras=%s' % (_back_client.code, _back_client.msg, _back_client.error, str(_back_client.i18n_msg_paras))) self.assertTrue(compare_object_by_json(_back_server, _back_client.return_obj), msg='测试连接池-获取连接并执行, 执行参数不一致') print('测试连接池-获取连接超时') _c1 = _pool.get_connection() _c2 = _pool.get_connection() try: _c3 = _pool.get_connection() self.assertTrue(False, msg='测试连接池-获取连接超时失败,应抛出超时') except TimeoutError: pass except Exception as e: self.assertTrue(False, msg='测试连接池-获取连接超时失败,未期望的异常:%s' % str(e)) self.assertTrue(3 == _pool.current_size, msg='测试连接池-获取连接超时-当前连接池大小错误:%d' % _pool.current_size) print('测试连接池-释放连接') _pool.free_connection(_connection) _c3 = _pool.get_connection() # 这样c3可用获取连接并使用 self.assertTrue(3 == _pool.current_size, msg='测试连接池-释放连接-当前连接池大小错误:%d' % _pool.current_size) print('测试连接池-自动释放连接') _pool.free_connection(_c1) _pool.free_connection(_c2) _pool.free_connection(_c3) time.sleep(10) self.assertTrue(0 == _pool.current_size, msg='测试连接池-自动释放连接-当前连接池大小错误:%d' % _pool.current_size)