def test_mutiple_thread(self):
if not TEST_FLAG['test_mutiple_thread']:
return
print("测试同一个连接多线程并发")
# 建立连接
_connect_para = SimpleGRpcConnection.generate_connect_para(
conn_str='127.0.0.1:50051'
)
_connection = SimpleGRpcConnection(_connect_para)
# 创建多线程执行
_result = list()
for _i in range(80):
_result.append(False)
_call_thread = threading.Thread(
target=self._mutiple_thread_fun,
args=(_i, _connection, _result),
name='Thread-gRpc-call'
)
_call_thread.setDaemon(True)
_call_thread.start()
# 等待线程执行完成
time.sleep(20)
_connection.close()
# 检查结果
for _res_i in _result:
if not _res_i:
self.assertTrue(False, '测试同一个连接多线程并发失败')
def open_writer(self, index: int = 0) -> dict:
"""
打开写入对象并返回对象属性
@param {int} index=0 - 当前的数据处理线程索引
@returns {dict} - 打开的文件对象属性字典
{'handle': 写入对象, 'close_able': 是否可关闭, lock: 锁对象}
"""
self._writer_handles_lock.acquire()
try:
if index not in self._writer_handles.keys():
# 新增写入对象属性（远程连接对象）
self._writer_handles[index] = {
'handle': SimpleGRpcConnection(self.connect_para),
'close_able': True,
'lock': threading.RLock(),
'req_queue': queue.Queue(), # 用于向远端请求文件数据的队列
'ret_queue': queue.Queue(), # 用于获取远端请求文件数据的队列
}
# 启动后台线程监控数据读取请求
_write_data_thread = threading.Thread(
target=self._write_data_thread_fun,
args=(index, ),
name='Thread-gRpc-Write-Data')
_write_data_thread.setDaemon(True)
_write_data_thread.start()
# 返回打开的写入对象
return self._writer_handles[index]
finally:
self._writer_handles_lock.release()
def _grpc_call(self,
service_name: str,
para_list: list,
is_support_bytes=False):
"""
调用远程grpc服务
注：该函数不处理返回对象中的bytes信息
@param {str} service_name - 要调用的服务名
@param {list} para_list - 要转换的参数列表，list的每一项为一个参数项，每个参数项如下：
[para_name, call_value] : para_name可以为''
@param {bool} is_support_bytes=False - 是否支持参数中的字节数组传递（只支持最后一个参数）
@returns {object} - 返回远程调用返回的对象
@throws {RuntimeError} - 如果远程调用出现错误抛出异常
"""
# 生成请求对象
_para_obj = SimpleGRpcTools.parameters_to_json(
para_list, is_support_bytes=is_support_bytes)
_req_obj = SimpleGRpcTools.generate_request_obj(
service_name=service_name,
para_json=_para_obj.para_json,
has_para_bytes=_para_obj.has_para_bytes,
para_bytes=_para_obj.para_bytes)
# 连接并请求服务
_connection = SimpleGRpcConnection(self.connect_para)
try:
_cresult: CResult = _connection.call(_req_obj)
if not _cresult.is_success():
# 执行失败，抛出异常
raise RuntimeError('call %s error: %s' %
(service_name, str(_cresult)))
# 处理返回值
if _cresult.return_json == '':
return None
else:
return json.loads(_cresult.return_json)
finally:
_connection.close()
def server_bidirectional_stream_call_one_by_one():
"""
双向流模式，客户端处理，一个请求对应一个响应的情况
通过队列传递交互传递参数
"""
# 清空队列
TEMP_QUEUE.queue.clear()
TEMP_QUEUE.put(None)
# 处理
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.1', port=50051
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult_iterator = _connection.call(
bidirectional_stream_one_by_one_generator(),
call_mode=EnumCallMode.BidirectionalStream)
for _cresult in _cresult_iterator:
# 放入队列
s = _cresult.return_json
TEMP_QUEUE.put(s)
if not _cresult.is_success():
return False
_connection.close()
return True
def server_side_stream_call(a, b, *args, **kwargs):
"""
服务端流模式，真正调用服务端的函数
"""
# 转换参数
_para_values = RunTool.get_current_function_parameter_values(is_simple_mode=True)
_para_obj = SimpleGRpcTools.parameters_to_json(_para_values)
_req_obj = SimpleGRpcTools.generate_request_obj(
service_name='service_server_side_stream', para_json=_para_obj.para_json,
has_para_bytes=_para_obj.has_para_bytes, para_bytes=_para_obj.para_bytes
)
# 发送请求, 默认使用全局的日志对象
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.1', port=50051
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult_iterator = _connection.call(_req_obj,
call_mode=EnumCallMode.ServerSideStream)
_ret = True
for _cresult in _cresult_iterator:
if not _cresult.is_success():
_ret = False
_connection.close()
return _ret
def client_simple_call_para_server_tsl(a, b, *args, c=10, d={'d1': 'd1value'}, **kwargs):
"""
测试简单调用，进行单向认证（客户端验证服务端证书）
"""
# 转换参数
_para_values = RunTool.get_current_function_parameter_values(is_simple_mode=True)
_para_obj = SimpleGRpcTools.parameters_to_json(_para_values)
_req_obj = SimpleGRpcTools.generate_request_obj(
service_name='service_simple_call_para', para_json=_para_obj.para_json,
has_para_bytes=_para_obj.has_para_bytes, para_bytes=_para_obj.para_bytes
)
# 发送请求
with open(_TEMP_DIR + '/../../simple_grpc/server.crt', 'rb') as f:
_root_certificates = f.read()
# 发送请求, 默认使用全局的日志对象
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='localhost', port=50053, is_use_ssl=True,
root_certificates=_root_certificates
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult = _connection.call(_req_obj)
_connection.close()
_cresult.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping(
_cresult.return_json,
'client_simple_call_para'
)
return _cresult
def open_file(self, index: int = 0):
"""
打开文件并返回文件对象
@param {int} index=0 - 当前的数据处理线程索引
@returns {dict} - 打开的文件对象属性字典
{'handle': connection, 'close_able': 是否可关闭, lock: 锁对象, req_queue: 请求队列, ret_queue: 获取文件数据队列}
"""
self._mutiple_read = True
self._file_handles_lock.acquire()
_file_dict = self._file_handles.get(index, None)
self._file_handles_lock.release()
if _file_dict is None:
# 获取不到，创建新文件访问对象
_id = self._grpc_call('pull_transfer_open_file_handle',
[['', self.src_file]])['id']
_lock = threading.RLock()
_file_dict = {
'handle': SimpleGRpcConnection(self.connect_para),
'close_able': True,
'lock': _lock,
'file_id': _id, # 已打开的文件访问id
'req_queue': queue.Queue(), # 用于向远端请求文件数据的队列
'ret_queue': queue.Queue(), # 用于获取远端请求文件数据的队列
}
self._file_handles_lock.acquire()
self._file_handles[index] = _file_dict
self._file_handles_lock.release()
# 启动后台线程监控数据读取请求
_read_file_data_thread = threading.Thread(
target=self._read_file_data_thread_fun,
args=(index, ),
name='Thread-gRpc-Read-File-Data')
_read_file_data_thread.setDaemon(True)
_read_file_data_thread.start()
# 返回结果
return _file_dict
def client_side_stream_call():
"""
客户端流模式，真正调用服务端的函数
"""
# 发送请求, 默认使用全局的日志对象
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.1', port=50051
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult = _connection.call(client_side_stream_generator(),
call_mode=EnumCallMode.ClientSideStream)
_connection.close()
_cresult.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping(
_cresult.return_json,
'service_client_side_stream'
)
return _cresult
def server_bidirectional_stream_call_n_by_one():
"""
双向流模式，客户端处理，多个请求对应一个响应的情况
"""
# 发送请求, 默认使用全局的日志对象
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.1', port=50051
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult_iterator = _connection.call(bidirectional_stream_n_by_one_generator(),
call_mode=EnumCallMode.BidirectionalStream)
_ret = True
for _cresult in _cresult_iterator:
# 打印
print('bidirectional_stream_n_by_one client get: ' + _cresult.return_json)
if not _cresult.is_success():
_ret = False
_connection.close()
return _ret
def client_simple_throw_excepiton():
"""
测试简单调用，抛出异常
"""
# 转换参数
_para_values = RunTool.get_current_function_parameter_values(
is_simple_mode=True)
_para_obj = SimpleGRpcTools.parameters_to_json(_para_values)
_req_obj = SimpleGRpcTools.generate_request_obj(
service_name='service_simple_throw_excepiton',
para_json=_para_obj.para_json,
has_para_bytes=_para_obj.has_para_bytes,
para_bytes=_para_obj.para_bytes)
# 发送请求, 默认使用全局的日志对象
_connect_para = SimpleGRpcConnection.generate_connect_para(
conn_str='127.0.0.1:50051')
_connection = SimpleGRpcConnection(_connect_para)
_cresult = _connection.call(_req_obj)
_connection.close()
_cresult.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping(
_cresult.return_json, 'client_simple_throw_excepiton')
return _cresult
def client_simple_call_para(a, b, *args, c=10, d={'d1': 'd1value'}, **kwargs):
"""
测试简单调用，直接返回参数
"""
# 转换参数
_para_values = RunTool.get_current_function_parameter_values(is_simple_mode=True)
_para_obj = SimpleGRpcTools.parameters_to_json(_para_values)
_req_obj = SimpleGRpcTools.generate_request_obj(
service_name='service_simple_call_para', para_json=_para_obj.para_json,
has_para_bytes=_para_obj.has_para_bytes, para_bytes=_para_obj.para_bytes
)
# 发送请求, 默认使用全局的日志对象
_connect_para = SimpleGRpcConnection.generate_connect_para(
conn_str='127.0.0.1:50051', servicer_name='servicer_simple_call',
test_on_connect=True, test_use_health_check=False
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult = _connection.call(_req_obj)
_connection.close()
_cresult.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping(
_cresult.return_json,
'client_simple_call_para'
)
return _cresult
def test_health_check(self):
"""
测试服务健康状态
"""
if not TEST_FLAG['test_health_check']:
return
print("测试服务健康状态")
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.1', port=50051, servicer_name='servicer_simple_call', test_use_health_check=True
)
_connection = SimpleGRpcConnection(_connect_para)
print("测试服务健康状态 - 服务中")
_resp_obj = _connection.test()
# _resp_obj = SimpleGRpcTools.health_check(_connect_para, 'servicer_simple_call')
self.assertTrue(_resp_obj.status == health_pb2.HealthCheckResponse.SERVING,
'测试服务健康状态失败 - 服务中')
print("测试服务健康状态 - 停止服务")
self.server_no_ssl_no_zoo.set_service_status('servicer_simple_call',
health_pb2.HealthCheckResponse.NOT_SERVING)
_resp_obj = _connection.test()
# _resp_obj = SimpleGRpcTools.health_check(_connect_para, 'servicer_simple_call')
self.assertTrue(_resp_obj.status == health_pb2.HealthCheckResponse.NOT_SERVING,
'测试服务健康状态失败 - 停止服务')
# 恢复服务
self.server_no_ssl_no_zoo.set_service_status('servicer_simple_call',
health_pb2.HealthCheckResponse.SERVING)
_connection.close()
print("测试服务健康状态 - 服务不存在")
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.2', port=50051, servicer_name='servicer_simple_call'
)
_connection = SimpleGRpcConnection(_connect_para)
_resp_obj = _connection.test()
# _resp_obj = SimpleGRpcTools.health_check(_connect_para, 'servicer_simple_call')
self.assertTrue(_resp_obj.status == health_pb2.HealthCheckResponse.UNKNOWN,
'测试服务健康状态失败 - 服务不存在')
def test_error(self):
"""
测试错误信息
"""
if not TEST_FLAG['test_error']:
return
print("测试错误信息")
print("测试错误信息 - 连接失败")
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.1', port=60051
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult = _connection.call(SimpleGRpcTools.generate_request_obj('test'))
_connection.close()
self.assertTrue(
_cresult.code == '20408' and grpc.StatusCode.UNAVAILABLE.name == _cresult.i18n_msg_paras[0],
'测试错误信息失败 - 连接失败 '
)
print("测试错误信息 - 服务名不存在")
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.1', port=50051
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult = _connection.call(SimpleGRpcTools.generate_request_obj('test', ''))
_connection.close()
self.assertTrue(
_cresult.code == '11403',
'测试错误信息失败 - 服务名不存在 '
)
print("测试错误信息 - 超时")
_connect_para = SimpleGRpcConnection.generate_connect_para(
ip='127.0.0.1', port=50051, timeout=0.1
)
_connection = SimpleGRpcConnection(_connect_para)
_cresult = _connection.call(SimpleGRpcTools.generate_request_obj('service_simple_overtime'))
_connection.close()
self.assertTrue(
_cresult.code == '30403',
'测试错误信息失败 - 超时'
)
def test_grpc_to_local(self):
if not TEST_FLAG['test_grpc_to_local']:
return
print('测试获取gRpc远程文件')
# DebugTool.set_debug(set_on=True)
_copy_file = os.path.join(_temp_path, 'grpc_to_local.bin')
# 删除临时文件
if os.path.exists(_copy_file):
FileTool.remove_file(_copy_file)
for _ext in ('.lock', '.tmp', '.info'):
if os.path.exists(_copy_file + _ext):
FileTool.remove_file(_copy_file + _ext)
# 连接参数
_connect_para = SimpleGRpcConnection.generate_connect_para(
conn_str='127.0.0.1:50051')
# 单线程-缓存大于文件大小
_tips = '单线程-缓存大于文件大小'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 单线程-缓存小于文件大小
_tips = '单线程-缓存小于文件大小'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 单线程-暂停重复
_tips = '单线程-暂停重复'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.1)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地gRpc推送-%s(暂停): %s" %
(_tips, _reader.status))
_reader.thread_interval = 0.0
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 单线程-停止后续传
_tips = '单线程-停止后续传'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.1)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地gRpc推送-%s(暂停): %s" % (_tips, _reader.status))
# 续传处理
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程
_tips = '多线程'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=230,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程-缓存大于文件大小
_tips = '多线程-缓存大于文件大小'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
thread_num=5,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程-每次传输块大小大于文件
_tips = '多线程-每次传输块大小大于文件'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
thread_num=5,
block_size=900000,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程-暂停重复
_tips = '多线程-暂停重复'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.3)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地gRpc推送-%s(暂停): %s" %
(_tips, _reader.status))
_reader.thread_interval = 0.0
_status = _reader.start(wait_finished=True)
if _status != 'finished':
print(NetTool.get_file_md5(_temp_file))
print(NetTool.get_file_md5(_copy_file + '.tmp'))
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程-停止后续传
_tips = '多线程-停止后续传'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.3)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地gRpc推送-%s(暂停): %s" % (_tips, _reader.status))
# 续传处理
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
if _status != 'finished':
print(NetTool.get_file_md5(_temp_file))
print(NetTool.get_file_md5(_copy_file + '.tmp'))
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
def test_connect_pool(self):
"""
测试连接池
"""
print("开始测试连接池")
_connect_para = SimpleGRpcConnection.generate_connect_para(
conn_str='127.0.0.1:50051',
servicer_name='servicer_simple_call',
test_on_connect=False,
test_use_health_check=False)
# 建立连接池
_pool = SimpleGRpcConnectionPool(_connect_para,
name='ConnectionPool',
maxsize=3,
minsize=0,
realse_free_time=5,
test_on_get=True,
test_on_free=True,
test_while_idle=True,
test_idle_time=5,
validation_query='',
get_connection_timeout=1,
logger=self._asyn_logger,
init_break_if_connect_error=True)
print("测试连接池-获取连接并执行")
# 尝试获取连接
_connection = _pool.get_connection()
_back_server = service_simple_call_para('a1', 'b1')
_req_obj = get_client_simple_call_para('a1', 'b1')
_back_client = _connection.call(_req_obj)
_back_client.return_obj = SimpleGRpcTools.json_to_object_by_para_mapping(
_back_client.return_json, 'client_simple_call_para')
self.assertTrue(
_back_client.is_success(),
msg=
'测试连接池-获取连接并执行失败，执行RPC失败: code=%s, msg=%s, error=%s, i18n_msg_paras=%s'
% (_back_client.code, _back_client.msg, _back_client.error,
str(_back_client.i18n_msg_paras)))
self.assertTrue(compare_object_by_json(_back_server,
_back_client.return_obj),
msg='测试连接池-获取连接并执行, 执行参数不一致')
print('测试连接池-获取连接超时')
_c1 = _pool.get_connection()
_c2 = _pool.get_connection()
try:
_c3 = _pool.get_connection()
self.assertTrue(False, msg='测试连接池-获取连接超时失败，应抛出超时')
except TimeoutError:
pass
except Exception as e:
self.assertTrue(False, msg='测试连接池-获取连接超时失败，未期望的异常:%s' % str(e))
self.assertTrue(3 == _pool.current_size,
msg='测试连接池-获取连接超时-当前连接池大小错误：%d' % _pool.current_size)
print('测试连接池-释放连接')
_pool.free_connection(_connection)
_c3 = _pool.get_connection() # 这样c3可用获取连接并使用
self.assertTrue(3 == _pool.current_size,
msg='测试连接池-释放连接-当前连接池大小错误：%d' % _pool.current_size)
print('测试连接池-自动释放连接')
_pool.free_connection(_c1)
_pool.free_connection(_c2)
_pool.free_connection(_c3)
time.sleep(10)
self.assertTrue(0 == _pool.current_size,
msg='测试连接池-自动释放连接-当前连接池大小错误：%d' % _pool.current_size)