def _test_write_and_read(self, obj):
buff0 = TCyMemoryBuffer()
op0 = TCyBinaryProtocol(buff0)
op0.write_struct(obj)
value0 = buff0.getvalue()
buff1 = TCyMemoryBuffer(value0)
op1 = TCyBinaryProtocol(buff1)
obj1 = obj.__class__()
return op1.read_struct(obj1)
def loop_reader(self, transport, queue):
"""
:param tsocket:
:param queue:
:return:
"""
"""
:param tsocket:
:param queue:
:return:
"""
# transport = TCyFramedTransport(None)
while self.connection_ok:
try:
# 1. 读取一帧数据
trans_input = transport.read_frame(
) # TCyMemoryBuffer, 有可能一直堵在这里, 通过 loop_hb_detect.close()来终止
trans_input.reset_frame() # 跳过Frame Size
proto_input = TCyBinaryProtocol(trans_input, client=False)
name, type, seqid = proto_input.readMessageBegin()
# 如果是心跳，则直接返回
if type == MESSAGE_TYPE_HEART_BEAT:
trans_input.reset()
queue.put(trans_input)
self.last_hb_time = time.time()
# print "Received Heartbeat Signal........"
continue
else:
self.last_request_time = time.time()
# print "Read Request"
trans_input.reset_frame()
self.task_pool.spawn(self.handle_request, proto_input,
queue,
(name, type, seqid, time.time()))
except TTransportException as e:
# EOF是很正常的现象
if e.type != TTransportException.END_OF_FILE:
print_exception(info_logger)
info_logger.warning(
"....Worker Connection To LB Failed, LoopWrite Stop")
queue.put(None) # 表示要结束了
break
except:
print_exception(info_logger)
queue.put(None) # 表示要结束了
break
def _test_protocol_write(self, obj):
# cython版本
buff0 = TCyMemoryBuffer()
op0 = TCyBinaryProtocol(buff0)
op0.write_struct(obj)
value0 = buff0.getvalue()
# Python版本
buff1 = TMemoryBuffer()
op1 = TUtf8BinaryProtocol(buff1)
obj.write(op1)
value1 = buff1.get_raw_value()
self.assertEqual(value1, value0, "write result not the same")
def _test_protocol_write(self, obj):
# cython版本
buff0 = TCyMemoryBuffer()
op0 = TCyBinaryProtocol(buff0)
op0.write_struct(obj)
value0 = buff0.getvalue()
# Python版本
buff1 = TMemoryBuffer()
op1 = TUtf8BinaryProtocol(buff1)
obj.write(op1)
value1 = buff1.get_raw_value()
self.assertEqual(value1, value0, "write result not the same")
def loop_reader(self, transport, queue):
"""
:param tsocket:
:param queue:
:return:
"""
"""
:param tsocket:
:param queue:
:return:
"""
# transport = TCyFramedTransport(None)
while self.connection_ok:
try:
# 1. 读取一帧数据
trans_input = transport.read_frame() # TCyMemoryBuffer, 有可能一直堵在这里, 通过 loop_hb_detect.close()来终止
trans_input.reset_frame() # 跳过Frame Size
proto_input = TCyBinaryProtocol(trans_input, client=False)
name, type, seqid = proto_input.readMessageBegin()
# 如果是心跳，则直接返回
if type == MESSAGE_TYPE_HEART_BEAT:
trans_input.reset()
queue.put(trans_input)
self.last_hb_time = time.time()
# print "Received Heartbeat Signal........"
continue
else:
self.last_request_time = time.time()
# print "Read Request"
trans_input.reset_frame()
self.task_pool.spawn(self.handle_request, proto_input, queue, (name, type, seqid, time.time()))
except TTransportException as e:
# EOF是很正常的现象
if e.type != TTransportException.END_OF_FILE:
print_exception(info_logger)
info_logger.warning("....Worker Connection To LB Failed, LoopWrite Stop")
queue.put(None) # 表示要结束了
break
except:
print_exception(info_logger)
queue.put(None) # 表示要结束了
break
def handle_request(self, proto_input, queue, request_meta):
"""
从 proto_input中读取数据，然后调用processor处理请求，结果暂时缓存在内存中, 最后一口气交给 queue,
由专门的 greenlet将数据写回到socket上
request_meta = (name, type, seqid, start_time)
"""
start_time0 = time.time()
# 1. 获取一个可用的trans_output
if len(self.out_protocols) > 0:
trans_output, proto_output = self.out_protocols.popleft()
trans_output.prepare_4_frame() # 预留4个字节的Frame Size
else:
trans_output = TCyMemoryBuffer()
trans_output.prepare_4_frame()
proto_output = TCyBinaryProtocol(trans_output, client=False) # 无状态的
try:
# 2.1 处理正常的请求
self.processor.process(proto_input, proto_output)
queue.put(trans_output)
except Exception as e:
# 2.2 处理异常(主要是结果序列化时参数类型不对的情况)
trans_output.prepare_4_frame()
name = request_meta[0]
seqId = request_meta[2]
msg = '%s, Exception: %s, Trace: %s' % (name, e, traceback.format_exc())
x = TApplicationException(TApplicationException.INVALID_PROTOCOL, msg)
proto_output.writeMessageBegin(name, TMessageType.EXCEPTION, seqId)
x.write(proto_output)
proto_output.writeMessageEnd()
proto_output.trans.flush()
queue.put(trans_output)
finally:
start_time = request_meta[3]
now = time.time()
elapsed = now - start_time
if elapsed > 2:
# 做异常记录
exception_logger.info("Exception Request: %s %s seqId: %s_%s, Elaspsed: %.3f, Execute: %.3f", request_meta[0], request_meta[1], self.pid, request_meta[2], elapsed, now - start_time0)
# 3. 回收 transport 和 protocol
self.out_protocols.append((trans_output, proto_output))
def test_read_write_exception(self):
"""
测试可行性
py.test test/test_protocol.py::SimpleTest::test_read_write_exception -s
"""
ex = RpcException(1, "Hello")
self._test_protocol_write(ex)
total_time = 1
t = time.time()
for i in range(total_time):
buff0 = TCyMemoryBuffer()
op0 = TCyBinaryProtocol(buff0)
op0.write_struct(ex)
value0 = buff0.getvalue()
t = time.time() - t
print "T: %.3fms" % (t * 1000)
print "Value0:", ["%03d" % ord(i) for i in value0]
buff0 = TCyMemoryBuffer(value0)
op0 = TCyBinaryProtocol(buff0)
ex0 = op0.read_struct(RpcException())
print "Code: ", ex0.code
print "Msg: ", ex0.msg
t = time.time()
for i in range(total_time):
buff1 = TMemoryBuffer()
op1 = TBinaryProtocol(buff1)
ex.write(op1)
value1 = buff1.get_raw_value()
t = time.time() - t
print "T: %.3fms" % (t * 1000)
print "Value1:", ["%03d" % ord(i) for i in value1]
self.assertEqual(value1, value0, "write struct test failed")
def new_rpc_exit_message():
buf = TCyMemoryBuffer()
protocol = TCyBinaryProtocol(buf)
protocol.writeMessageBegin("worker_shutdonw", MESSAGE_TYPE_STOP, 0)
protocol.writeMessageEnd()
return buf
def _test_write_and_read(self, obj):
buff0 = TCyMemoryBuffer()
op0 = TCyBinaryProtocol(buff0)
op0.write_struct(obj)
value0 = buff0.getvalue()
buff1 = TCyMemoryBuffer(value0)
op1 = TCyBinaryProtocol(buff1)
obj1 = obj.__class__()
return op1.read_struct(obj1)
def handle_request(self, proto_input, queue, request_meta):
"""
从 proto_input中读取数据，然后调用processor处理请求，结果暂时缓存在内存中, 最后一口气交给 queue,
由专门的 greenlet将数据写回到socket上
request_meta = (name, type, seqid, start_time)
"""
start_time0 = time.time()
# 1. 获取一个可用的trans_output
if len(self.out_protocols) > 0:
trans_output, proto_output = self.out_protocols.popleft()
trans_output.prepare_4_frame() # 预留4个字节的Frame Size
else:
trans_output = TCyMemoryBuffer()
trans_output.prepare_4_frame()
proto_output = TCyBinaryProtocol(trans_output,
client=False) # 无状态的
try:
# 2.1 处理正常的请求
self.processor.process(proto_input, proto_output)
queue.put(trans_output)
except Exception as e:
# 2.2 处理异常(主要是结果序列化时参数类型不对的情况)
trans_output.prepare_4_frame()
name = request_meta[0]
seqId = request_meta[2]
msg = '%s, Exception: %s, Trace: %s' % (name, e,
traceback.format_exc())
x = TApplicationException(TApplicationException.INVALID_PROTOCOL,
msg)
proto_output.writeMessageBegin(name, TMessageType.EXCEPTION, seqId)
x.write(proto_output)
proto_output.writeMessageEnd()
proto_output.trans.flush()
queue.put(trans_output)
finally:
start_time = request_meta[3]
now = time.time()
elapsed = now - start_time
if elapsed > 2:
# 做异常记录
exception_logger.info(
"Exception Request: %s %s seqId: %s_%s, Elaspsed: %.3f, Execute: %.3f",
request_meta[0], request_meta[1], self.pid,
request_meta[2], elapsed, now - start_time0)
# 3. 回收 transport 和 protocol
self.out_protocols.append((trans_output, proto_output))
def get_service_protocol(service,
transport=None,
logger=None,
fastbinary=False,
fast=False):
"""
多个不同的service可以共用一个base_protocol; 如果指定了service, 则在base_protocol的基础上添加一个新的wrap
存在问题: TFramedTransport存在buf, 自动重连之后状态还存在
:param service:
:param base_protocol:
:param fastbinary 参数无效，不再使用
:return:
"""
if fast:
transport = transport or _fast_transport
return TCyBinaryProtocol(transport, service=service, logger=logger)
else:
transport = transport or _transport
return TUtf8BinaryProtocol(transport,
service_name=service,
logger=logger)
def test_read_write_exception(self):
"""
测试可行性
py.test test/test_protocol.py::SimpleTest::test_read_write_exception -s
"""
ex = RpcException(1, "Hello")
self._test_protocol_write(ex)
total_time = 1
t = time.time()
for i in range(total_time):
buff0 = TCyMemoryBuffer()
op0 = TCyBinaryProtocol(buff0)
op0.write_struct(ex)
value0 = buff0.getvalue()
t = time.time() - t
print "T: %.3fms" % (t * 1000)
print "Value0:", ["%03d" % ord(i) for i in value0]
buff0 = TCyMemoryBuffer(value0)
op0 = TCyBinaryProtocol(buff0)
ex0 = op0.read_struct(RpcException())
print "Code: ", ex0.code
print "Msg: ", ex0.msg
t = time.time()
for i in range(total_time):
buff1 = TMemoryBuffer()
op1 = TBinaryProtocol(buff1)
ex.write(op1)
value1 = buff1.get_raw_value()
t = time.time() - t
print "T: %.3fms" % (t * 1000)
print "Value1:", ["%03d" % ord(i) for i in value1]
self.assertEqual(value1, value0, "write struct test failed")
