def call_processor(self, input, client_type, protocol_type,
client_principal):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf, client_types=[client_type])
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf)
ctx = TCppConnectionContext(client_principal)
self.processor.process(prot, prot, ctx)
# And on the way out, we need to strip off the header,
# because the C++ code will expect to add it.
read_buf = TMemoryBuffer(prot_buf.getvalue())
trans = THeaderTransport(read_buf, client_types=[client_type])
trans.readFrame(0)
return trans.cstringio_buf.read()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
def test_forward_compatibility_nested(self):
obj = OldStructureNested()
obj.features = [{}]
obj.features[0][1] = 314
obj.features[0][2] = 271
trans = TMemoryBuffer()
proto = self.createProto(trans)
obj.write(proto)
obj_new = NewStructureNested()
trans = TMemoryBuffer(trans.getvalue())
proto = proto.__class__(trans)
fastproto.decode(obj_new, trans, [obj_new.__class__, obj_new.thrift_spec,
obj_new.isUnion()], utf8strings=0,
protoid=self.PROTO,
forward_compatibility=True)
self.assertAlmostEqual(obj_new.features[0][1], 314.0)
self.assertAlmostEqual(obj_new.features[0][2], 271.0)
trans2 = TMemoryBuffer()
proto2 = self.createProto(trans2)
obj_new.write(proto2)
obj_new2 = NewStructureNested()
trans2 = TMemoryBuffer(trans2.getvalue())
proto2 = proto2.__class__(trans2)
fastproto.decode(obj_new2, trans2, [obj_new2.__class__, obj_new2.thrift_spec,
obj_new2.isUnion()], utf8strings=0,
protoid=self.PROTO)
self.assertAlmostEqual(obj_new2.features[0][1], 314.0)
self.assertAlmostEqual(obj_new2.features[0][2], 271.0)
def create_transaction_with_smart_contract(self, code, fee, keys):
tr = Transaction()
contract = SmartContractInvocation()
contract.smartContractDeploy = SmartContractDeploy()
if code == "":
code = 'import com.credits.scapi.annotations.*; import com.credits.scapi.v0.*; public class ' \
'MySmartContract extends SmartContract { public MySmartContract() {} public String hello2(String ' \
'say) { return \"Hello\" + say; } }'
contractText = self.normalizeCode(code)
result = self.compile_smart(contractText)
contract.smartContractDeploy.byteCodeObjects = result.byteCodeObjects
tr.smartContract = contract
tr.smartContract.smartContractDeploy.sourceCode = contractText
tr.source = keys.public_key_bytes
w = self.client.WalletTransactionsCountGet(tr.source)
lastInnerId = bytearray(
(w.lastTransactionInnerId + 1).to_bytes(6, 'little'))
tr.id = int.from_bytes(lastInnerId, byteorder='little', signed=False)
tr.target = self.createContractAddress(tr.source, lastInnerId,
contract)
tr.amount = Amount()
tr.amount.integral = 0
tr.amount.fraction = 0
tr.balance = Amount()
tr.balance.integral = 0
tr.balance.fraction = 0
tr.currency = 1
tr.fee = AmountCommission()
tr.fee.commission = self.double_to_fee(fee)
tr.userFields = ""
ufNum1 = bytearray(b'\x01')
contract.smartContractDeploy.hashState = ""
contract.smartContractDeploy.tokenStandard = 0
contract.method = ""
contract.params = []
contract.usedContracts = []
contract.forgetNewState = False
transportOut = TMemoryBuffer()
protocolOut = TBinaryProtocol(transportOut)
contract.write(protocolOut)
scBytes = transportOut.getvalue()
sMap = '=6s32s32slqhb1s4s' + str(
len(scBytes)
) + 's' #4s' + str(scriptLength) + 's4s' + str(codeNameLength) + 's4s' + str(codeLength) + 's' #len(userField_bytes)
serial_transaction_for_sign = pack(
sMap, #'=' - without alignment
lastInnerId, #6s - 6 byte InnerID (char[] C Type)
tr.source, #32s - 32 byte source public key (char[] C Type)
tr.target, #32s - 32 byte target pyblic key (char[] C Type)
tr.amount.integral, #i - 4 byte integer(int C Type)
tr.amount.fraction, #q - 8 byte integer(long long C Type)
tr.fee.commission, #h - 2 byte integer (short C Type)
tr.currency, #b - 1 byte integer (signed char C Type)
ufNum1,
bytes(len(scBytes).to_bytes(4, byteorder="little")),
scBytes)
signing_key = ed25519.SigningKey(
keys.private_key_bytes) # Create object for calulate signing
tr.signature = signing_key.sign(serial_transaction_for_sign)
return tr
def round_robin(self, compress=None):
original = b'A' * MAX_FRAME_SIZE
mb = TMemoryBuffer()
trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER)
trans.set_max_frame_size(MAX_FRAME_SIZE + MIN_HEADER_SIZE)
if compress:
trans.add_transform(compress)
trans.write(original)
trans.flush()
frame = mb.getvalue()
# Cleanup the memory buffer
mb.close()
del mb
if compress is None:
# Partial Decode the frame and see if its correct size wise
sz = struct.unpack('!I', frame[:4])[0]
self.assertEqual(sz, BIG_FRAME_MAGIC)
sz = struct.unpack('!Q', frame[4:12])[0]
self.assertEqual(len(frame), sz + 12)
# Read it back
mb = TMemoryBuffer(frame)
trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER)
trans.set_max_frame_size(len(frame))
trans.readFrame(0)
result = trans.read(MAX_FRAME_SIZE)
mb.close()
del mb
self.assertEqual(result, original, 'round-robin different from original')
def test_no_trace_headers(self, getrandbits):
getrandbits.return_value = 1234
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client = TestService.Client(client_prot)
try:
client.example_simple()
except TTransportException:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
self.assertEqual(self.observer.on_server_span_created.call_count, 1)
context, server_span = self.observer.on_server_span_created.call_args[
0]
self.assertEqual(server_span.trace_id, 1234)
self.assertEqual(server_span.parent_id, None)
self.assertEqual(server_span.id, 1234)
self.assertEqual(self.server_observer.on_start.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_args[0], (None, ))
def test_with_headers(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client_header_trans = client_prot.trans
client_header_trans.set_header("Trace", "1234")
client_header_trans.set_header("Parent", "2345")
client_header_trans.set_header("Span", "3456")
client_header_trans.set_header("Sampled", "1")
client_header_trans.set_header("Flags", "1")
client = TestService.Client(client_prot)
try:
client.example_simple()
except TTransportException:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
self.assertEqual(self.observer.on_server_span_created.call_count, 1)
context, server_span = self.observer.on_server_span_created.call_args[0]
self.assertEqual(server_span.trace_id, 1234)
self.assertEqual(server_span.parent_id, 2345)
self.assertEqual(server_span.id, 3456)
self.assertTrue(server_span.sampled)
self.assertEqual(server_span.flags, 1)
self.assertEqual(self.server_observer.on_start.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_args[0], (None,))
def test_auth_headers(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client_header_trans = client_prot.trans
client_header_trans.set_header("Authentication", self.VALID_TOKEN)
client_header_trans.set_header("Trace", "1234")
client_header_trans.set_header("Parent", "2345")
client_header_trans.set_header("Span", "3456")
client_header_trans.set_header("Sampled", "1")
client_header_trans.set_header("Flags", "1")
client = TestService.Client(client_prot)
try:
client.example_simple()
except TTransportException:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
context, _ = self.observer.on_server_span_created.call_args[0]
try:
self.assertTrue(context.authentication.valid)
self.assertEqual(context.authentication.account_id, "test_user_id")
except jwt.exceptions.InvalidAlgorithmError:
raise unittest.SkipTest("cryptography is not installed")
def test_primitive_serialization(self):
val = int(12)
buf = TMemoryBuffer()
proto = TCompactProtocol(buf)
proto.writeI32(val)
reader = TCompactProtocol(StringIO(buf.getvalue()))
self.assertEqual(reader.readI32(), val)
def inner_processor(message_buffer):
in_transport = TMemoryBuffer(message_buffer.getvalue())
out_transport = TMemoryBuffer()
in_prot = proto_factory.getProtocol(in_transport)
out_prot = proto_factory.getProtocol(out_transport)
method = processor.process(in_prot, out_prot)
return (method, out_transport.getvalue())
def post(self, request):
out_format = request.POST.get('OutFormat', 'json')
info = CouponInfo(
cid=coupon['cid'],
name=coupon['name'],
amount=coupon['amount'],
progress=coupon['progress'],
start_time=coupon['start_time'],
expired_time=coupon['expired_time'],
ctype=coupon['ctype'],
use_type=coupon['use_type'],
min_invest_time=coupon['min_invest_time'],
max_invest_time=coupon['max_invest_time'],
min_amount=coupon['min_amount'],
des=coupon['desc'],
)
res = CouponInfoResponse(code=0, message='success', info=info)
tMemory_b = TMemoryBuffer()
if out_format == 'binary':
tBinaryProtocol_b = TCompactProtocol(tMemory_b)
content_type = 'application/octet-stream'
elif out_format == 'tjson':
tBinaryProtocol_b = TJSONProtocol(tMemory_b)
content_type = 'application/json'
else:
tBinaryProtocol_b = TSimpleJSONProtocol(tMemory_b)
content_type = 'application/json'
res.write(tBinaryProtocol_b)
memory_buffer = tMemory_b.getvalue()
return HttpResponse(content=memory_buffer, content_type=content_type)
def test_b3_trace_headers(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocolFactory().getProtocol(client_memory_trans)
client_header_trans = client_prot.trans
client_header_trans.set_header(b"B3-TraceId", b"1234")
client_header_trans.set_header(b"B3-ParentSpanId", b"2345")
client_header_trans.set_header(b"B3-SpanId", b"3456")
client_header_trans.set_header(b"B3-Sampled", b"1")
client_header_trans.set_header(b"B3-Flags", b"1")
client = TestService.Client(client_prot)
try:
client.example_simple()
except (TTransportException, EOFError):
pass # we don't have a test response for the client
itrans = TMemoryBuffer(client_memory_trans.getvalue())
iprot = THeaderProtocolFactory().getProtocol(itrans)
self.processor.process(iprot, self.oprot)
self.assertEqual(self.observer.on_server_span_created.call_count, 1)
context, server_span = self.observer.on_server_span_created.call_args[
0]
self.assertEqual(server_span.trace_id, 1234)
self.assertEqual(server_span.parent_id, 2345)
self.assertEqual(server_span.id, 3456)
self.assertTrue(server_span.sampled)
self.assertEqual(server_span.flags, 1)
with self.assertRaises(NoAuthenticationError):
context.request_context.user.id
self.assertEqual(self.server_observer.on_start.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_args[0], (None, ))
def test_with_headers(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client_header_trans = client_prot.trans
client_header_trans.set_header("Trace", "1234")
client_header_trans.set_header("Parent", "2345")
client_header_trans.set_header("Span", "3456")
client = BaseplateService.Client(client_prot)
try:
client.is_healthy()
except:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
self.assertEqual(self.observer.on_root_span_created.call_count, 1)
context, root_span = self.observer.on_root_span_created.call_args[0]
self.assertEqual(root_span.trace_id, "1234")
self.assertEqual(root_span.parent_id, "2345")
self.assertEqual(root_span.id, "3456")
self.assertTrue(self.root_observer.on_start.called)
self.assertTrue(self.root_observer.on_stop.called)
def make_ver_req(peer_ip, peer_port, node_id):
m = TMemoryBuffer()
# p = TBinaryProtocol(m)
p = TCompactProtocol(m)
msg_name = "ver_req"
nonce = "0" #get_random_id()
head = ttypes_header.msg_header(0, msg_name, nonce)
head.write(p)
# print("nonce:%s, node_id:%s" %(nonce, node_id))
addr_me = network_address("127.0.0.1", 1)
addr_you = network_address(peer_ip, peer_port)
time_stamp = int(time.time())
req = ver_req_body(node_id, 0, 0, time_stamp, addr_me, addr_you, 1)
# p.writeMessageBegin()
req.write(p)
# p.writeMessageEnd()
m.flush()
v = m.getvalue()
# print v
# print [hex(ord(i)) for i in v]
c = pack_head(m)
# pprint(c)
# print [hex(ord(i)) for i in c]
return c
def __auth_headers(self, headers, body, support_account_key):
auth_headers = dict()
if self.credential and self.credential.type and self.credential.secretKeyId:
if self.credential.type in SIGNATURE_SUPPORT:
auth_headers[HOST] = self.host
# timestamp
auth_headers[TIMESTAMP] = str(int(time.time() + self.__clock_offset))
auth_headers[MI_DATE] = formatdate(usegmt=True)
# content md5
auth_headers[CONTENT_MD5] = hashlib.md5(body).hexdigest()
headers_to_sign = defaultdict(lambda :[])
for k, v in headers.iteritems():
headers_to_sign[str(k).lower()].append(v)
for k, v in auth_headers.iteritems():
headers_to_sign[str(k).lower()].append(v)
signature = base64.b64encode(self.sign(self.__form_sign_content("POST", self.uri,
headers_to_sign))).strip()
auth_string = "Galaxy-V2 %s:%s" % (self.credential.secretKeyId, signature)
auth_headers[AUTHORIZATION] = auth_string
else:
auth_header = HttpAuthorizationHeader()
auth_header.secretKeyId = self.credential.secretKeyId
auth_header.userType = self.credential.type
auth_header.secretKey = self.credential.secretKey
auth_header.supportAccountKey = support_account_key
mb = TMemoryBuffer()
protocol = TJSONProtocol(mb)
auth_header.write(protocol)
auth_headers[AUTHORIZATION] = str(mb.getvalue())
return auth_headers
def round_robin(self, compress=None):
original = b'A' * MAX_FRAME_SIZE
mb = TMemoryBuffer()
trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER)
trans.set_max_frame_size(MAX_FRAME_SIZE + MIN_HEADER_SIZE)
if compress:
trans.add_transform(compress)
trans.write(original)
trans.flush()
frame = mb.getvalue()
# Cleanup the memory buffer
mb.close()
del mb
if compress is None:
# Partial Decode the frame and see if its correct size wise
sz = struct.unpack('!I', frame[:4])[0]
self.assertEqual(sz, BIG_FRAME_MAGIC)
sz = struct.unpack('!Q', frame[4:12])[0]
self.assertEqual(len(frame), sz + 12)
# Read it back
mb = TMemoryBuffer(frame)
trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER)
trans.set_max_frame_size(len(frame))
trans.readFrame(0)
result = trans.read(MAX_FRAME_SIZE)
mb.close()
del mb
self.assertEqual(result, original, 'round-robin different from original')
def test_with_headers(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client_header_trans = client_prot.trans
client_header_trans.set_header("Trace", "1234")
client_header_trans.set_header("Parent", "2345")
client_header_trans.set_header("Span", "3456")
client_header_trans.set_header("Sampled", "1")
client_header_trans.set_header("Flags", "1")
client = TestService.Client(client_prot)
try:
client.example_simple()
except TTransportException:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
self.assertEqual(self.observer.on_server_span_created.call_count, 1)
context, server_span = self.observer.on_server_span_created.call_args[
0]
self.assertEqual(server_span.trace_id, 1234)
self.assertEqual(server_span.parent_id, 2345)
self.assertEqual(server_span.id, 3456)
self.assertTrue(server_span.sampled)
self.assertEqual(server_span.flags, 1)
self.assertFalse(context.authentication.defined)
self.assertEqual(self.server_observer.on_start.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_args[0], (None, ))
def test_with_headers(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client_header_trans = client_prot.trans
client_header_trans.set_header("Trace", "1234")
client_header_trans.set_header("Parent", "2345")
client_header_trans.set_header("Span", "3456")
client = BaseplateService.Client(client_prot)
try:
client.is_healthy()
except:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
self.assertEqual(self.observer.on_root_span_created.call_count, 1)
context, root_span = self.observer.on_root_span_created.call_args[0]
self.assertEqual(root_span.trace_id, "1234")
self.assertEqual(root_span.parent_id, "2345")
self.assertEqual(root_span.id, "3456")
self.assertTrue(self.root_observer.on_start.called)
self.assertTrue(self.root_observer.on_stop.called)
def handle_request(self, event):
# t = time.time()
# 0.2ms
# 1. 将zeromq的消息转换成为 thrift的 protocols
trans_input = TMemoryBuffer(event.msg)
trans_output = TMemoryBuffer()
proto_input = self.proto_factory_input.getProtocol(trans_input)
proto_output = self.proto_factory_output.getProtocol(trans_output)
# 2. 交给processor来处理
try:
self.processor.process(proto_input, proto_output)
# 3. 将thirft的结果转换成为 zeromq 格式的数据
msg = trans_output.getvalue()
# print "Return Msg: ", msg, event.id
if self.profile:
event.id.extend(["", "%.4f" % time.time()])
self.events.emit(msg, event.id)
else:
self.events.emit(msg, event.id)
except Exception as e:
# 如何出现了异常该如何处理呢
# 程序不能挂
logging.exception("Exception: %s", e)
def decode_helper(self, obj, split=1.0):
trans = TMemoryBuffer()
if self.PROTO == 0:
proto = TBinaryProtocol.TBinaryProtocol(trans)
else:
proto = TCompactProtocol.TCompactProtocol(trans)
obj.write(proto)
index = int(split * len(trans.getvalue()))
trans = ReadOnlyBufferWithRefill(index, trans.getvalue())
obj_new = obj.__class__()
fastproto.decode(obj_new, trans, [obj.__class__, obj.thrift_spec,
obj.isUnion()], utf8strings=0, protoid=self.PROTO)
self.assertEqual(obj, obj_new)
# Verify the entire buffer is read
self.assertEqual(len(trans._readBuffer.read()), 0)
if split != 1.0:
self.assertEqual(1, trans.refill_called)
def to_string(log, page_info):
str_page_info = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryServerProtocol(tMemory_b)
page_info.write(tBinaryProtocol_b)
str_page_info = tMemory_b.getvalue()
except EOFError, e:
log.warning("cann't write DownLoadRsp to string")
def write_communication_to_buffer(comm):
'''
Serialize communication to buffer (binary string) and return
buffer.
'''
transport = TMemoryBuffer()
protocol = factory.createProtocol(transport)
comm.write(protocol)
return transport.getvalue()
def to_string(self, download_req_info):
str_req = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryProtocol(tMemory_b)
download_req_info.write(tBinaryProtocol_b)
str_req = tMemory_b.getvalue()
except EOFError, e:
self.log.warning("cann't write DownLoadReq to string")
def to_string(page_info):
str_page_info = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryServerProtocol(tMemory_b)
page_info.write(tBinaryProtocol_b)
str_page_info = tMemory_b.getvalue()
except EOFError, e:
pass
def to_string(self, link_info):
str_entity = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryProtocol.TBinaryProtocol(tMemory_b)
link_info.write(tBinaryProtocol_b)
str_entity = tMemory_b.getvalue()
except EOFError, e:
self.log.warning("can't write LinkAttr to string")
def decode_helper(self, obj, split=1.0):
trans = TMemoryBuffer()
if self.PROTO == 0:
proto = TBinaryProtocol.TBinaryProtocol(trans)
else:
proto = TCompactProtocol.TCompactProtocol(trans)
obj.write(proto)
index = int(split * len(trans.getvalue()))
trans = ReadOnlyBufferWithRefill(index, trans.getvalue())
obj_new = obj.__class__()
fastproto.decode(obj_new, trans, [obj.__class__, obj.thrift_spec,
obj.isUnion()], utf8strings=0, protoid=self.PROTO)
self.assertEqual(obj, obj_new)
# Verify the entire buffer is read
self.assertEqual(len(trans._readBuffer.read()), 0)
if split != 1.0:
self.assertEqual(1, trans.refill_called)
def write_communication_to_buffer(comm):
'''
Serialize communication to buffer (binary string) and return
buffer.
'''
transport = TMemoryBuffer()
protocol = factory.createProtocol(transport)
comm.write(protocol)
return transport.getvalue()
def call_processor(self, input, headers, client_type, protocol_type,
context_data):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
ctx = TCppConnectionContext(context_data)
self.processor.process(prot, prot, ctx)
# Check for empty result. If so, return an empty string
# here. This is probably a oneway request, but we can't
# reliably tell. The C++ code does basically the same
# thing.
response = prot_buf.getvalue()
if len(response) == 0:
return response
# And on the way out, we need to strip off the header,
# because the C++ code will expect to add it.
read_buf = TMemoryBuffer(response)
trans = THeaderTransport(read_buf, client_types=[client_type])
trans.readFrame(len(response))
return trans.cstringio_buf.read()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
def call_processor(self, input, headers, client_type, protocol_type,
context_data):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
ctx = TCppConnectionContext(context_data)
self.processor.process(prot, prot, ctx)
# Check for empty result. If so, return an empty string
# here. This is probably a oneway request, but we can't
# reliably tell. The C++ code does basically the same
# thing.
response = prot_buf.getvalue()
if len(response) == 0:
return response
# And on the way out, we need to strip off the header,
# because the C++ code will expect to add it.
read_buf = TMemoryBuffer(response)
trans = THeaderTransport(read_buf, client_types=[client_type])
trans.readFrame(len(response))
return trans.cstringio_buf.read()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
def test_forward_compatibility_nested_nested(self):
obj = OldStructureNested()
obj.features = [{}]
obj.features[0][1] = 314
obj.features[0][2] = 271
objN = OldStructureNestedNested()
objN.field = obj
trans = TMemoryBuffer()
proto = self.createProto(trans)
objN.write(proto)
obj_new = NewStructureNestedNested()
trans = TMemoryBuffer(trans.getvalue())
proto = proto.__class__(trans)
fastproto.decode(
obj_new,
trans, [obj_new.__class__, obj_new.thrift_spec,
obj_new.isUnion()],
utf8strings=0,
protoid=self.PROTO,
forward_compatibility=True)
self.assertAlmostEqual(obj_new.field.features[0][1], 314.0)
self.assertAlmostEqual(obj_new.field.features[0][2], 271.0)
trans2 = TMemoryBuffer()
proto2 = self.createProto(trans2)
obj_new.write(proto2)
obj_new2 = NewStructureNestedNested()
trans2 = TMemoryBuffer(trans2.getvalue())
proto2 = proto2.__class__(trans2)
fastproto.decode(
obj_new2,
trans2,
[obj_new2.__class__, obj_new2.thrift_spec,
obj_new2.isUnion()],
utf8strings=0,
protoid=self.PROTO)
self.assertAlmostEqual(obj_new2.field.features[0][1], 314.0)
self.assertAlmostEqual(obj_new2.field.features[0][2], 271.0)
def call_processor(self, input, headers, client_type, protocol_type,
context_data, callback):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
should_sample = self._shouldSample()
timestamps = CallTimestamps()
timestamps.processBegin = 0
timestamps.processEnd = 0
if self.observer and should_sample:
timestamps.processBegin = int(time.time() * 10**6)
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
ctx = TCppConnectionContext(context_data)
ret = self.processor.process(prot, prot, ctx)
done_callback = partial(_ProcessorAdapter.done,
prot_buf=prot_buf,
client_type=client_type,
callback=callback)
if self.observer:
if should_sample:
timestamps.processEnd = int(time.time() * 10**6)
# This only bumps counters if `processBegin != 0` and
# `processEnd != 0` and these will only be non-zero if
# we are sampling this request.
self.observer.callCompleted(timestamps)
# This future is created by and returned from the processor's
# ThreadPoolExecutor, which keeps a reference to it. So it is
# fine for this future to end its lifecycle here.
if isinstance(ret, Future):
ret.add_done_callback(lambda x, d=done_callback: d())
else:
done_callback()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
def req_to_string(req):
str_req = ""
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryProtocol(tMemory_b)
req.write(tBinaryProtocol_b)
str_req = tMemory_b.getvalue()
except:
log.error('crawled_failt\terror:%s' % (traceback.format_exc()))
return str_req
def to_string(self, link_info):
str_entity = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryProtocol(tMemory_b)
link_info.write(tBinaryProtocol_b)
str_entity = tMemory_b.getvalue()
except:
self.log.warning("cann't write EntityExtractorInfo to string")
return str_entity
def serialize_object(obj):
"""
序列化 thrift 对象
:param obj:
:return:
"""
memory = TMemoryBuffer()
binary_protocol = TBinaryProtocol(memory)
obj.write(binary_protocol)
return memory.getvalue()
def to_string(self, page_info):
str_page_info = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryServerProtocol(tMemory_b)
page_info.write(tBinaryProtocol_b)
str_page_info = tMemory_b.getvalue()
except EOFError as e:
self._log.warning("cann't write data to string")
return str_page_info
def to_string(self, download_rsp):
str_rsq = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryProtocol(tMemory_b)
download_rsp.write(tBinaryProtocol_b)
str_rsq = tMemory_b.getvalue()
self.log.info('data-length is {}'.format(str(len(str_rsq))))
except EOFError, e:
self.log.warning("cann't write PageParseInfo to string")
def to_string(self, pageparse_info):
str_parse = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryProtocol(tMemory_b)
pageparse_info.write(tBinaryProtocol_b)
str_parse = tMemory_b.getvalue()
self.log.info('data-length is {}'.format(str(len(str_parse))))
except EOFError, e:
self.log.warning("cann't write PageParseInfo to string")
def test_client_proxy_flow(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
class Pool(object):
@contextlib.contextmanager
def connection(self):
yield client_prot
client_factory = ThriftContextFactory(Pool(), TestService.Client)
span = mock.MagicMock()
child_span = span.make_child().__enter__()
child_span.trace_id = 1
child_span.parent_id = 1
child_span.id = 1
child_span.sampled = True
child_span.flags = None
# We decode the token to unicode to make sure that it is converted to
# bytes correctly by the AuthenticationContext. We do this because a
# unicode token in Python 2 ends up causing a UnicodeDecodeError when
# Thrift tries to write the header.
unicode_token = self.VALID_TOKEN.decode()
auth_context = AuthenticationContext(
token=unicode_token,
secrets=self.secrets,
)
edge_context = EdgeRequestContext(
authentication_context=auth_context,
header=self.SERIALIZED_REQUEST_HEADER,
)
edge_context.attach_context(child_span.context)
client = client_factory.make_object_for_context("test", span)
try:
client.example_simple()
except TTransportException:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
context, _ = self.observer.on_server_span_created.call_args[0]
try:
self.assertEqual(context.request_context.user.id, "test_user_id")
self.assertEqual(context.request_context.user.roles, set())
self.assertEqual(context.request_context.user.is_logged_in, True)
self.assertEqual(context.request_context.user.loid, "t2_deadbeef")
self.assertEqual(context.request_context.user.cookie_created_ms,
100000)
self.assertEqual(context.request_context.oauth_client.id, None)
self.assertFalse(
context.request_context.oauth_client.is_type("third_party"))
self.assertEqual(context.request_context.session.id, "beefdead")
except jwt.exceptions.InvalidAlgorithmError:
raise unittest.SkipTest("cryptography is not installed")
def get_src_json(parse_info):
from bdp.i_crawler.i_entity_extractor.ttypes import EntitySource
from thrift.protocol.TJSONProtocol import TSimpleJSONProtocol
from thrift.transport.TTransport import TMemoryBuffer
entity_source = EntitySource(url=parse_info.base_info.url, site_id=parse_info.base_info.site_id,
site=parse_info.base_info.site,
download_time=parse_info.crawl_info.download_time)
t_mem_buf = TMemoryBuffer()
t_json_protocol = TSimpleJSONProtocol(t_mem_buf)
entity_source.write(t_json_protocol)
return json.loads(t_mem_buf.getvalue())
def to_string(self, obj):
str_parse = None
try:
tMemory_b = TMemoryBuffer()
tBinaryProtocol_b = TBinaryProtocol(tMemory_b)
obj.write(tBinaryProtocol_b)
str_parse = tMemory_b.getvalue()
except EOFError as e:
print e.message
pass
return str_parse
def write_to_read(write_struct, read_struct):
write_buffer = TMemoryBuffer()
write_protocol = TBinaryProtocol(write_buffer)
write_struct.write(write_protocol)
# The implementation of TMemoryBuffer is slightly different from C++
# the read/write buffer is not shared, thus we have to create another
# TMemoryBuffer
read_buffer = TMemoryBuffer(write_buffer.getvalue())
read_protocol = TBinaryProtocol(read_buffer)
read_struct.read(read_protocol)
def encode_and_decode(self, obj):
trans = TMemoryBuffer()
if self.PROTO == 0:
proto = TBinaryProtocol.TBinaryProtocol(trans)
else:
proto = TCompactProtocol.TCompactProtocol(trans)
obj.write(proto)
obj_new = obj.__class__()
trans = TMemoryBuffer(trans.getvalue())
proto = proto.__class__(trans)
obj_new.read(proto)
def http_handler(request):
if request.method != 'POST':
return HttpResponseNotAllowed(['GET', 'PUT'])
server_info.client_ip = request.META['REMOTE_ADDR']
if server_info.client_ip[0:7] == '::ffff:':
server_info.client_ip = server_info.client_ip[7:]
server_info.client_port = None
itrans = TMemoryBuffer(request.body)
otrans = TMemoryBuffer()
iproto = TBinaryProtocol(itrans)
oproto = TBinaryProtocol(otrans)
thrift_handler(iproto, oproto)
return HttpResponse(otrans.getvalue(), content_type="application/x-thrift")
def message_received(self, frame):
tmi = TMemoryBuffer(frame)
tmo = TMemoryBuffer()
iprot = THeaderProtocol(tmi)
oprot = THeaderProtocol(tmo)
try:
yield from self.processor.process(iprot, oprot, self.server_context)
msg = tmo.getvalue()
if len(msg) > 0:
self.transport.write(msg)
except Exception:
logging.exception("Exception while processing request")
self.transport.close()
def write_communication_to_buffer(comm):
'''
Serialize communication to buffer (binary string) and return
buffer.
Args:
comm (Communication): communication to serialize
Returns:
Communication: Communication read from buffer
'''
transport = TMemoryBuffer()
protocol = factory.createProtocol(transport)
comm.write(protocol)
return transport.getvalue()
def test_expected_exception_not_passed_to_server_span_finish(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client = TestService.Client(client_prot)
try:
client.example_throws(crash=False)
except TTransportException:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
self.assertEqual(self.server_observer.on_start.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_count, 1)
self.assertEqual(self.server_observer.on_finish.call_args[0], (None,))
def call(self, procedure, args):
if isinstance(procedure, str):
try:
procedure = processor._procedures[procedure]
except KeyError:
raise TApplicationException(TApplicationException.UNKNOWN_METHOD,
"Unknown method '{0}'".format(name))
otrans = TMemoryBuffer()
oproto = TBinaryProtocol(otrans)
# perf.begin('send')
oproto.writeMessageBegin(procedure.name, TMessageType.CALL, self._processor.seqid)
self._processor.seqid = self._processor.seqid + 1
self._processor.send_struct(Namespace(args), procedure.parameters_struct, oproto)
oproto.writeMessageEnd()
# perf.end('send')
self._http.request('POST', self._url, otrans.getvalue(), {})
resp = self._http.getresponse()
data = resp.read()
iproto = TBinaryProtocol(TMemoryBuffer(data))
# perf.begin('wait')
(fname, mtype, rseqid) = iproto.readMessageBegin()
# perf.end('wait')
# perf.begin('recv')
if mtype == TMessageType.EXCEPTION:
x = TApplicationException()
x.read(iproto)
iproto.readMessageEnd()
x.args = (x.message,)
raise x
result = self._processor.recv_struct(procedure.results_struct, iproto)
iproto.readMessageEnd()
# perf.end('recv')
# perf.end('call')
if result.result is not None:
return result.result
for field in procedure.results_struct.fields:
if getattr(result, field.name) is not None:
raise getattr(result, field.name)
return None
def schedule_timeout(self, fname, seqid):
timeout = self.timeouts[fname]
if not timeout:
return
tmo = TMemoryBuffer()
thp = THeaderTransport(tmo)
oprot = THeaderProtocol(thp)
exc = TApplicationException(
TApplicationException.TIMEOUT, "Call to {} timed out".format(fname)
)
oprot.writeMessageBegin(fname, TMessageType.EXCEPTION, seqid)
exc.write(oprot)
oprot.writeMessageEnd()
thp.flush()
timeout_task = self.loop.create_task(
self.message_received(tmo.getvalue(), delay=timeout),
)
self.update_pending_tasks(seqid, timeout_task)
def handle_request(self, event):
# 1. 将zeromq的消息转换成为 thrift的 protocols
trans_input = TMemoryBuffer(event.msg)
trans_output = TMemoryBuffer()
proto_input = self.proto_factory_input.getProtocol(trans_input)
proto_output = self.proto_factory_output.getProtocol(trans_output)
# 2. 交给processor来处理
try:
self.processor.process(proto_input, proto_output)
# 3. 将thirft的结果转换成为 zeromq 格式的数据
msg = trans_output.getvalue()
# print "Return Msg: ", msg, event.id
self.events.emit(msg, event.id)
except Exception as e:
# 如何出现了异常该如何处理呢
# 程序不能挂
print "Exception: ", e
def message_received(self, frame):
# We support the deprecated FRAMED transport for old fb303
# clients that were otherwise failing miserably.
client_types = {
THeaderTransport.HEADERS_CLIENT_TYPE,
THeaderTransport.FRAMED_DEPRECATED,
}
tm = TMemoryBuffer(frame)
prot = THeaderProtocol(tm, client_types=client_types)
try:
yield from self.processor.process(
prot, prot, self.server_context,
)
msg = tm.getvalue()
if len(msg) > 0:
self.transport.write(msg)
except Exception:
logger.exception("Exception while processing request")
self.transport.close()
def test_no_headers(self):
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client = BaseplateService.Client(client_prot)
try:
client.is_healthy()
except:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
self.assertEqual(self.observer.on_root_span_created.call_count, 1)
context, root_span = self.observer.on_root_span_created.call_args[0]
self.assertEqual(root_span.trace_id, "no-trace")
self.assertEqual(root_span.parent_id, "no-parent")
self.assertEqual(root_span.id, "no-span")
mock_root_observer = self.observer.on_root_span_created.return_value
self.assertTrue(mock_root_observer.on_start.called)
self.assertTrue(mock_root_observer.on_stop.called)
def call_processor(self, input, headers, client_type, protocol_type,
context_data, callback):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
ctx = TCppConnectionContext(context_data)
ret = self.processor.process(prot, prot, ctx)
done_callback = partial(_ProcessorAdapter.done,
prot_buf=prot_buf,
client_type=client_type,
callback=callback)
# This future is created by and returned from the processor's
# ThreadPoolExecutor, which keeps a reference to it. So it is
# fine for this future to end its lifecycle here.
if isinstance(ret, Future):
ret.add_done_callback(lambda x, d=done_callback: d())
else:
done_callback()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
def test_no_trace_headers(self, getrandbits):
getrandbits.return_value = 1234
client_memory_trans = TMemoryBuffer()
client_prot = THeaderProtocol(client_memory_trans)
client = BaseplateService.Client(client_prot)
try:
client.is_healthy()
except:
pass # we don't have a test response for the client
self.itrans._readBuffer = StringIO(client_memory_trans.getvalue())
self.processor.process(self.iprot, self.oprot, self.server_context)
self.assertEqual(self.observer.on_server_span_created.call_count, 1)
context, server_span = self.observer.on_server_span_created.call_args[0]
self.assertEqual(server_span.trace_id, 1234)
self.assertEqual(server_span.parent_id, None)
self.assertEqual(server_span.id, 1234)
self.assertTrue(self.server_observer.on_start.called)
self.assertTrue(self.server_observer.on_finish.called)
def testNoInitialValue(self):
buf = TMemoryBuffer()
data = buf.read(5)
buf.write(b"world")
self.assertEquals(data, b"")
self.assertEquals(buf.getvalue(), b"world")
def testReadWrite(self):
buf = TMemoryBuffer(b"hello")
data = buf.read(5)
buf.write(b"world")
self.assertEquals(data, b"hello")
self.assertEquals(buf.getvalue(), b"world")
def writeToJSON(obj):
trans = TMemoryBuffer()
proto = TSimpleJSONProtocol.TSimpleJSONProtocol(trans)
obj.write(proto)
return trans.getvalue()
def thriftDumps(tweet, ProtocolClass=TBinaryProtocolAccelerated):
buf = TMemoryBuffer()
protocol = ProtocolClass(buf)
tweet.write(protocol)
return buf.getvalue()
def to_bytes(obj): """Creates the standard binary representation of a thrift object.""" b = TMemoryBuffer() p = TBinaryProtocol(b) obj.write(p) return b.getvalue()
