def call_processor(self, input, client_type, protocol_type, client_principal): try: # The input string has already had the header removed, but # the python processor will expect it to be there. In # order to reconstitute the message with headers, we use # the THeaderProtocol object to write into a memory # buffer, then pass that buffer to the python processor. write_buf = TMemoryBuffer() trans = THeaderTransport(write_buf, client_types=[client_type]) trans.set_protocol_id(protocol_type) trans.write(input) trans.flush() prot_buf = TMemoryBuffer(write_buf.getvalue()) prot = THeaderProtocol(prot_buf) ctx = TCppConnectionContext(client_principal) self.processor.process(prot, prot, ctx) # And on the way out, we need to strip off the header, # because the C++ code will expect to add it. read_buf = TMemoryBuffer(prot_buf.getvalue()) trans = THeaderTransport(read_buf, client_types=[client_type]) trans.readFrame(0) return trans.cstringio_buf.read() except: # Don't let exceptions escape back into C++ traceback.print_exc()
def test_forward_compatibility_nested(self): obj = OldStructureNested() obj.features = [{}] obj.features[0][1] = 314 obj.features[0][2] = 271 trans = TMemoryBuffer() proto = self.createProto(trans) obj.write(proto) obj_new = NewStructureNested() trans = TMemoryBuffer(trans.getvalue()) proto = proto.__class__(trans) fastproto.decode(obj_new, trans, [obj_new.__class__, obj_new.thrift_spec, obj_new.isUnion()], utf8strings=0, protoid=self.PROTO, forward_compatibility=True) self.assertAlmostEqual(obj_new.features[0][1], 314.0) self.assertAlmostEqual(obj_new.features[0][2], 271.0) trans2 = TMemoryBuffer() proto2 = self.createProto(trans2) obj_new.write(proto2) obj_new2 = NewStructureNested() trans2 = TMemoryBuffer(trans2.getvalue()) proto2 = proto2.__class__(trans2) fastproto.decode(obj_new2, trans2, [obj_new2.__class__, obj_new2.thrift_spec, obj_new2.isUnion()], utf8strings=0, protoid=self.PROTO) self.assertAlmostEqual(obj_new2.features[0][1], 314.0) self.assertAlmostEqual(obj_new2.features[0][2], 271.0)
def create_transaction_with_smart_contract(self, code, fee, keys): tr = Transaction() contract = SmartContractInvocation() contract.smartContractDeploy = SmartContractDeploy() if code == "": code = 'import com.credits.scapi.annotations.*; import com.credits.scapi.v0.*; public class ' \ 'MySmartContract extends SmartContract { public MySmartContract() {} public String hello2(String ' \ 'say) { return \"Hello\" + say; } }' contractText = self.normalizeCode(code) result = self.compile_smart(contractText) contract.smartContractDeploy.byteCodeObjects = result.byteCodeObjects tr.smartContract = contract tr.smartContract.smartContractDeploy.sourceCode = contractText tr.source = keys.public_key_bytes w = self.client.WalletTransactionsCountGet(tr.source) lastInnerId = bytearray( (w.lastTransactionInnerId + 1).to_bytes(6, 'little')) tr.id = int.from_bytes(lastInnerId, byteorder='little', signed=False) tr.target = self.createContractAddress(tr.source, lastInnerId, contract) tr.amount = Amount() tr.amount.integral = 0 tr.amount.fraction = 0 tr.balance = Amount() tr.balance.integral = 0 tr.balance.fraction = 0 tr.currency = 1 tr.fee = AmountCommission() tr.fee.commission = self.double_to_fee(fee) tr.userFields = "" ufNum1 = bytearray(b'\x01') contract.smartContractDeploy.hashState = "" contract.smartContractDeploy.tokenStandard = 0 contract.method = "" contract.params = [] contract.usedContracts = [] contract.forgetNewState = False transportOut = TMemoryBuffer() protocolOut = TBinaryProtocol(transportOut) contract.write(protocolOut) scBytes = transportOut.getvalue() sMap = '=6s32s32slqhb1s4s' + str( len(scBytes) ) + 's' #4s' + str(scriptLength) + 's4s' + str(codeNameLength) + 's4s' + str(codeLength) + 's' #len(userField_bytes) serial_transaction_for_sign = pack( sMap, #'=' - without alignment lastInnerId, #6s - 6 byte InnerID (char[] C Type) tr.source, #32s - 32 byte source public key (char[] C Type) tr.target, #32s - 32 byte target pyblic key (char[] C Type) tr.amount.integral, #i - 4 byte integer(int C Type) tr.amount.fraction, #q - 8 byte integer(long long C Type) tr.fee.commission, #h - 2 byte integer (short C Type) tr.currency, #b - 1 byte integer (signed char C Type) ufNum1, bytes(len(scBytes).to_bytes(4, byteorder="little")), scBytes) signing_key = ed25519.SigningKey( keys.private_key_bytes) # Create object for calulate signing tr.signature = signing_key.sign(serial_transaction_for_sign) return tr
def round_robin(self, compress=None): original = b'A' * MAX_FRAME_SIZE mb = TMemoryBuffer() trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER) trans.set_max_frame_size(MAX_FRAME_SIZE + MIN_HEADER_SIZE) if compress: trans.add_transform(compress) trans.write(original) trans.flush() frame = mb.getvalue() # Cleanup the memory buffer mb.close() del mb if compress is None: # Partial Decode the frame and see if its correct size wise sz = struct.unpack('!I', frame[:4])[0] self.assertEqual(sz, BIG_FRAME_MAGIC) sz = struct.unpack('!Q', frame[4:12])[0] self.assertEqual(len(frame), sz + 12) # Read it back mb = TMemoryBuffer(frame) trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER) trans.set_max_frame_size(len(frame)) trans.readFrame(0) result = trans.read(MAX_FRAME_SIZE) mb.close() del mb self.assertEqual(result, original, 'round-robin different from original')
def test_no_trace_headers(self, getrandbits): getrandbits.return_value = 1234 client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client = TestService.Client(client_prot) try: client.example_simple() except TTransportException: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) self.assertEqual(self.observer.on_server_span_created.call_count, 1) context, server_span = self.observer.on_server_span_created.call_args[ 0] self.assertEqual(server_span.trace_id, 1234) self.assertEqual(server_span.parent_id, None) self.assertEqual(server_span.id, 1234) self.assertEqual(self.server_observer.on_start.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_args[0], (None, ))
def test_with_headers(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client_header_trans = client_prot.trans client_header_trans.set_header("Trace", "1234") client_header_trans.set_header("Parent", "2345") client_header_trans.set_header("Span", "3456") client_header_trans.set_header("Sampled", "1") client_header_trans.set_header("Flags", "1") client = TestService.Client(client_prot) try: client.example_simple() except TTransportException: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) self.assertEqual(self.observer.on_server_span_created.call_count, 1) context, server_span = self.observer.on_server_span_created.call_args[0] self.assertEqual(server_span.trace_id, 1234) self.assertEqual(server_span.parent_id, 2345) self.assertEqual(server_span.id, 3456) self.assertTrue(server_span.sampled) self.assertEqual(server_span.flags, 1) self.assertEqual(self.server_observer.on_start.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_args[0], (None,))
def test_auth_headers(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client_header_trans = client_prot.trans client_header_trans.set_header("Authentication", self.VALID_TOKEN) client_header_trans.set_header("Trace", "1234") client_header_trans.set_header("Parent", "2345") client_header_trans.set_header("Span", "3456") client_header_trans.set_header("Sampled", "1") client_header_trans.set_header("Flags", "1") client = TestService.Client(client_prot) try: client.example_simple() except TTransportException: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) context, _ = self.observer.on_server_span_created.call_args[0] try: self.assertTrue(context.authentication.valid) self.assertEqual(context.authentication.account_id, "test_user_id") except jwt.exceptions.InvalidAlgorithmError: raise unittest.SkipTest("cryptography is not installed")
def test_primitive_serialization(self): val = int(12) buf = TMemoryBuffer() proto = TCompactProtocol(buf) proto.writeI32(val) reader = TCompactProtocol(StringIO(buf.getvalue())) self.assertEqual(reader.readI32(), val)
def inner_processor(message_buffer): in_transport = TMemoryBuffer(message_buffer.getvalue()) out_transport = TMemoryBuffer() in_prot = proto_factory.getProtocol(in_transport) out_prot = proto_factory.getProtocol(out_transport) method = processor.process(in_prot, out_prot) return (method, out_transport.getvalue())
def post(self, request): out_format = request.POST.get('OutFormat', 'json') info = CouponInfo( cid=coupon['cid'], name=coupon['name'], amount=coupon['amount'], progress=coupon['progress'], start_time=coupon['start_time'], expired_time=coupon['expired_time'], ctype=coupon['ctype'], use_type=coupon['use_type'], min_invest_time=coupon['min_invest_time'], max_invest_time=coupon['max_invest_time'], min_amount=coupon['min_amount'], des=coupon['desc'], ) res = CouponInfoResponse(code=0, message='success', info=info) tMemory_b = TMemoryBuffer() if out_format == 'binary': tBinaryProtocol_b = TCompactProtocol(tMemory_b) content_type = 'application/octet-stream' elif out_format == 'tjson': tBinaryProtocol_b = TJSONProtocol(tMemory_b) content_type = 'application/json' else: tBinaryProtocol_b = TSimpleJSONProtocol(tMemory_b) content_type = 'application/json' res.write(tBinaryProtocol_b) memory_buffer = tMemory_b.getvalue() return HttpResponse(content=memory_buffer, content_type=content_type)
def test_b3_trace_headers(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocolFactory().getProtocol(client_memory_trans) client_header_trans = client_prot.trans client_header_trans.set_header(b"B3-TraceId", b"1234") client_header_trans.set_header(b"B3-ParentSpanId", b"2345") client_header_trans.set_header(b"B3-SpanId", b"3456") client_header_trans.set_header(b"B3-Sampled", b"1") client_header_trans.set_header(b"B3-Flags", b"1") client = TestService.Client(client_prot) try: client.example_simple() except (TTransportException, EOFError): pass # we don't have a test response for the client itrans = TMemoryBuffer(client_memory_trans.getvalue()) iprot = THeaderProtocolFactory().getProtocol(itrans) self.processor.process(iprot, self.oprot) self.assertEqual(self.observer.on_server_span_created.call_count, 1) context, server_span = self.observer.on_server_span_created.call_args[ 0] self.assertEqual(server_span.trace_id, 1234) self.assertEqual(server_span.parent_id, 2345) self.assertEqual(server_span.id, 3456) self.assertTrue(server_span.sampled) self.assertEqual(server_span.flags, 1) with self.assertRaises(NoAuthenticationError): context.request_context.user.id self.assertEqual(self.server_observer.on_start.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_args[0], (None, ))
def test_with_headers(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client_header_trans = client_prot.trans client_header_trans.set_header("Trace", "1234") client_header_trans.set_header("Parent", "2345") client_header_trans.set_header("Span", "3456") client = BaseplateService.Client(client_prot) try: client.is_healthy() except: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) self.assertEqual(self.observer.on_root_span_created.call_count, 1) context, root_span = self.observer.on_root_span_created.call_args[0] self.assertEqual(root_span.trace_id, "1234") self.assertEqual(root_span.parent_id, "2345") self.assertEqual(root_span.id, "3456") self.assertTrue(self.root_observer.on_start.called) self.assertTrue(self.root_observer.on_stop.called)
def make_ver_req(peer_ip, peer_port, node_id): m = TMemoryBuffer() # p = TBinaryProtocol(m) p = TCompactProtocol(m) msg_name = "ver_req" nonce = "0" #get_random_id() head = ttypes_header.msg_header(0, msg_name, nonce) head.write(p) # print("nonce:%s, node_id:%s" %(nonce, node_id)) addr_me = network_address("127.0.0.1", 1) addr_you = network_address(peer_ip, peer_port) time_stamp = int(time.time()) req = ver_req_body(node_id, 0, 0, time_stamp, addr_me, addr_you, 1) # p.writeMessageBegin() req.write(p) # p.writeMessageEnd() m.flush() v = m.getvalue() # print v # print [hex(ord(i)) for i in v] c = pack_head(m) # pprint(c) # print [hex(ord(i)) for i in c] return c
def __auth_headers(self, headers, body, support_account_key): auth_headers = dict() if self.credential and self.credential.type and self.credential.secretKeyId: if self.credential.type in SIGNATURE_SUPPORT: auth_headers[HOST] = self.host # timestamp auth_headers[TIMESTAMP] = str(int(time.time() + self.__clock_offset)) auth_headers[MI_DATE] = formatdate(usegmt=True) # content md5 auth_headers[CONTENT_MD5] = hashlib.md5(body).hexdigest() headers_to_sign = defaultdict(lambda :[]) for k, v in headers.iteritems(): headers_to_sign[str(k).lower()].append(v) for k, v in auth_headers.iteritems(): headers_to_sign[str(k).lower()].append(v) signature = base64.b64encode(self.sign(self.__form_sign_content("POST", self.uri, headers_to_sign))).strip() auth_string = "Galaxy-V2 %s:%s" % (self.credential.secretKeyId, signature) auth_headers[AUTHORIZATION] = auth_string else: auth_header = HttpAuthorizationHeader() auth_header.secretKeyId = self.credential.secretKeyId auth_header.userType = self.credential.type auth_header.secretKey = self.credential.secretKey auth_header.supportAccountKey = support_account_key mb = TMemoryBuffer() protocol = TJSONProtocol(mb) auth_header.write(protocol) auth_headers[AUTHORIZATION] = str(mb.getvalue()) return auth_headers
def round_robin(self, compress=None): original = b'A' * MAX_FRAME_SIZE mb = TMemoryBuffer() trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER) trans.set_max_frame_size(MAX_FRAME_SIZE + MIN_HEADER_SIZE) if compress: trans.add_transform(compress) trans.write(original) trans.flush() frame = mb.getvalue() # Cleanup the memory buffer mb.close() del mb if compress is None: # Partial Decode the frame and see if its correct size wise sz = struct.unpack('!I', frame[:4])[0] self.assertEqual(sz, BIG_FRAME_MAGIC) sz = struct.unpack('!Q', frame[4:12])[0] self.assertEqual(len(frame), sz + 12) # Read it back mb = TMemoryBuffer(frame) trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER) trans.set_max_frame_size(len(frame)) trans.readFrame(0) result = trans.read(MAX_FRAME_SIZE) mb.close() del mb self.assertEqual(result, original, 'round-robin different from original')
def test_with_headers(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client_header_trans = client_prot.trans client_header_trans.set_header("Trace", "1234") client_header_trans.set_header("Parent", "2345") client_header_trans.set_header("Span", "3456") client_header_trans.set_header("Sampled", "1") client_header_trans.set_header("Flags", "1") client = TestService.Client(client_prot) try: client.example_simple() except TTransportException: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) self.assertEqual(self.observer.on_server_span_created.call_count, 1) context, server_span = self.observer.on_server_span_created.call_args[ 0] self.assertEqual(server_span.trace_id, 1234) self.assertEqual(server_span.parent_id, 2345) self.assertEqual(server_span.id, 3456) self.assertTrue(server_span.sampled) self.assertEqual(server_span.flags, 1) self.assertFalse(context.authentication.defined) self.assertEqual(self.server_observer.on_start.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_args[0], (None, ))
def test_with_headers(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client_header_trans = client_prot.trans client_header_trans.set_header("Trace", "1234") client_header_trans.set_header("Parent", "2345") client_header_trans.set_header("Span", "3456") client = BaseplateService.Client(client_prot) try: client.is_healthy() except: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) self.assertEqual(self.observer.on_root_span_created.call_count, 1) context, root_span = self.observer.on_root_span_created.call_args[0] self.assertEqual(root_span.trace_id, "1234") self.assertEqual(root_span.parent_id, "2345") self.assertEqual(root_span.id, "3456") self.assertTrue(self.root_observer.on_start.called) self.assertTrue(self.root_observer.on_stop.called)
def handle_request(self, event): # t = time.time() # 0.2ms # 1. 将zeromq的消息转换成为 thrift的 protocols trans_input = TMemoryBuffer(event.msg) trans_output = TMemoryBuffer() proto_input = self.proto_factory_input.getProtocol(trans_input) proto_output = self.proto_factory_output.getProtocol(trans_output) # 2. 交给processor来处理 try: self.processor.process(proto_input, proto_output) # 3. 将thirft的结果转换成为 zeromq 格式的数据 msg = trans_output.getvalue() # print "Return Msg: ", msg, event.id if self.profile: event.id.extend(["", "%.4f" % time.time()]) self.events.emit(msg, event.id) else: self.events.emit(msg, event.id) except Exception as e: # 如何出现了异常该如何处理呢 # 程序不能挂 logging.exception("Exception: %s", e)
def decode_helper(self, obj, split=1.0): trans = TMemoryBuffer() if self.PROTO == 0: proto = TBinaryProtocol.TBinaryProtocol(trans) else: proto = TCompactProtocol.TCompactProtocol(trans) obj.write(proto) index = int(split * len(trans.getvalue())) trans = ReadOnlyBufferWithRefill(index, trans.getvalue()) obj_new = obj.__class__() fastproto.decode(obj_new, trans, [obj.__class__, obj.thrift_spec, obj.isUnion()], utf8strings=0, protoid=self.PROTO) self.assertEqual(obj, obj_new) # Verify the entire buffer is read self.assertEqual(len(trans._readBuffer.read()), 0) if split != 1.0: self.assertEqual(1, trans.refill_called)
def to_string(log, page_info): str_page_info = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryServerProtocol(tMemory_b) page_info.write(tBinaryProtocol_b) str_page_info = tMemory_b.getvalue() except EOFError, e: log.warning("cann't write DownLoadRsp to string")
def write_communication_to_buffer(comm): ''' Serialize communication to buffer (binary string) and return buffer. ''' transport = TMemoryBuffer() protocol = factory.createProtocol(transport) comm.write(protocol) return transport.getvalue()
def to_string(self, download_req_info): str_req = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryProtocol(tMemory_b) download_req_info.write(tBinaryProtocol_b) str_req = tMemory_b.getvalue() except EOFError, e: self.log.warning("cann't write DownLoadReq to string")
def to_string(page_info): str_page_info = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryServerProtocol(tMemory_b) page_info.write(tBinaryProtocol_b) str_page_info = tMemory_b.getvalue() except EOFError, e: pass
def to_string(self, link_info): str_entity = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryProtocol.TBinaryProtocol(tMemory_b) link_info.write(tBinaryProtocol_b) str_entity = tMemory_b.getvalue() except EOFError, e: self.log.warning("can't write LinkAttr to string")
def decode_helper(self, obj, split=1.0): trans = TMemoryBuffer() if self.PROTO == 0: proto = TBinaryProtocol.TBinaryProtocol(trans) else: proto = TCompactProtocol.TCompactProtocol(trans) obj.write(proto) index = int(split * len(trans.getvalue())) trans = ReadOnlyBufferWithRefill(index, trans.getvalue()) obj_new = obj.__class__() fastproto.decode(obj_new, trans, [obj.__class__, obj.thrift_spec, obj.isUnion()], utf8strings=0, protoid=self.PROTO) self.assertEqual(obj, obj_new) # Verify the entire buffer is read self.assertEqual(len(trans._readBuffer.read()), 0) if split != 1.0: self.assertEqual(1, trans.refill_called)
def write_communication_to_buffer(comm): ''' Serialize communication to buffer (binary string) and return buffer. ''' transport = TMemoryBuffer() protocol = factory.createProtocol(transport) comm.write(protocol) return transport.getvalue()
def call_processor(self, input, headers, client_type, protocol_type, context_data): try: # The input string has already had the header removed, but # the python processor will expect it to be there. In # order to reconstitute the message with headers, we use # the THeaderProtocol object to write into a memory # buffer, then pass that buffer to the python processor. write_buf = TMemoryBuffer() trans = THeaderTransport(write_buf) trans._THeaderTransport__client_type = client_type trans._THeaderTransport__write_headers = headers trans.set_protocol_id(protocol_type) trans.write(input) trans.flush() prot_buf = TMemoryBuffer(write_buf.getvalue()) prot = THeaderProtocol(prot_buf, client_types=[client_type]) ctx = TCppConnectionContext(context_data) self.processor.process(prot, prot, ctx) # Check for empty result. If so, return an empty string # here. This is probably a oneway request, but we can't # reliably tell. The C++ code does basically the same # thing. response = prot_buf.getvalue() if len(response) == 0: return response # And on the way out, we need to strip off the header, # because the C++ code will expect to add it. read_buf = TMemoryBuffer(response) trans = THeaderTransport(read_buf, client_types=[client_type]) trans.readFrame(len(response)) return trans.cstringio_buf.read() except: # Don't let exceptions escape back into C++ traceback.print_exc()
def call_processor(self, input, headers, client_type, protocol_type, context_data): try: # The input string has already had the header removed, but # the python processor will expect it to be there. In # order to reconstitute the message with headers, we use # the THeaderProtocol object to write into a memory # buffer, then pass that buffer to the python processor. write_buf = TMemoryBuffer() trans = THeaderTransport(write_buf) trans._THeaderTransport__client_type = client_type trans._THeaderTransport__write_headers = headers trans.set_protocol_id(protocol_type) trans.write(input) trans.flush() prot_buf = TMemoryBuffer(write_buf.getvalue()) prot = THeaderProtocol(prot_buf, client_types=[client_type]) ctx = TCppConnectionContext(context_data) self.processor.process(prot, prot, ctx) # Check for empty result. If so, return an empty string # here. This is probably a oneway request, but we can't # reliably tell. The C++ code does basically the same # thing. response = prot_buf.getvalue() if len(response) == 0: return response # And on the way out, we need to strip off the header, # because the C++ code will expect to add it. read_buf = TMemoryBuffer(response) trans = THeaderTransport(read_buf, client_types=[client_type]) trans.readFrame(len(response)) return trans.cstringio_buf.read() except: # Don't let exceptions escape back into C++ traceback.print_exc()
def test_forward_compatibility_nested_nested(self): obj = OldStructureNested() obj.features = [{}] obj.features[0][1] = 314 obj.features[0][2] = 271 objN = OldStructureNestedNested() objN.field = obj trans = TMemoryBuffer() proto = self.createProto(trans) objN.write(proto) obj_new = NewStructureNestedNested() trans = TMemoryBuffer(trans.getvalue()) proto = proto.__class__(trans) fastproto.decode( obj_new, trans, [obj_new.__class__, obj_new.thrift_spec, obj_new.isUnion()], utf8strings=0, protoid=self.PROTO, forward_compatibility=True) self.assertAlmostEqual(obj_new.field.features[0][1], 314.0) self.assertAlmostEqual(obj_new.field.features[0][2], 271.0) trans2 = TMemoryBuffer() proto2 = self.createProto(trans2) obj_new.write(proto2) obj_new2 = NewStructureNestedNested() trans2 = TMemoryBuffer(trans2.getvalue()) proto2 = proto2.__class__(trans2) fastproto.decode( obj_new2, trans2, [obj_new2.__class__, obj_new2.thrift_spec, obj_new2.isUnion()], utf8strings=0, protoid=self.PROTO) self.assertAlmostEqual(obj_new2.field.features[0][1], 314.0) self.assertAlmostEqual(obj_new2.field.features[0][2], 271.0)
def call_processor(self, input, headers, client_type, protocol_type, context_data, callback): try: # The input string has already had the header removed, but # the python processor will expect it to be there. In # order to reconstitute the message with headers, we use # the THeaderProtocol object to write into a memory # buffer, then pass that buffer to the python processor. should_sample = self._shouldSample() timestamps = CallTimestamps() timestamps.processBegin = 0 timestamps.processEnd = 0 if self.observer and should_sample: timestamps.processBegin = int(time.time() * 10**6) write_buf = TMemoryBuffer() trans = THeaderTransport(write_buf) trans._THeaderTransport__client_type = client_type trans._THeaderTransport__write_headers = headers trans.set_protocol_id(protocol_type) trans.write(input) trans.flush() prot_buf = TMemoryBuffer(write_buf.getvalue()) prot = THeaderProtocol(prot_buf, client_types=[client_type]) ctx = TCppConnectionContext(context_data) ret = self.processor.process(prot, prot, ctx) done_callback = partial(_ProcessorAdapter.done, prot_buf=prot_buf, client_type=client_type, callback=callback) if self.observer: if should_sample: timestamps.processEnd = int(time.time() * 10**6) # This only bumps counters if `processBegin != 0` and # `processEnd != 0` and these will only be non-zero if # we are sampling this request. self.observer.callCompleted(timestamps) # This future is created by and returned from the processor's # ThreadPoolExecutor, which keeps a reference to it. So it is # fine for this future to end its lifecycle here. if isinstance(ret, Future): ret.add_done_callback(lambda x, d=done_callback: d()) else: done_callback() except: # Don't let exceptions escape back into C++ traceback.print_exc()
def req_to_string(req): str_req = "" try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryProtocol(tMemory_b) req.write(tBinaryProtocol_b) str_req = tMemory_b.getvalue() except: log.error('crawled_failt\terror:%s' % (traceback.format_exc())) return str_req
def to_string(self, link_info): str_entity = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryProtocol(tMemory_b) link_info.write(tBinaryProtocol_b) str_entity = tMemory_b.getvalue() except: self.log.warning("cann't write EntityExtractorInfo to string") return str_entity
def serialize_object(obj): """ 序列化 thrift 对象 :param obj: :return: """ memory = TMemoryBuffer() binary_protocol = TBinaryProtocol(memory) obj.write(binary_protocol) return memory.getvalue()
def to_string(self, page_info): str_page_info = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryServerProtocol(tMemory_b) page_info.write(tBinaryProtocol_b) str_page_info = tMemory_b.getvalue() except EOFError as e: self._log.warning("cann't write data to string") return str_page_info
def to_string(self, download_rsp): str_rsq = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryProtocol(tMemory_b) download_rsp.write(tBinaryProtocol_b) str_rsq = tMemory_b.getvalue() self.log.info('data-length is {}'.format(str(len(str_rsq)))) except EOFError, e: self.log.warning("cann't write PageParseInfo to string")
def to_string(self, pageparse_info): str_parse = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryProtocol(tMemory_b) pageparse_info.write(tBinaryProtocol_b) str_parse = tMemory_b.getvalue() self.log.info('data-length is {}'.format(str(len(str_parse)))) except EOFError, e: self.log.warning("cann't write PageParseInfo to string")
def test_client_proxy_flow(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) class Pool(object): @contextlib.contextmanager def connection(self): yield client_prot client_factory = ThriftContextFactory(Pool(), TestService.Client) span = mock.MagicMock() child_span = span.make_child().__enter__() child_span.trace_id = 1 child_span.parent_id = 1 child_span.id = 1 child_span.sampled = True child_span.flags = None # We decode the token to unicode to make sure that it is converted to # bytes correctly by the AuthenticationContext. We do this because a # unicode token in Python 2 ends up causing a UnicodeDecodeError when # Thrift tries to write the header. unicode_token = self.VALID_TOKEN.decode() auth_context = AuthenticationContext( token=unicode_token, secrets=self.secrets, ) edge_context = EdgeRequestContext( authentication_context=auth_context, header=self.SERIALIZED_REQUEST_HEADER, ) edge_context.attach_context(child_span.context) client = client_factory.make_object_for_context("test", span) try: client.example_simple() except TTransportException: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) context, _ = self.observer.on_server_span_created.call_args[0] try: self.assertEqual(context.request_context.user.id, "test_user_id") self.assertEqual(context.request_context.user.roles, set()) self.assertEqual(context.request_context.user.is_logged_in, True) self.assertEqual(context.request_context.user.loid, "t2_deadbeef") self.assertEqual(context.request_context.user.cookie_created_ms, 100000) self.assertEqual(context.request_context.oauth_client.id, None) self.assertFalse( context.request_context.oauth_client.is_type("third_party")) self.assertEqual(context.request_context.session.id, "beefdead") except jwt.exceptions.InvalidAlgorithmError: raise unittest.SkipTest("cryptography is not installed")
def get_src_json(parse_info): from bdp.i_crawler.i_entity_extractor.ttypes import EntitySource from thrift.protocol.TJSONProtocol import TSimpleJSONProtocol from thrift.transport.TTransport import TMemoryBuffer entity_source = EntitySource(url=parse_info.base_info.url, site_id=parse_info.base_info.site_id, site=parse_info.base_info.site, download_time=parse_info.crawl_info.download_time) t_mem_buf = TMemoryBuffer() t_json_protocol = TSimpleJSONProtocol(t_mem_buf) entity_source.write(t_json_protocol) return json.loads(t_mem_buf.getvalue())
def to_string(self, obj): str_parse = None try: tMemory_b = TMemoryBuffer() tBinaryProtocol_b = TBinaryProtocol(tMemory_b) obj.write(tBinaryProtocol_b) str_parse = tMemory_b.getvalue() except EOFError as e: print e.message pass return str_parse
def write_to_read(write_struct, read_struct): write_buffer = TMemoryBuffer() write_protocol = TBinaryProtocol(write_buffer) write_struct.write(write_protocol) # The implementation of TMemoryBuffer is slightly different from C++ # the read/write buffer is not shared, thus we have to create another # TMemoryBuffer read_buffer = TMemoryBuffer(write_buffer.getvalue()) read_protocol = TBinaryProtocol(read_buffer) read_struct.read(read_protocol)
def encode_and_decode(self, obj): trans = TMemoryBuffer() if self.PROTO == 0: proto = TBinaryProtocol.TBinaryProtocol(trans) else: proto = TCompactProtocol.TCompactProtocol(trans) obj.write(proto) obj_new = obj.__class__() trans = TMemoryBuffer(trans.getvalue()) proto = proto.__class__(trans) obj_new.read(proto)
def http_handler(request): if request.method != 'POST': return HttpResponseNotAllowed(['GET', 'PUT']) server_info.client_ip = request.META['REMOTE_ADDR'] if server_info.client_ip[0:7] == '::ffff:': server_info.client_ip = server_info.client_ip[7:] server_info.client_port = None itrans = TMemoryBuffer(request.body) otrans = TMemoryBuffer() iproto = TBinaryProtocol(itrans) oproto = TBinaryProtocol(otrans) thrift_handler(iproto, oproto) return HttpResponse(otrans.getvalue(), content_type="application/x-thrift")
def message_received(self, frame): tmi = TMemoryBuffer(frame) tmo = TMemoryBuffer() iprot = THeaderProtocol(tmi) oprot = THeaderProtocol(tmo) try: yield from self.processor.process(iprot, oprot, self.server_context) msg = tmo.getvalue() if len(msg) > 0: self.transport.write(msg) except Exception: logging.exception("Exception while processing request") self.transport.close()
def write_communication_to_buffer(comm): ''' Serialize communication to buffer (binary string) and return buffer. Args: comm (Communication): communication to serialize Returns: Communication: Communication read from buffer ''' transport = TMemoryBuffer() protocol = factory.createProtocol(transport) comm.write(protocol) return transport.getvalue()
def test_expected_exception_not_passed_to_server_span_finish(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client = TestService.Client(client_prot) try: client.example_throws(crash=False) except TTransportException: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) self.assertEqual(self.server_observer.on_start.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_count, 1) self.assertEqual(self.server_observer.on_finish.call_args[0], (None,))
def call(self, procedure, args): if isinstance(procedure, str): try: procedure = processor._procedures[procedure] except KeyError: raise TApplicationException(TApplicationException.UNKNOWN_METHOD, "Unknown method '{0}'".format(name)) otrans = TMemoryBuffer() oproto = TBinaryProtocol(otrans) # perf.begin('send') oproto.writeMessageBegin(procedure.name, TMessageType.CALL, self._processor.seqid) self._processor.seqid = self._processor.seqid + 1 self._processor.send_struct(Namespace(args), procedure.parameters_struct, oproto) oproto.writeMessageEnd() # perf.end('send') self._http.request('POST', self._url, otrans.getvalue(), {}) resp = self._http.getresponse() data = resp.read() iproto = TBinaryProtocol(TMemoryBuffer(data)) # perf.begin('wait') (fname, mtype, rseqid) = iproto.readMessageBegin() # perf.end('wait') # perf.begin('recv') if mtype == TMessageType.EXCEPTION: x = TApplicationException() x.read(iproto) iproto.readMessageEnd() x.args = (x.message,) raise x result = self._processor.recv_struct(procedure.results_struct, iproto) iproto.readMessageEnd() # perf.end('recv') # perf.end('call') if result.result is not None: return result.result for field in procedure.results_struct.fields: if getattr(result, field.name) is not None: raise getattr(result, field.name) return None
def schedule_timeout(self, fname, seqid): timeout = self.timeouts[fname] if not timeout: return tmo = TMemoryBuffer() thp = THeaderTransport(tmo) oprot = THeaderProtocol(thp) exc = TApplicationException( TApplicationException.TIMEOUT, "Call to {} timed out".format(fname) ) oprot.writeMessageBegin(fname, TMessageType.EXCEPTION, seqid) exc.write(oprot) oprot.writeMessageEnd() thp.flush() timeout_task = self.loop.create_task( self.message_received(tmo.getvalue(), delay=timeout), ) self.update_pending_tasks(seqid, timeout_task)
def handle_request(self, event): # 1. 将zeromq的消息转换成为 thrift的 protocols trans_input = TMemoryBuffer(event.msg) trans_output = TMemoryBuffer() proto_input = self.proto_factory_input.getProtocol(trans_input) proto_output = self.proto_factory_output.getProtocol(trans_output) # 2. 交给processor来处理 try: self.processor.process(proto_input, proto_output) # 3. 将thirft的结果转换成为 zeromq 格式的数据 msg = trans_output.getvalue() # print "Return Msg: ", msg, event.id self.events.emit(msg, event.id) except Exception as e: # 如何出现了异常该如何处理呢 # 程序不能挂 print "Exception: ", e
def message_received(self, frame): # We support the deprecated FRAMED transport for old fb303 # clients that were otherwise failing miserably. client_types = { THeaderTransport.HEADERS_CLIENT_TYPE, THeaderTransport.FRAMED_DEPRECATED, } tm = TMemoryBuffer(frame) prot = THeaderProtocol(tm, client_types=client_types) try: yield from self.processor.process( prot, prot, self.server_context, ) msg = tm.getvalue() if len(msg) > 0: self.transport.write(msg) except Exception: logger.exception("Exception while processing request") self.transport.close()
def test_no_headers(self): client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client = BaseplateService.Client(client_prot) try: client.is_healthy() except: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) self.assertEqual(self.observer.on_root_span_created.call_count, 1) context, root_span = self.observer.on_root_span_created.call_args[0] self.assertEqual(root_span.trace_id, "no-trace") self.assertEqual(root_span.parent_id, "no-parent") self.assertEqual(root_span.id, "no-span") mock_root_observer = self.observer.on_root_span_created.return_value self.assertTrue(mock_root_observer.on_start.called) self.assertTrue(mock_root_observer.on_stop.called)
def call_processor(self, input, headers, client_type, protocol_type, context_data, callback): try: # The input string has already had the header removed, but # the python processor will expect it to be there. In # order to reconstitute the message with headers, we use # the THeaderProtocol object to write into a memory # buffer, then pass that buffer to the python processor. write_buf = TMemoryBuffer() trans = THeaderTransport(write_buf) trans._THeaderTransport__client_type = client_type trans._THeaderTransport__write_headers = headers trans.set_protocol_id(protocol_type) trans.write(input) trans.flush() prot_buf = TMemoryBuffer(write_buf.getvalue()) prot = THeaderProtocol(prot_buf, client_types=[client_type]) ctx = TCppConnectionContext(context_data) ret = self.processor.process(prot, prot, ctx) done_callback = partial(_ProcessorAdapter.done, prot_buf=prot_buf, client_type=client_type, callback=callback) # This future is created by and returned from the processor's # ThreadPoolExecutor, which keeps a reference to it. So it is # fine for this future to end its lifecycle here. if isinstance(ret, Future): ret.add_done_callback(lambda x, d=done_callback: d()) else: done_callback() except: # Don't let exceptions escape back into C++ traceback.print_exc()
def test_no_trace_headers(self, getrandbits): getrandbits.return_value = 1234 client_memory_trans = TMemoryBuffer() client_prot = THeaderProtocol(client_memory_trans) client = BaseplateService.Client(client_prot) try: client.is_healthy() except: pass # we don't have a test response for the client self.itrans._readBuffer = StringIO(client_memory_trans.getvalue()) self.processor.process(self.iprot, self.oprot, self.server_context) self.assertEqual(self.observer.on_server_span_created.call_count, 1) context, server_span = self.observer.on_server_span_created.call_args[0] self.assertEqual(server_span.trace_id, 1234) self.assertEqual(server_span.parent_id, None) self.assertEqual(server_span.id, 1234) self.assertTrue(self.server_observer.on_start.called) self.assertTrue(self.server_observer.on_finish.called)
def testNoInitialValue(self): buf = TMemoryBuffer() data = buf.read(5) buf.write(b"world") self.assertEquals(data, b"") self.assertEquals(buf.getvalue(), b"world")
def testReadWrite(self): buf = TMemoryBuffer(b"hello") data = buf.read(5) buf.write(b"world") self.assertEquals(data, b"hello") self.assertEquals(buf.getvalue(), b"world")
def writeToJSON(obj): trans = TMemoryBuffer() proto = TSimpleJSONProtocol.TSimpleJSONProtocol(trans) obj.write(proto) return trans.getvalue()
def thriftDumps(tweet, ProtocolClass=TBinaryProtocolAccelerated): buf = TMemoryBuffer() protocol = ProtocolClass(buf) tweet.write(protocol) return buf.getvalue()
def to_bytes(obj): """Creates the standard binary representation of a thrift object.""" b = TMemoryBuffer() p = TBinaryProtocol(b) obj.write(p) return b.getvalue()