def memory_usage_fastproto(): hp = hpy() trans = TDevNullTransport() global ooe for pid in (0, 2): before = hp.heap() for i in range(iters): buf = fastproto.encode(ooe, [OneOfEach, OneOfEach.thrift_spec, False], utf8strings=0, protoid=pid) trans.write(buf) gc.collect() after = hp.heap() leftover = after - before print("Memory leftover after running fastproto.encode with " "protocol id {0} for {1} times".format(pid, iters)) print(leftover) for pid in (0, 2): before = hp.heap() for i in range(iters): trans = TTransport.TMemoryBuffer(binary_buf if pid == 0 else compact_buf) ooe_local = OneOfEach() fastproto.decode(ooe_local, trans, [OneOfEach, OneOfEach.thrift_spec, False], utf8strings=0, protoid=pid) gc.collect() after = hp.heap() leftover = after - before print("Memory leftover after running fastproto.decode with " "protocol id {0} for {1} times".format(pid, iters)) print(leftover)
def buildOneOfEachB(self): return OneOfEach( aBool=True, aByte=1, anInteger16=234, anInteger32=12345, anInteger64=12345678910, aString=b'\x00hello', aBinary=b'\x00\x01\x00', aDouble=1234567.901, aFloat=12345.0, aList=[12, 34, 567, 89], aSet=set([b"hello", b"world", b"good", b"bye"]), aMap={b"hello": 1, b"world": 20}, aStruct=AStruct(aString=b"str", anInteger=109))
def test_decode_failure_lists_fieldname(self): obj = OneOfEach(aString="Привет".encode("cp866")) # Can't use self.assertRaises since AbstractTest cannot inherit # from unittest.TestCase raised_exception = None try: self.decode_helper(obj, split=1.0, utf8strings=1) except UnicodeDecodeError as ex: raised_exception = ex self.assertIsNotNone(raised_exception) self.assertIn("when decoding field 'aString'", str(raised_exception)) self.assertIn( "('utf-8', b'\\x8f\\xe0\\xa8\\xa2\\xa5\\xe2', 0, 1, 'invalid start byte')", str(raised_exception), ) self.assertTrue(isinstance(raised_exception, ThriftUnicodeDecodeError))
def buildOneOfEach(self): return OneOfEach( aBool=True, aByte=1, anInteger16=234, anInteger32=12345, anInteger64=12345678910, aString="\x00hello", aBinary=b"\x00\x01\x00", aDouble=1234567.901, aFloat=12345.0, aList=[12, 34, 567, 89], aSet={"hello", "world", "good", "bye"}, aMap={"hello": 1, "world": 20}, aStruct=AStruct(aString="str", anInteger=109), )
def fastproto_decode(q, protoid): hp = hpy() p = psutil.Process(os.getpid()) before = hp.heap() for i in range(iters): trans = TTransport.TMemoryBuffer(binary_buf if protoid == 0 else compact_buf) ooe_local = OneOfEach() fastproto.decode(ooe_local, trans, [OneOfEach, OneOfEach.thrift_spec, False], utf8strings=0, protoid=protoid) if (i + 1) % 100000 == 0: q.put((i + 1, p.memory_info())) gc.collect() after = hp.heap() leftover = after - before q.put("Memory leftover in Python after {} times: {}".format( iters, leftover))
from thrift.transport import TTransport import timeit import gc from multiprocessing import Process, Queue import os import psutil try: from guppy import hpy except ImportError: hpy = None from FastProto.ttypes import AStruct, OneOfEach ooe = OneOfEach() ooe.aBool = True ooe.aByte = 1 ooe.anInteger16 = 234 ooe.anInteger32 = 2345678 ooe.anInteger64 = 23456789012345 ooe.aString = "This is my rifle" * 100 ooe.aDouble = 2.3456789012 ooe.aFloat = 12345.678 ooe.aList = [12, 34, 56, 78, 90, 100, 123, 456, 789] ooe.aSet = set(["This", "is", "my", "rifle"]) ooe.aMap = {"What": 4, "a": 1, "wonderful": 9, "day": 3, "!": 1} ooe.aStruct = AStruct(aString="isn't it?", anInteger=999) trans = TTransport.TMemoryBuffer() proto = TBinaryProtocol.TBinaryProtocol(trans)
import os import timeit from multiprocessing import Process, Queue import psutil from thrift.protocol import fastproto, TBinaryProtocol, TCompactProtocol from thrift.transport import TTransport try: from guppy import hpy except ImportError: hpy = None from FastProto.ttypes import AStruct, OneOfEach ooe = OneOfEach() ooe.aBool = True ooe.aByte = 1 ooe.anInteger16 = 234 ooe.anInteger32 = 2345678 ooe.anInteger64 = 23456789012345 ooe.aString = "This is my rifle" * 100 ooe.aDouble = 2.3456789012 ooe.aFloat = 12345.678 ooe.aList = [12, 34, 56, 78, 90, 100, 123, 456, 789] ooe.aSet = set(["This", "is", "my", "rifle"]) ooe.aMap = {"What": 4, "a": 1, "wonderful": 9, "day": 3, "!": 1} ooe.aStruct = AStruct(aString="isn't it?", anInteger=999) trans = TTransport.TMemoryBuffer() proto = TBinaryProtocol.TBinaryProtocol(trans)
def test_empty_container(self): self.encode_helper(OneOfEach(aSet=set(), aList=[], aMap={})) self.decode_helper(OneOfEach(aSet=set(), aList=[], aMap={}))
def test_decode(self): self.decode_helper(self.buildOneOfEachB()) # Test when ensureMapBegin needs to verify the buffer has # at least a varint and 1 more byte. self.decode_helper(OneOfEach(aMap={b"h": 1}), split=0.1)