def memory_usage_fastproto():
hp = hpy()
trans = TDevNullTransport()
global ooe
for pid in (0, 2):
before = hp.heap()
for i in range(iters):
buf = fastproto.encode(ooe,
[OneOfEach, OneOfEach.thrift_spec, False],
utf8strings=0,
protoid=pid)
trans.write(buf)
gc.collect()
after = hp.heap()
leftover = after - before
print("Memory leftover after running fastproto.encode with "
"protocol id {0} for {1} times".format(pid, iters))
print(leftover)
for pid in (0, 2):
before = hp.heap()
for i in range(iters):
trans = TTransport.TMemoryBuffer(binary_buf if pid ==
0 else compact_buf)
ooe_local = OneOfEach()
fastproto.decode(ooe_local,
trans, [OneOfEach, OneOfEach.thrift_spec, False],
utf8strings=0,
protoid=pid)
gc.collect()
after = hp.heap()
leftover = after - before
print("Memory leftover after running fastproto.decode with "
"protocol id {0} for {1} times".format(pid, iters))
print(leftover)
def buildOneOfEachB(self):
return OneOfEach(
aBool=True,
aByte=1,
anInteger16=234,
anInteger32=12345,
anInteger64=12345678910,
aString=b'\x00hello',
aBinary=b'\x00\x01\x00',
aDouble=1234567.901,
aFloat=12345.0,
aList=[12, 34, 567, 89],
aSet=set([b"hello", b"world", b"good", b"bye"]),
aMap={b"hello": 1, b"world": 20},
aStruct=AStruct(aString=b"str", anInteger=109))
def test_decode_failure_lists_fieldname(self):
obj = OneOfEach(aString="Привет".encode("cp866"))
# Can't use self.assertRaises since AbstractTest cannot inherit
# from unittest.TestCase
raised_exception = None
try:
self.decode_helper(obj, split=1.0, utf8strings=1)
except UnicodeDecodeError as ex:
raised_exception = ex
self.assertIsNotNone(raised_exception)
self.assertIn("when decoding field 'aString'", str(raised_exception))
self.assertIn(
"('utf-8', b'\\x8f\\xe0\\xa8\\xa2\\xa5\\xe2', 0, 1, 'invalid start byte')",
str(raised_exception),
)
self.assertTrue(isinstance(raised_exception, ThriftUnicodeDecodeError))
def buildOneOfEach(self):
return OneOfEach(
aBool=True,
aByte=1,
anInteger16=234,
anInteger32=12345,
anInteger64=12345678910,
aString="\x00hello",
aBinary=b"\x00\x01\x00",
aDouble=1234567.901,
aFloat=12345.0,
aList=[12, 34, 567, 89],
aSet={"hello", "world", "good", "bye"},
aMap={"hello": 1, "world": 20},
aStruct=AStruct(aString="str", anInteger=109),
)
def fastproto_decode(q, protoid):
hp = hpy()
p = psutil.Process(os.getpid())
before = hp.heap()
for i in range(iters):
trans = TTransport.TMemoryBuffer(binary_buf if protoid ==
0 else compact_buf)
ooe_local = OneOfEach()
fastproto.decode(ooe_local,
trans, [OneOfEach, OneOfEach.thrift_spec, False],
utf8strings=0,
protoid=protoid)
if (i + 1) % 100000 == 0:
q.put((i + 1, p.memory_info()))
gc.collect()
after = hp.heap()
leftover = after - before
q.put("Memory leftover in Python after {} times: {}".format(
iters, leftover))
import os
import timeit
from multiprocessing import Process, Queue
import psutil
from thrift.protocol import fastproto, TBinaryProtocol, TCompactProtocol
from thrift.transport import TTransport
try:
from guppy import hpy
except ImportError:
hpy = None
from FastProto.ttypes import AStruct, OneOfEach
ooe = OneOfEach()
ooe.aBool = True
ooe.aByte = 1
ooe.anInteger16 = 234
ooe.anInteger32 = 2345678
ooe.anInteger64 = 23456789012345
ooe.aString = "This is my rifle" * 100
ooe.aDouble = 2.3456789012
ooe.aFloat = 12345.678
ooe.aList = [12, 34, 56, 78, 90, 100, 123, 456, 789]
ooe.aSet = set(["This", "is", "my", "rifle"])
ooe.aMap = {"What": 4, "a": 1, "wonderful": 9, "day": 3, "!": 1}
ooe.aStruct = AStruct(aString="isn't it?", anInteger=999)
trans = TTransport.TMemoryBuffer()
proto = TBinaryProtocol.TBinaryProtocol(trans)
def test_empty_container(self):
self.encode_helper(OneOfEach(aSet=set(), aList=[], aMap={}))
self.decode_helper(OneOfEach(aSet=set(), aList=[], aMap={}))
def test_decode(self):
self.decode_helper(self.buildOneOfEachB())
# Test when ensureMapBegin needs to verify the buffer has
# at least a varint and 1 more byte.
self.decode_helper(OneOfEach(aMap={b"h": 1}), split=0.1)