def call_processor(self, input, client_type, protocol_type,
client_principal):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf, client_types=[client_type])
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf)
ctx = TCppConnectionContext(client_principal)
self.processor.process(prot, prot, ctx)
# And on the way out, we need to strip off the header,
# because the C++ code will expect to add it.
read_buf = TMemoryBuffer(prot_buf.getvalue())
trans = THeaderTransport(read_buf, client_types=[client_type])
trans.readFrame(0)
return trans.cstringio_buf.read()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
def round_robin(self, compress=None):
original = b"A" * MAX_FRAME_SIZE
mb = TMemoryBuffer()
trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER)
trans.set_max_frame_size(MAX_FRAME_SIZE + MIN_HEADER_SIZE)
if compress:
trans.add_transform(compress)
trans.write(original)
trans.flush()
frame = mb.getvalue()
# Cleanup the memory buffer
mb.close()
del mb
if compress is None:
# Partial Decode the frame and see if its correct size wise
sz = struct.unpack("!I", frame[:4])[0]
self.assertEqual(sz, BIG_FRAME_MAGIC)
sz = struct.unpack("!Q", frame[4:12])[0]
self.assertEqual(len(frame), sz + 12)
# Read it back
mb = TMemoryBuffer(frame)
trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER)
trans.set_max_frame_size(len(frame))
trans.readFrame(0)
result = trans.read(MAX_FRAME_SIZE)
mb.close()
del mb
self.assertEqual(result, original,
"round-robin different from original")
def round_robin(self, compress=None):
original = b'A' * MAX_FRAME_SIZE
mb = TMemoryBuffer()
trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER)
trans.set_max_frame_size(MAX_FRAME_SIZE + MIN_HEADER_SIZE)
if compress:
trans.add_transform(compress)
trans.write(original)
trans.flush()
frame = mb.getvalue()
# Cleanup the memory buffer
mb.close()
del mb
if compress is None:
# Partial Decode the frame and see if its correct size wise
sz = struct.unpack('!I', frame[:4])[0]
self.assertEqual(sz, BIG_FRAME_MAGIC)
sz = struct.unpack('!Q', frame[4:12])[0]
self.assertEqual(len(frame), sz + 12)
# Read it back
mb = TMemoryBuffer(frame)
trans = THeaderTransport(mb, client_type=CLIENT_TYPE.HEADER)
trans.set_max_frame_size(len(frame))
trans.readFrame(0)
result = trans.read(MAX_FRAME_SIZE)
mb.close()
del mb
self.assertEqual(result, original, 'round-robin different from original')
def call_processor(self, input, headers, client_type, protocol_type,
context_data, callback):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
should_sample = self._shouldSample()
timestamps = CallTimestamps()
timestamps.processBegin = 0
timestamps.processEnd = 0
if self.observer and should_sample:
timestamps.processBegin = int(time.time() * 10**6)
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
ctx = TCppConnectionContext(context_data)
ret = self.processor.process(prot, prot, ctx)
done_callback = partial(_ProcessorAdapter.done,
prot_buf=prot_buf,
client_type=client_type,
callback=callback)
if self.observer:
if should_sample:
timestamps.processEnd = int(time.time() * 10**6)
# This only bumps counters if `processBegin != 0` and
# `processEnd != 0` and these will only be non-zero if
# we are sampling this request.
self.observer.callCompleted(timestamps)
# This future is created by and returned from the processor's
# ThreadPoolExecutor, which keeps a reference to it. So it is
# fine for this future to end its lifecycle here.
if isinstance(ret, Future):
ret.add_done_callback(lambda x, d=done_callback: d())
else:
done_callback()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
def schedule_timeout(self, fname, seqid):
timeout = self.timeouts[fname]
if not timeout:
return
tmo = TMemoryBuffer()
thp = THeaderTransport(tmo)
oprot = THeaderProtocol(thp)
exc = TApplicationException(TApplicationException.TIMEOUT,
"Call to {} timed out".format(fname))
oprot.writeMessageBegin(fname, TMessageType.EXCEPTION, seqid)
exc.write(oprot)
oprot.writeMessageEnd()
thp.flush()
timeout_task = self.loop.create_task(
self.message_received(tmo.getvalue(), delay=timeout), )
self.update_pending_tasks(seqid, timeout_task)
class TestEof(unittest.TestCase):
def setUp(self):
self.trans = TTransport.TMemoryBuffer()
self.trans = THeaderTransport(self.trans)
self.prot = TBinaryProtocol.TBinaryProtocol(self.trans)
self.x = Xtruct()
self.x.string_thing = "Zero"
self.x.byte_thing = 0
def testOversizeFrameRecv(self):
"""Test that an oversize frame on recv gets a TTransportException"""
self.trans.set_max_frame_size(200)
self.x.write(self.prot)
self.trans.flush()
inTrans = TTransport.TMemoryBuffer(
self.trans.getTransport().getvalue())
inTrans = THeaderTransport(inTrans)
inProt = TBinaryProtocol.TBinaryProtocol(inTrans)
inTrans.set_max_frame_size(2)
try:
self.x.read(inProt)
except TTransport.TTransportException:
return
self.fail("Should have gotten TTransportException")
def testOversizeFrameSend(self):
"""Test that an oversize frame on send gets a TTransportException"""
self.trans.set_max_frame_size(2)
self.x.write(self.prot)
try:
self.trans.flush()
except TTransport.TTransportException:
return
self.fail("Should have gotten TTransportException")
def call_processor(self, input, headers, client_type, protocol_type,
context_data):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
ctx = TCppConnectionContext(context_data)
self.processor.process(prot, prot, ctx)
# Check for empty result. If so, return an empty string
# here. This is probably a oneway request, but we can't
# reliably tell. The C++ code does basically the same
# thing.
response = prot_buf.getvalue()
if len(response) == 0:
return response
# And on the way out, we need to strip off the header,
# because the C++ code will expect to add it.
read_buf = TMemoryBuffer(response)
trans = THeaderTransport(read_buf, client_types=[client_type])
trans.readFrame(len(response))
return trans.cstringio_buf.read()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
def call_processor(self, input, headers, client_type, protocol_type,
context_data):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
ctx = TCppConnectionContext(context_data)
self.processor.process(prot, prot, ctx)
# Check for empty result. If so, return an empty string
# here. This is probably a oneway request, but we can't
# reliably tell. The C++ code does basically the same
# thing.
response = prot_buf.getvalue()
if len(response) == 0:
return response
# And on the way out, we need to strip off the header,
# because the C++ code will expect to add it.
read_buf = TMemoryBuffer(response)
trans = THeaderTransport(read_buf, client_types=[client_type])
trans.readFrame(len(response))
return trans.cstringio_buf.read()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
class TestEof(unittest.TestCase):
def setUp(self):
self.trans = TTransport.TMemoryBuffer()
self.trans = THeaderTransport(self.trans)
self.prot = TBinaryProtocol.TBinaryProtocol(self.trans)
self.x = Xtruct()
self.x.string_thing = "Zero"
self.x.byte_thing = 0
def testOversizeFrameRecv(self):
"""Test that an oversize frame on recv gets a TTransportException"""
self.trans.set_max_frame_size(200)
self.x.write(self.prot)
self.trans.flush()
inTrans = TTransport.TMemoryBuffer(self.trans.getTransport().getvalue())
inTrans = THeaderTransport(inTrans)
inProt = TBinaryProtocol.TBinaryProtocol(inTrans)
inTrans.set_max_frame_size(2)
try:
self.x.read(inProt)
except TTransport.TTransportException:
return
self.fail("Should have gotten TTransportException")
def testOversizeFrameSend(self):
"""Test that an oversize frame on send gets a TTransportException"""
self.trans.set_max_frame_size(2)
self.x.write(self.prot)
try:
self.trans.flush()
except TTransport.TTransportException:
return
self.fail("Should have gotten TTransportException")
def schedule_timeout(self, fname, seqid):
timeout = self.timeouts[fname]
if not timeout:
return
tmo = TMemoryBuffer()
thp = THeaderTransport(tmo)
oprot = THeaderProtocol(thp)
exc = TApplicationException(
TApplicationException.TIMEOUT, "Call to {} timed out".format(fname)
)
oprot.writeMessageBegin(fname, TMessageType.EXCEPTION, seqid)
exc.write(oprot)
oprot.writeMessageEnd()
thp.flush()
timeout_task = self.loop.create_task(
self.message_received(tmo.getvalue(), delay=timeout),
)
self.update_pending_tasks(seqid, timeout_task)
def call_processor(self, input, headers, client_type, protocol_type,
context_data, callback):
try:
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
ctx = TCppConnectionContext(context_data)
ret = self.processor.process(prot, prot, ctx)
done_callback = partial(_ProcessorAdapter.done,
prot_buf=prot_buf,
client_type=client_type,
callback=callback)
# This future is created by and returned from the processor's
# ThreadPoolExecutor, which keeps a reference to it. So it is
# fine for this future to end its lifecycle here.
if isinstance(ret, Future):
ret.add_done_callback(lambda x, d=done_callback: d())
else:
done_callback()
except:
# Don't let exceptions escape back into C++
traceback.print_exc()
class FuzzyTransportTest(object):
"""Test class that sets up a THeaderTransport and a TFuzzyHeaderTransport.
Used for writing and comparing messages using both transports.
"""
fuzz_fields = []
# Sample TestService method arguments
sampleListStruct = ttypes.ListStruct(
a=[True, False],
b=[1, 2, 3],
c=[1.2, 3.4],
d=["ab", "cd"],
e=[list(sm.xrange(n)) for n in sm.xrange(20)],
f=[{
1: 2
}, {
3: 4,
5: 6
}],
g=[{"a", "b"}, {"c"}, set()])
def setUp(self):
"""Create two buffers, transports, and protocols.
self._h_trans uses THeaderTransport
self._f_trans uses TFuzzyHeaderTransport
"""
cls = self.__class__
# THeaderTransport attributes
self._h_buf = TMemoryBuffer()
self._h_trans = THeaderTransport(self._h_buf)
self._h_prot = THeaderProtocol(self._h_trans)
# TFuzzyHeaderTransport attributes
self._f_buf = TMemoryBuffer()
self._f_trans = TFuzzyHeaderTransport(self._f_buf,
fuzz_fields=cls.fuzz_fields,
fuzz_all_if_empty=False,
verbose=False)
self._f_prot = THeaderProtocol(self._f_trans)
def writeObject(self, obj=sampleListStruct):
"""Write an object to the test and reference protocols.
Return the contents of both buffers.
"""
obj.write(self._h_prot)
obj.write(self._f_prot)
self._h_trans.flush()
self._f_trans.flush()
h_buf = self._h_buf.getvalue()
f_buf = self._f_buf.getvalue()
return h_buf, f_buf
def differentIndices(self, header, fuzzy):
"""Return a list of byte positions at which two messages' bytes differ.
Header should be the contents of self._h_buf
Fuzzy should be the contents of self._f_buf
"""
indices = []
for i, (h, f) in enumerate(itertools.izip(header, fuzzy)):
if h != f:
indices.append(i)
return indices
def assertEqualsExceptIndices(self, header, fuzzy, indices):
"""Assert that the buffers `header` and `fuzzy` are equal,
except possibly at the byte positions included in `indices`.
This ensures that the message produced by TFuzzyHeaderProtocol (fuzzy)
is equal to the message produced by THeaderProtocol (header), except
at the byte positions that are expected to be fuzzed."""
self.assertEquals(len(header), len(fuzzy))
for diff in self.differentIndices(header, fuzzy):
self.assertIn(diff, indices)
def call_processor(
self, input, headers, client_type, protocol_type, context_data, callback
):
try:
# TCppServer threads are not created by Python so they are
# missing settrace() hooks. We need to manually set the
# hook here for things to work (e.g. coverage and pdb).
if sys.gettrace() is None and threading._trace_hook is not None:
sys.settrace(threading._trace_hook)
# The input string has already had the header removed, but
# the python processor will expect it to be there. In
# order to reconstitute the message with headers, we use
# the THeaderProtocol object to write into a memory
# buffer, then pass that buffer to the python processor.
should_sample = self._shouldSample()
timestamps = CallTimestamps()
if self.observer and should_sample:
timestamps.setProcessBeginNow()
write_buf = TMemoryBuffer()
trans = THeaderTransport(write_buf)
trans.set_max_frame_size(MAX_BIG_FRAME_SIZE)
trans._THeaderTransport__client_type = client_type
trans._THeaderTransport__write_headers = headers
trans.set_protocol_id(protocol_type)
trans.write(input)
trans.flush()
prot_buf = TMemoryBuffer(write_buf.getvalue())
prot = THeaderProtocol(prot_buf, client_types=[client_type])
prot.trans.set_max_frame_size(MAX_BIG_FRAME_SIZE)
ctx = TCppConnectionContext(context_data)
ret = self.processor.process(prot, prot, ctx)
done_callback = partial(
_ProcessorAdapter.done,
prot_buf=prot_buf,
client_type=client_type,
callback=callback,
)
if self.observer:
if should_sample:
timestamps.setProcessEndNow()
# This only bumps counters if `processBegin != 0` and
# `processEnd != 0` and these will only be non-zero if
# we are sampling this request.
self.observer.callCompleted(timestamps)
# This future is created by and returned from the processor's
# ThreadPoolExecutor, which keeps a reference to it. So it is
# fine for this future to end its lifecycle here.
if isinstance(ret, Future):
ret.add_done_callback(lambda x, d=done_callback: d())
else:
done_callback()
except: # noqa
# Don't let exceptions escape back into C++
traceback.print_exc()
class FuzzyTransportTest(object):
"""Test class that sets up a THeaderTransport and a TFuzzyHeaderTransport.
Used for writing and comparing messages using both transports.
"""
fuzz_fields = []
# Sample TestService method arguments
sampleListStruct = ttypes.ListStruct(
a=[True, False],
b=[1, 2, 3],
c=[1.2, 3.4],
d=["ab", "cd"],
e=[list(sm.xrange(n)) for n in sm.xrange(20)],
f=[{1: 2}, {3: 4, 5: 6}],
g=[{"a", "b"}, {"c"}, set()]
)
def setUp(self):
"""Create two buffers, transports, and protocols.
self._h_trans uses THeaderTransport
self._f_trans uses TFuzzyHeaderTransport
"""
cls = self.__class__
# THeaderTransport attributes
self._h_buf = TMemoryBuffer()
self._h_trans = THeaderTransport(self._h_buf)
self._h_prot = THeaderProtocol(self._h_trans)
# TFuzzyHeaderTransport attributes
self._f_buf = TMemoryBuffer()
self._f_trans = TFuzzyHeaderTransport(
self._f_buf, fuzz_fields=cls.fuzz_fields,
fuzz_all_if_empty=False, verbose=False)
self._f_prot = THeaderProtocol(self._f_trans)
def writeObject(self, obj=sampleListStruct):
"""Write an object to the test and reference protocols.
Return the contents of both buffers.
"""
obj.write(self._h_prot)
obj.write(self._f_prot)
self._h_trans.flush()
self._f_trans.flush()
h_buf = self._h_buf.getvalue()
f_buf = self._f_buf.getvalue()
return h_buf, f_buf
def differentIndices(self, header, fuzzy):
"""Return a list of byte positions at which two messages' bytes differ.
Header should be the contents of self._h_buf
Fuzzy should be the contents of self._f_buf
"""
indices = []
for i, (h, f) in enumerate(itertools.izip(header, fuzzy)):
if h != f:
indices.append(i)
return indices
def assertEqualsExceptIndices(self, header, fuzzy, indices):
"""Assert that the buffers `header` and `fuzzy` are equal,
except possibly at the byte positions included in `indices`.
This ensures that the message produced by TFuzzyHeaderProtocol (fuzzy)
is equal to the message produced by THeaderProtocol (header), except
at the byte positions that are expected to be fuzzed."""
self.assertEquals(len(header), len(fuzzy))
for diff in self.differentIndices(header, fuzzy):
self.assertIn(diff, indices)
