def setUp(self):
self.monday = ThriftField("string", 1, "monday")
self.tuesday = ThriftField("string", 1, "tuesday")
self.todo_1 = ThriftField("string", 2, "run")
self.todo_2 = ThriftField("string", 2, "swim")
self.weekday = ThriftField("bool", 1, True)
self.calendar_1 = ThriftField("struct", 1,
ThriftStruct((self.monday, self.todo_1)))
self.calendar_2 = ThriftField(
"struct", 1, ThriftStruct((self.tuesday, self.todo_2)))
self.calendar_3 = ThriftField("struct", 1,
ThriftStruct((self.monday, self.todo_1)))
self.calendar_4 = ThriftField("struct", 2,
ThriftStruct((self.monday, self.todo_1)))
def test_diff_of_structs(self):
f1 = ThriftField("string", 1, "one")
f2 = ThriftField("i32", 2, 2)
f3 = ThriftField("bool", 3, True)
f4 = ThriftField("i32", 2, 4)
s1 = ThriftStruct([f1, f2, f3])
s2 = ThriftStruct([f1, f4])
t_diff = Diff.of_structs(s1, s2)
self.assertListEqual([(f1, f1), (f2, f4)], t_diff.common_fields)
self.assertListEqual([f1], t_diff.fields_with_same_value)
self.assertEqual(1, len(t_diff._fields_with_different_value))
self.assertListEqual([f3], t_diff.fields_only_in_a)
self.assertTrue(len(t_diff.fields_only_in_b) == 0)
def test_finagle(self):
queue = deque()
pcap_file = get_pcap_path('finagle-thrift')
handler = StreamHandler(queue, read_values=True, finagle_thrift=True)
sniffer = Sniffer(None, 9090, handler, offline=pcap_file)
sniffer.join()
self.assertEquals(len(queue), 22)
# is this finagle-thrift indeed?
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, '__can__finagle__trace__v3__')
self.assertEquals(msg.type, 'call')
self.assertEquals(len(msg.args), 0)
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, '__can__finagle__trace__v3__')
self.assertEquals(msg.type, 'reply')
self.assertEquals(len(msg.args), 0)
# the search() call
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'search')
self.assertEquals(msg.type, 'call')
# inspect the header & the contexts
self.assertEquals(len(msg.header), 4)
self.assertEquals(msg.header[0],
ThriftField('i64', 1, -8277104800942727271))
self.assertEquals(msg.header[1],
ThriftField('i64', 2, -8277104800942727271))
self.assertEquals(msg.header[2], ThriftField('i64', 7, 0))
contexts = msg.header[3].value
self.assertEquals(contexts[0][0].value,
'com.twitter.finagle.tracing.TraceContext')
self.assertEquals(contexts[1][0].value, 'com.twitter.finagle.Deadline')
self.assertEquals(msg.args,
ThriftStruct([ThriftField('string', 1, 'foo')]))
# the reply
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'search')
self.assertEquals(msg.type, 'reply')
self.assertEquals(
msg.args,
ThriftStruct([ThriftField('list', 0, ['one', 'two', 'three'])]))
def test_is_diff_compatible(self):
def _diff_compatibility(msg1, msg2, allowed):
ok_to_diff, reason = Diff.can_diff(msg1, msg2)
if allowed:
self.assertTrue(ok_to_diff, reason)
else:
self.assertFalse(ok_to_diff)
self.assertIsNotNone(reason)
# Method name doesn't match
rpc1 = ThriftMessage("ping", None, None, ThriftStruct([]))
rpc2 = ThriftMessage("pong", None, None, ThriftStruct([]))
_diff_compatibility(rpc1, rpc2, False)
# Method names match, so does argument signature
rpc1 = ThriftMessage("ping", None, None,
ThriftStruct([ThriftField("string", 1, "a")]))
rpc2 = ThriftMessage("ping", None, None,
ThriftStruct([ThriftField("string", 1, "b")]))
_diff_compatibility(rpc1, rpc2, True)
# Method names match, but argument signature is different
rpc1 = ThriftMessage("ping", None, None,
ThriftStruct([ThriftField("string", 1, "a")]))
rpc2 = ThriftMessage("ping", None, None,
ThriftStruct([ThriftField("i32", 1, "b")]))
_diff_compatibility(rpc1, rpc2, False)
def test_diff_of_messages(self):
f1 = ThriftField("string", 1, "one")
f2 = ThriftField("i32", 2, 2)
s1 = ThriftStruct([f1])
s2 = ThriftStruct([f2])
m1 = ThriftMessage("ping", "call", 1, s1)
m2 = ThriftMessage("ping", "call", 2, s2)
with self.assertRaisesRegexp(ValueError, 'argument signature'):
Diff.of_messages(m1, m2)
# Diff messages with all primitive args
self.assertEqual(0, len(Diff.of_messages(m1, m1)),
"Message based diff only consider args of type struct")
# Diff messages with mixed type of args
f3 = ThriftField("struct", 1, ThriftStruct([f1, f2]))
m3 = ThriftMessage("ping", "call", 2, ThriftStruct([f1, f2, f3, f3]))
diffs = Diff.of_messages(m3, m3)
self.assertEqual(2, len(diffs))
# We diffed identical messages, verify that diff reflects that
self.assertListEqual([f1, f2], diffs[0].fields_with_same_value)
self.assertEqual(0, len(diffs[0].field_with_different_value))
def read(cls, data,
protocol=None,
fallback_protocol=TBinaryProtocol,
finagle_thrift=False,
max_fields=MAX_FIELDS,
max_list_size=MAX_LIST_SIZE,
max_map_size=MAX_MAP_SIZE,
max_set_size=MAX_SET_SIZE,
read_values=False):
""" tries to deserialize a message, might fail if data is missing """
# do we have enough data?
if len(data) < cls.MIN_MESSAGE_SIZE:
raise ValueError('not enough data')
if protocol is None:
protocol = cls.detect_protocol(data, fallback_protocol)
trans = TTransport.TMemoryBuffer(data)
proto = protocol(trans)
# finagle-thrift prepends a RequestHeader
#
# See: http://git.io/vsziG
header = None
if finagle_thrift:
try:
header = ThriftStruct.read(
proto,
max_fields,
max_list_size,
max_map_size,
max_set_size,
read_values)
except:
# reset stream, maybe it's not finagle-thrift
trans = TTransport.TMemoryBuffer(data)
proto = protocol(trans)
# unpack the message
method, mtype, seqid = proto.readMessageBegin()
mtype = cls.message_type_to_str(mtype)
if len(method) == 0 or method.isspace() or method.startswith(' '):
raise ValueError('no method name')
if len(method) > cls.MAX_METHOD_LENGTH:
raise ValueError('method name too long')
# we might have made it until this point by mere chance, so filter out
# suspicious method names
valid = range(33, 127)
if any(ord(char) not in valid for char in method):
raise ValueError('invalid method name' % method)
args = ThriftStruct.read(
proto,
max_fields,
max_list_size,
max_map_size,
max_set_size,
read_values)
proto.readMessageEnd()
# Note: this is a bit fragile, the right thing would be to count bytes
# as we read them (i.e.: when calling readI32, etc).
msglen = trans._buffer.tell()
return cls(method, mtype, seqid, args, header, msglen), msglen
def test_is_isomorphic_to(self):
struct_1 = self.calendar_1.value
struct_2 = self.calendar_2.value
struct_3 = ThriftStruct([self.monday])
self.assertTrue(struct_1.is_isomorphic_to(struct_2))
self.assertFalse(struct_1.is_isomorphic_to(struct_3))
def read(cls,
data,
protocol=None,
fallback_protocol=TBinaryProtocol,
finagle_thrift=False,
max_fields=MAX_FIELDS,
max_list_size=MAX_LIST_SIZE,
max_map_size=MAX_MAP_SIZE,
max_set_size=MAX_SET_SIZE,
read_values=False):
""" tries to deserialize a message, might fail if data is missing """
# do we have enough data?
if len(data) < cls.MIN_MESSAGE_SIZE:
raise ValueError('not enough data')
if protocol is None:
protocol = cls.detect_protocol(data, fallback_protocol)
trans = TTransport.TMemoryBuffer(data)
proto = protocol(trans)
# finagle-thrift prepends a RequestHeader
#
# See: http://git.io/vsziG
header = None
if finagle_thrift:
try:
header = ThriftStruct.read(proto, max_fields, max_list_size,
max_map_size, max_set_size,
read_values)
except:
# reset stream, maybe it's not finagle-thrift
trans = TTransport.TMemoryBuffer(data)
proto = protocol(trans)
# unpack the message
method, mtype, seqid = proto.readMessageBegin()
mtype = cls.message_type_to_str(mtype)
if len(method) == 0 or method.isspace() or method.startswith(' '):
raise ValueError('no method name')
if len(method) > cls.MAX_METHOD_LENGTH:
raise ValueError('method name too long')
# we might have made it until this point by mere chance, so filter out
# suspicious method names
valid = range(33, 127)
if any(ord(char) not in valid for char in method):
raise ValueError('invalid method name' % method)
args = ThriftStruct.read(proto, max_fields, max_list_size,
max_map_size, max_set_size, read_values)
proto.readMessageEnd()
# Note: this is a bit fragile, the right thing would be to count bytes
# as we read them (i.e.: when calling readI32, etc).
msglen = trans._buffer.tell()
return cls(method, mtype, seqid, args, header, msglen), msglen
def _test_protocol(self, protoname):
queue = deque()
pcap_file = get_pcap_path('calc-service-%s' % protoname)
handler = StreamHandler(queue, read_values=True, debug=True)
sniffer = Sniffer('ignore', 9090, handler, offline=pcap_file)
sniffer.join()
self.assertEquals(len(queue), 10)
# the ping call
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'ping')
self.assertEquals(msg.type, 'call')
self.assertEquals(len(msg.args), 0)
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'ping')
self.assertEquals(msg.type, 'reply')
self.assertEquals(len(msg.args), 0)
# a succesful add
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'add')
self.assertEquals(msg.type, 'call')
self.assertEquals(len(msg.args), 2)
self.assertEquals(msg.args[0], ThriftField('i32', 1, 1))
self.assertEquals(msg.args[1], ThriftField('i32', 2, 1))
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'add')
self.assertEquals(msg.type, 'reply')
self.assertEquals(len(msg.args), 1)
self.assertEquals(msg.args[0], ThriftField('i32', 0, 2))
# a failed calculate call
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'calculate')
self.assertEquals(msg.type, 'call')
self.assertEquals(len(msg.args), 2)
self.assertEquals(msg.args[0], ThriftField('i32', 1, 1))
self.assertEquals(
msg.args[1],
ThriftField('struct', 2,
ThriftStruct(
[ThriftField('i32', 1, 1),
ThriftField('i32', 2, 0),
ThriftField('i32', 3, 4)])))
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'calculate')
self.assertEquals(msg.type, 'reply')
self.assertEquals(len(msg.args), 1)
self.assertEquals(
msg.args[0],
ThriftField('struct', 1,
ThriftStruct([ThriftField('i32', 1, 4),
ThriftField('string', 2, 'Cannot divide by 0')])))
# a successful calculate call
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'calculate')
self.assertEquals(msg.type, 'call')
self.assertEquals(len(msg.args), 2)
self.assertEquals(
msg.args[1],
ThriftField('struct', 2, ThriftStruct([ThriftField('i32', 1, 15),
ThriftField('i32', 2, 10),
ThriftField('i32', 3, 2)])))
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'calculate')
self.assertEquals(msg.type, 'reply')
self.assertEquals(len(msg.args), 1)
self.assertEquals(msg.args[0], ThriftField('i32', 0, 5))
# getStruct
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'getStruct')
self.assertEquals(msg.type, 'call')
self.assertEquals(len(msg.args), 1)
self.assertEquals(msg.args[0], ThriftField('i32', 1, 1))
_, src, dst, msg = queue.popleft()
self.assertEquals(msg.method, 'getStruct')
self.assertEquals(msg.type, 'reply')
self.assertEquals(len(msg.args), 1)
self.assertEquals(
msg.args[0],
ThriftField('struct', 0, ThriftStruct([ThriftField('i32', 1, 1),
ThriftField('string', 2, '5')])))