def test_replace_method():
roots, test_proto, content = generate_proto_binary('test_wrapped.proto',
b'test: "abcdefg"',
'Wrapped')
print('generated proto message: {}'.format(content))
roots, test_proto, content = generate_proto_binary(
'test_wrapped.proto', b'''
wrapped_type: "Wrapped"
wrapped_payload: "%s"
''' % repr(content).encode('utf-8')[2:-1], 'Wrapper')
parser = BinParser(roots)
result = parser.parse(test_proto, 'Wrapper', content)
print('result before: {}'.format(result))
assert result.wrapped_type == 'Wrapped'
original = parser.def_parser.find_definition(b'Wrapper')
print('original: {}'.format(original))
class Replacement:
def __init__(self, original):
self.original = original
def replacement(self, *args):
print('replacement')
return {'replaced': self.original(*args)}
rp = Replacement(original)
parser.def_parser.update_definition(b'Wrapper', rp.replacement)
result = parser.parse(test_proto, 'Wrapper', content)
print('result after: {}'.format(result))
print('new def: {}'.format(parser.def_parser.find_definition('Wrapper')))
assert result['replaced'].wrapped_type == 'Wrapped'
def test_repeated_serialize():
roots, test_proto, content = generate_proto_binary(
'test_repeated.proto',
b'''simple_repeats: {
some_ints: [1, -1, 2, -2, 3, -3, 4, -4, 5, -5]
}
multiple_repeats: [{
some_fixed: [1, -1, 2, -2, 3, -3]
some_strings: ["foo", "bar", "\x01weird\x10"]
},
{
some_fixed: [4, -4, 5, -5, 6, -6]
}]
multiple_oneof: {
some_fixed: [123456789, 987654321]
some_strings: [""]
}
''',
)
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
result1 = parser.parse(test_proto, 'Test', content)
print('result1: {}'.format(result1))
serializer = Serializer(parser)
content = serializer.serialize(result1, 'Test')
print('generated by serializer: {}'.format(content))
result2 = parser.parse(test_proto, 'Test', content)
assert result1 == result2
def test_dotted():
roots, test_proto, content = generate_proto_binary(
'test_dotted_type.proto',
b'''dotted_repeats: [{
int_field: 100,
},
{
int_field: 1000,
}]
inner_dotted: {
string_field: "abcde"
}
imported_dotted: {
a_string: "xyz"
an_int: 54321
}
one_of_inner_dotted: {
string_field: "Lorem ipsum"
}
''',
)
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
parser.def_parser.parse(test_proto)
for f, p in parser.def_parser.definitions():
print('{} => {}'.format(f, p))
print('all definitions parsed')
result = parser.parse(test_proto, 'Test', content)
print('result: {}'.format(result))
assert result.inner_dotted.string_field == "abcde"
def test_simple_serialize():
roots, test_proto, content = generate_proto_binary(
'test.proto',
b'test: 123\ntest_whatever: "123456"',
)
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
result1 = parser.parse(test_proto, 'Test', content)
serializer = Serializer(parser)
result2 = parser.parse(
test_proto,
'Test',
serializer.serialize(result1, 'Test'),
)
assert result1 == result2
def test_replace_ctor():
roots, test_proto, content = generate_proto_binary('test_wrapped.proto',
b'test: "abcdefg"',
'Wrapped')
print('generated proto message: {}'.format(content))
roots, test_proto, content = generate_proto_binary(
'test_wrapped.proto', b'''
wrapped_type: "Wrapped"
wrapped_payload: "%s"
''' % repr(content).encode('utf-8')[2:-1], 'Wrapper')
parser = BinParser(roots)
class ModifiedWrapper:
original = None
def __init__(self, *args):
print('modified init')
raw = ModifiedWrapper.original(*args)
self.payload = parser.parse(
test_proto,
raw.wrapped_type,
raw.wrapped_payload,
)
self.wrapped_type = raw.wrapped_type
print('modified init finished')
result = parser.parse(test_proto, 'Wrapper', content)
print('result before: {}'.format(result))
assert result.wrapped_type == 'Wrapped'
ModifiedWrapper.original = parser.def_parser.find_definition(b'Wrapper')
print('ModifiedWrapper.original: {}'.format(ModifiedWrapper.original))
parser.def_parser.update_definition(b'Wrapper', ModifiedWrapper)
result = parser.parse(test_proto, 'Wrapper', content)
for k, v in parser.def_parser.definitions():
print('def: {} => {}'.format(k, v))
print('result after: {}'.format(result))
assert result.wrapped_type == 'Wrapped'
assert result.payload.test == 'abcdefg'
def test_basic_types():
roots, test_proto, content = generate_proto_binary(
'simple_types.proto', b'''
tint32: 123456
tint64: 123456789123456789
tsint32: -123456
tsint64: -123456789123456789
tuint32: 123456
tuint64: 123456789123456789
tbool: true
tfixed64: 123456789123456789
tsfixed64: -123456789123456789
tdouble: 123456.123456
tstring: 'abcdefg'
tbytes: '\x01\x02\x03'
tfixed32: 123456
tsfixed32: -123456
''', 'SimpleTypes')
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
result = parser.parse(test_proto, 'SimpleTypes', content)
print('result: {}'.format(result))
serializer = Serializer(parser)
assert serializer.serialize(result.tint32, 'int32') \
== b'\xc0\xc4\x07'
assert serializer.serialize(result.tint64, 'int64') \
== b'\x95\xbe\xc1\xe6\xba\xe9\xa6\xdb\x01'
assert serializer.serialize(result.tsint32, 'sint32') \
== b'\xff\x88\x0f'
assert serializer.serialize(result.tsint64, 'sint64') \
== b'\xa9\xfc\x82\xcd\xf5\xd2\xcd\xb6\x03'
assert serializer.serialize(result.tuint32, 'uint32') \
== b'\xc0\xc4\x07'
assert serializer.serialize(result.tuint64, 'uint64') \
== b'\x95\xbe\xc1\xe6\xba\xe9\xa6\xdb\x01'
assert serializer.serialize(result.tbool, 'bool') \
== b'\x01'
assert serializer.serialize(result.tfixed64, 'fixed64') \
== b'\x15_\xd0\xacK\x9b\xb6\x01'
assert serializer.serialize(result.tsfixed64, 'sfixed64') \
== b'\xeb\xa0/S\xb4dI\xfe'
assert serializer.serialize(result.tfixed32, 'fixed32') \
== b'@\xe2\x01\x00'
assert serializer.serialize(result.tsfixed32, 'sfixed32') \
== b'\xc0\x1d\xfe\xff'
assert serializer.serialize(result.tstring, 'string') \
== b'\x07abcdefg'
assert serializer.serialize(result.tbytes, 'bytes') \
== b'\x03\x01\x02\x03'
print('result.tdouble: {}'.format(result.tdouble))
assert serializer.serialize(result.tdouble, 'double') \
== b'\xa8\xff\xac\xf9\x01$\xfe@'
def test_invalid_source_path():
roots = [pkg_resources.resource_filename(
__name__,
'./resources',
)]
try:
BinParser(roots).parse(None, 'Test', 'does not matter')
except FileNotFoundError:
pass
roots, test_proto, content = generate_proto_binary(
'test.proto',
b'test: 123\ntest_whatever: "123456"',
)
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
parser.parse(test_proto, 'Test', content)
try:
parser.parse(None, 'Test', content)
except FileNotFoundError:
pass
def test_enum_serialize():
roots, test_proto, content = generate_proto_binary(
'test_enum.proto',
b'''test_1: 33
test_2: 2
test_3: 0
test_4: 5
''',
)
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
result1 = parser.parse(test_proto, 'Test', content)
print('result1: {}'.format(result1))
serializer = Serializer(parser)
content = serializer.serialize(result1, 'Test')
print('generated by serializer: {}'.format(content))
result2 = parser.parse(test_proto, 'Test', content)
assert result1 == result2
def test_simple_enum():
roots, test_proto, content = generate_proto_binary(
'test_enum.proto',
b'''test_1: 33
test_2: 2
test_3: 0
test_4: 5
''',
)
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
parser.def_parser.enum_ctor = simple_enum
result = parser.parse(test_proto, 'Test', content)
print('result: {}'.format(result))
assert result.test_1 == 'TestEnum.TEST_MEMBER_3'
def test_integer():
roots, test_proto, content = generate_proto_binary(
'test.proto',
b'test: 123\ntest_whatever: "123456"',
)
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
result = parser.parse(test_proto, 'Test', content)
print('result: {}'.format(result))
assert result.test == 123
serializer = Serializer(parser)
assert serializer.serialize(result.test, 'int32') \
== chr(123).encode('utf-8')
assert serializer.serialize(123456, 'int32') == b'\xc0\xc4\x07'
assert serializer.serialize(123456789, 'int32') == b'\x95\x9a\xef:'
class TicTacToeClient:
def __init__(self):
root = pkg_resources.resource_filename(
__name__,
'etc',
)
self.buffsize = 100
self.parser = BinParser([root])
self.proto_def = path.join(root, 'tictactoe.proto')
self.parser.def_parser.parse(self.proto_def)
self.serializer = Serializer(self.parser)
self.dparser = self.parser.def_parser
self.message_ctor = self.dparser.find_definition('TicTacToe')
self.board_ctor = self.dparser.find_definition('TicTacToeBoard')
self.cell_ctor = self.dparser.find_definition('TicTacToeCell')
self.create_board()
def create_board(self):
cells = [self.cell_ctor(x=None, o=None, unset=True) for _ in range(9)]
self.board = self.board_ctor(cells=cells, session=0)
def board_str(self):
tpl = '\n---------\n'.join(['{} | {} | {}'] * 3)
masks = {
self.cell_ctor(x=True, o=None, unset=None): 'x',
self.cell_ctor(x=None, o=True, unset=None): 'o',
self.cell_ctor(x=None, o=None, unset=True): ' ',
}
vals = [masks[x] for x in self.board.cells]
return tpl.format(*vals)
def what_error(self, error):
for e in error:
if e:
return e
async def exchange(self):
message = self.message_ctor(board=self.board, error=None)
payload = self.serializer.serialize(message, 'TicTacToe')
self.writer.write(payload)
await self.writer.drain()
data = b''
while True:
chunk = await self.reader.read(self.buffsize)
data += chunk
try:
message = self.parser.parse(self.proto_def, 'TicTacToe', data)
except Exception as e:
print('what happened: {}'.format(e))
else:
break
if message.error:
raise Exception('protocol error: {}'.format(
self.what_error(message.error), ))
self.board = message.board
async def connect(self, loop):
reader, writer = await asyncio.open_connection(
'127.0.0.1',
8888,
loop=loop,
)
self.reader = reader
self.writer = writer
await self.exchange()
print(self.board_str())
async def move(self, x, y):
self.board.cells[y * 3 + x] = self.cell_ctor(
x=True,
o=None,
unset=None,
)
print(self.board_str())
await self.exchange()
print(self.board_str())
def test_serialize_map():
roots, test_proto, content = generate_proto_binary(
'test_map.proto',
b'''simple_map: [{
key: "foo"
value: 1
},
{
key: "bar"
value: 2
},
{
key: "baz"
value: 3
}]
inner_map: [{
key: 1
value: {
sint_uint: [{
key: 1
value: 1
},
{
key: 2,
value: 2
},
{
key: 3
value: 3
}]
}
},
{
key: 2
value: {
sint_uint: [{
key: -1
value: 1
},
{
key: -2,
value: 2
},
{
key: -3
value: 3
}]
}
},
{
key: 3
value: {
sint_uint: [{
key: -1
value: 2
},
{
key: -2,
value: 4
},
{
key: -3
value: 6
}]
}
}]
inner: {
sint_uint: [{
key: 1
value: 1
},
{
key: 2,
value: 2
},
{
key: 3
value: 3
}]
}
inner_inner: {
bytes_inner_map: [{
key: 123
value: {
sint_uint: [{
key: -1
value: 4
},
{
key: -2,
value: 8
},
{
key: -3
value: 12
}]
}
},
{
key: 456
value: {
sint_uint: [{
key: -1
value: 8
},
{
key: -2,
value: 16
},
{
key: -3
value: 24
}]
}
},
{
key: 789
value: {
sint_uint: [{
key: 1
value: 16
},
{
key: 2,
value: 32
},
{
key: 3
value: 48
}]
}
}]
}
inner_inner_inner: {
string_inner_inner_map: [{
key: "foo"
value: {
bytes_inner_map: [{
key: 12
value: {
sint_uint: [{
key: 1
value: 16
},
{
key: 2,
value: 32
},
{
key: 3
value: 48
}]
}
},
{
key: 34
value: {
sint_uint: [{
key: 1
value: 16
},
{
key: 2,
value: 32
},
{
key: 3
value: 48
}]
}
}]
},
},
{
key: "bar"
value: {
bytes_inner_map: [{
key: 56
value: {
sint_uint: [{
key: 1
value: 16
},
{
key: 2,
value: 32
},
{
key: 3
value: 48
}]
}
},
{
key: 78
value: {
sint_uint: [{
key: 1
value: 16
},
{
key: 2,
value: 32
},
{
key: 3
value: 48
}]
}
}]
}
}]
}
''',
)
print('generated proto message: {}'.format(content))
parser = BinParser(roots)
result1 = parser.parse(test_proto, 'Test', content)
print('result: {}'.format(result1))
serializer = Serializer(parser)
payload = serializer.serialize(result1, 'Test')
print('payload: {}'.format(payload))
result2 = parser.parse(
test_proto,
'Test',
payload,
)
assert result1 == result2
class Runner:
def __init__(
self,
idl,
pyidl,
bins,
times=100,
threads=1,
optimize_enum_ctor=True,
optimize_message_ctor=True,
):
self.idl = idl
self.pyidl = pyidl
self.bins = bins
self.threads = threads
self.times = times
self.protopy_parser = BinParser([path.dirname(self.idl)])
if optimize_enum_ctor:
self.protopy_parser.def_parser.enum_ctor = simple_enum
if optimize_message_ctor:
self.protopy_parser.def_parser.message_ctor = simple_message
self.protopy_parser.def_parser.parse(path.basename(self.idl))
sys.path.insert(0, path.dirname(self.pyidl))
self.pdef = import_module(path.splitext(path.basename(
self.pyidl))[0]).AllMessages
def run_protopy(self):
start, step = time(), 0
io_time, parse_time = 0, 0
size = 0
# yep.start()
for _ in range(self.times):
for b in self.bins:
with open(b, 'rb') as payload:
r = payload.read()
step = time() - start
io_time += step
start = step + start
size += len(r)
self.protopy_parser.parse(
path.basename(self.idl),
b'AllMessages',
r,
)
step = time() - start
parse_time += step
start = step + start
# yep.stop()
print('Protopy:')
print('========')
print('Bytes processed: {}'.format(size))
print('I/O time: {}'.format(io_time))
print('Parse time: {}'.format(parse_time))
def run_protobuf(self):
start, step = time(), 0
io_time, parse_time = 0, 0
size = 0
for _ in range(self.times):
for b in self.bins:
with open(b, 'rb') as payload:
r = payload.read()
step = time() - start
io_time += step
start = step + start
size += len(r)
d = self.pdef()
d.ParseFromString(r)
step = time() - start
parse_time += step
start = step + start
print('Google Protobuf:')
print('========')
print('Bytes processed: {}'.format(size))
print('I/O time: {}'.format(io_time))
print('Parse time: {}'.format(parse_time))
class TicTacToeServer:
def __init__(self):
self.buffsize = 100
self.sessions = {}
root = pkg_resources.resource_filename(
__name__,
'etc',
)
self.parser = BinParser([root])
self.proto_def = path.join(root, 'tictactoe.proto')
self.parser.def_parser.parse(self.proto_def)
self.serializer = Serializer(self.parser)
self.connected = False
self.dparser = self.parser.def_parser
self.cell_ctor = self.dparser.find_definition('TicTacToeCell')
self.game_ctor = self.dparser.find_definition('TicTacToe')
self.board_ctor = self.dparser.find_definition('TicTacToeBoard')
self.game = TicTacToeGame(self.cell_ctor)
self.last_session = 0
def gensession(self):
self.last_session += 1
return self.last_session
def diff_boards(self, a, b):
diff = {}
for i, (ca, cb), in enumerate(zip(a.cells, b.cells)):
if ca != cb:
changes = []
if ca.x != cb.x:
changes.append('x')
if ca.o != cb.o:
changes.append('o')
if ca.unset != cb.unset:
changes.append('unset')
diff[i] = changes
return diff
async def write_response(self, message, writer):
response = self.serializer.serialize(message, 'TicTacToe')
writer.write(response)
try:
await writer.drain()
except ConnectionResetError:
self.connected = False
writer.close()
async def session_lost(self, board, writer):
await self.write_response(
self.game_ctor(
board=None,
error=SessionLost(
self.dparser,
'Session {} is no longer active'.format(
board.session,
)).proto(),
),
writer,
)
del self.sessions[board.sessions]
self.connected = False
async def illegal_move(self, writer):
await self.write_response(
self.game_ctor(
board=None,
error=InvalidMove(
self.dparser,
'That move is not allowed'
).proto(),
),
writer,
)
self.connected = False
async def game_error(self, error, writer):
await self.write_response(
self.game_ctor(
board=None,
error=eror.proto(),
),
writer,
)
self.connected = False
async def turn(self, reader, writer):
data, message = b'', None
while True:
try:
try:
chunk = await reader.read(self.buffsize)
if not chunk:
self.connected = False
break
data += chunk
except ConnectionResetError:
self.connected = False
writer.close()
return
message = self.parser.parse(self.proto_def, 'TicTacToe', data)
if message.board.session:
try:
known = self.sessions[message.board.session]
except KeyError:
await self.session_lost(message.board, writer)
return
diff = self.diff_boards(known, message.board)
if len(diff) != 1:
await self.illegal_move(writer)
return
except Exception as e:
print('what happened: {}'.format(e))
traceback.print_tb(sys.exc_info()[2])
else:
break
if self.connected:
try:
self.game.move(message.board, self.dparser)
session = message.board.session or self.gensession()
if is_board_full(message.board):
del self.sessions[message.board.session]
else:
self.sessions[session] = message.board
message = self.game_ctor(
board=self.board_ctor(
cells=message.board.cells,
session=session,
),
error=None,
)
await self.write_response(message, writer)
except TicTacToeError as e:
await self.game_error(e, writer)
return
async def handle(self, reader, writer):
self.connected = True
while self.connected:
await self.turn(reader, writer)