def BoolEncoder(field_number, is_repeated, is_packed):
"""Returns an encoder for a boolean field."""
false_byte = b('\x00')
true_byte = b('\x01')
if is_packed:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
local_EncodeVarint = _EncodeVarint
def EncodePackedField(write, value):
write(tag_bytes)
local_EncodeVarint(write, len(value))
for element in value:
if element:
write(true_byte)
else:
write(false_byte)
return EncodePackedField
elif is_repeated:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
def EncodeRepeatedField(write, value):
for element in value:
write(tag_bytes)
if element:
write(true_byte)
else:
write(false_byte)
return EncodeRepeatedField
else:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
def EncodeField(write, value):
write(tag_bytes)
if value:
return write(true_byte)
return write(false_byte)
return EncodeField
def RemoveRedundantZeros(self, text):
# Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove
# these zeros in order to match the golden file.
text = text.replace(b('e+0'),b('e+')).replace(b('e+0'),b('e+')) \
.replace(b('e-0'),b('e-')).replace(b('e-0'),b('e-'))
# Floating point fields are printed with .0 suffix even if they are
# actualy integer numbers.
text = re.compile(b('\.0$'), re.MULTILINE).sub(b(''), text)
return text
def EncodeNonFiniteOrRaise(write, value):
if value == _POS_INF:
write(b('\x00\x00\x00\x00\x00\x00\xF0\x7F'))
elif value == _NEG_INF:
write(b('\x00\x00\x00\x00\x00\x00\xF0\xFF'))
elif value != value: # NaN
write(b('\x00\x00\x00\x00\x00\x00\xF8\x7F'))
else:
raise BaseException()
def EncodeNonFiniteOrRaise(write, value):
if value == _POS_INF:
write(b('\x00\x00\x00\x00\x00\x00\xF0\x7F'))
elif value == _NEG_INF:
write(b('\x00\x00\x00\x00\x00\x00\xF0\xFF'))
elif value != value: # NaN
write(b('\x00\x00\x00\x00\x00\x00\xF8\x7F'))
else:
raise BaseException()
def EncodeNonFiniteOrRaise(write, value):
# Remember that the serialized form uses little-endian byte order.
if value == _POS_INF:
write(b('\x00\x00\x80\x7F'))
elif value == _NEG_INF:
write(b('\x00\x00\x80\xFF'))
elif value != value: # NaN
write(b('\x00\x00\xC0\x7F'))
else:
raise BaseException()
def testPrintBadEnumValueExtensions(self):
message = unittest_pb2.TestAllExtensions()
message.Extensions[unittest_pb2.optional_nested_enum_extension] = 100
message.Extensions[unittest_pb2.optional_foreign_enum_extension] = 101
message.Extensions[unittest_pb2.optional_import_enum_extension] = 102
self.CompareToGoldenText(
text_format.MessageToString(message),
b('[protobuf_unittest.optional_nested_enum_extension]: 100\n') +
b('[protobuf_unittest.optional_foreign_enum_extension]: 101\n') +
b('[protobuf_unittest.optional_import_enum_extension]: 102\n'))
def testPrintBadEnumValue(self):
message = unittest_pb2.TestAllTypes()
message.optional_nested_enum = 100
message.optional_foreign_enum = 101
message.optional_import_enum = 102
self.CompareToGoldenText(
text_format.MessageToString(message),
b('optional_nested_enum: 100\n') +
b('optional_foreign_enum: 101\n') +
b('optional_import_enum: 102\n'))
def EncodeNonFiniteOrRaise(write, value):
# Remember that the serialized form uses little-endian byte order.
if value == _POS_INF:
write(b('\x00\x00\x80\x7F'))
elif value == _NEG_INF:
write(b('\x00\x00\x80\xFF'))
elif value != value: # NaN
write(b('\x00\x00\xC0\x7F'))
else:
raise BaseException()
def PrintFieldValue(field, value, out, indent=0,
as_utf8=False, as_one_line=False):
"""Print a single field value (not including name). For repeated fields,
the value should be a single element."""
if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
if as_one_line:
out.write(b(' { '))
PrintMessage(value, out, indent, as_utf8, as_one_line)
out.write(b('}'))
else:
out.write(b(' {\n'))
PrintMessage(value, out, indent + 2, as_utf8, as_one_line)
out.write(b(' ') * indent + b('}'))
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
enum_value = field.enum_type.values_by_number.get(value, None)
if enum_value is not None:
out.write(string_to_bytes(str(enum_value.name)))
else:
out.write(string_to_bytes(str(value)))
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
out.write(b('\"'))
if type(value) is unicode:
out.write(_CEscape(value.encode('utf-8'), as_utf8))
else:
out.write(_CEscape(value, as_utf8))
out.write(b('\"'))
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
if value:
out.write(b("true"))
else:
out.write(b("false"))
else:
out.write(string_to_bytes(str(value)))
def testPrintRepeatedFieldsAsOneLine(self):
message = unittest_pb2.TestAllTypes()
message.repeated_int32.append(1)
message.repeated_int32.append(1)
message.repeated_int32.append(3)
message.repeated_string.append("Google")
message.repeated_string.append("Zurich")
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
b('repeated_int32: 1 repeated_int32: 1 repeated_int32: 3 ') +
b('repeated_string: "Google" repeated_string: "Zurich"'))
def testMergeGroupNotClosed(self):
message = unittest_pb2.TestAllTypes()
text = b('RepeatedGroup: <')
self.assertRaisesWithMessage(text_format.ParseError,
'1:16 : Expected ">".', text_format.Merge,
text, message)
text = b('RepeatedGroup: {')
self.assertRaisesWithMessage(text_format.ParseError,
'1:16 : Expected "}".', text_format.Merge,
text, message)
def testMergeEmptyGroup(self):
message = unittest_pb2.TestAllTypes()
text = b('OptionalGroup: {}')
text_format.Merge(text, message)
self.assertTrue(message.HasField('optionalgroup'))
message.Clear()
message = unittest_pb2.TestAllTypes()
text = b('OptionalGroup: <>')
text_format.Merge(text, message)
self.assertTrue(message.HasField('optionalgroup'))
def __init__(self, text_message):
self._text_message = text_message
self._position = 0
self._line = -1
self._column = 0
self._token_start = None
self.token = b('')
self._lines = deque(text_message.split(b('\n')))
self._current_line = b('')
self._previous_line = 0
self._previous_column = 0
self._SkipWhitespace()
self.NextToken()
def __init__(self, text_message):
self._text_message = text_message
self._position = 0
self._line = -1
self._column = 0
self._token_start = None
self.token = b('')
self._lines = deque(text_message.split(b('\n')))
self._current_line = b('')
self._previous_line = 0
self._previous_column = 0
self._SkipWhitespace()
self.NextToken()
def testMergeBadEnumValue(self):
message = unittest_pb2.TestAllTypes()
text = b('optional_nested_enum: BARR')
self.assertRaisesWithMessage(
text_format.ParseError,
('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
'has no value named BARR.'), text_format.Merge, text, message)
message = unittest_pb2.TestAllTypes()
text = b('optional_nested_enum: 100')
self.assertRaisesWithMessage(
text_format.ParseError,
('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
'has no value with number 100.'), text_format.Merge, text,
message)
def MessageSetItemEncoder(field_number):
"""Encoder for extensions of MessageSet.
The message set message looks like this:
message MessageSet {
repeated group Item = 1 {
required int32 type_id = 2;
required string message = 3;
}
}
"""
start_bytes = b("").join([
TagBytes(1, wire_format.WIRETYPE_START_GROUP),
TagBytes(2, wire_format.WIRETYPE_VARINT),
_VarintBytes(field_number),
TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)])
end_bytes = TagBytes(1, wire_format.WIRETYPE_END_GROUP)
local_EncodeVarint = _EncodeVarint
def EncodeField(write, value):
write(start_bytes)
local_EncodeVarint(write, value.ByteSize())
value._InternalSerialize(write)
return write(end_bytes)
return EncodeField
def ParseFloat(text):
"""Parse a floating point number.
Args:
text: Text to parse.
Returns:
The number parsed.
Raises:
ValueError: If a floating point number couldn't be parsed.
"""
try:
# Assume Python compatible syntax.
return float(text)
except ValueError:
# Check alternative spellings.
if _FLOAT_INFINITY.match(text):
if text[0] == '-':
return float('-inf')
else:
return float('inf')
elif _FLOAT_NAN.match(text):
return float('nan')
else:
# assume '1.0f' format
try:
return float(text.rstrip(b('f')))
except ValueError:
raise ValueError('Couldn\'t parse float: %s' % text)
def testPrintNestedMessageAsOneLine(self):
message = unittest_pb2.TestAllTypes()
msg = message.repeated_nested_message.add()
msg.bb = 42
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
b('repeated_nested_message { bb: 42 }'))
def ExpectAllFieldsAndExtensionsInOrder(serialized):
"""Ensures that serialized is the serialization we expect for a message
filled with SetAllFieldsAndExtensions(). (Specifically, ensures that the
serialization is in canonical, tag-number order).
"""
my_extension_int = unittest_pb2.my_extension_int
my_extension_string = unittest_pb2.my_extension_string
expected_strings = []
message = unittest_pb2.TestFieldOrderings()
message.my_int = 1 # Field 1.
expected_strings.append(message.SerializeToString())
message.Clear()
message.Extensions[my_extension_int] = 23 # Field 5.
expected_strings.append(message.SerializeToString())
message.Clear()
message.my_string = 'foo' # Field 11.
expected_strings.append(message.SerializeToString())
message.Clear()
message.Extensions[my_extension_string] = 'bar' # Field 50.
expected_strings.append(message.SerializeToString())
message.Clear()
message.my_float = 1.0
expected_strings.append(message.SerializeToString())
message.Clear()
expected = b('').join(expected_strings)
if expected != serialized:
raise ValueError('Expected %r, found %r' % (expected, serialized))
def MessageSetItemEncoder(field_number):
"""Encoder for extensions of MessageSet.
The message set message looks like this:
message MessageSet {
repeated group Item = 1 {
required int32 type_id = 2;
required string message = 3;
}
}
"""
start_bytes = b("").join([
TagBytes(1, wire_format.WIRETYPE_START_GROUP),
TagBytes(2, wire_format.WIRETYPE_VARINT),
_VarintBytes(field_number),
TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)
])
end_bytes = TagBytes(1, wire_format.WIRETYPE_END_GROUP)
local_EncodeVarint = _EncodeVarint
def EncodeField(write, value):
write(start_bytes)
local_EncodeVarint(write, value.ByteSize())
value._InternalSerialize(write)
return write(end_bytes)
return EncodeField
def _VarintBytes(value):
"""Encode the given integer as a varint and return the bytes. This is only
called at startup time so it doesn't need to be fast."""
pieces = []
_EncodeVarint(pieces.append, value)
return b("").join(pieces)
def _VarintBytes(value):
"""Encode the given integer as a varint and return the bytes. This is only
called at startup time so it doesn't need to be fast."""
pieces = []
_EncodeVarint(pieces.append, value)
return b("").join(pieces)
def ExpectAllFieldsAndExtensionsInOrder(serialized):
"""Ensures that serialized is the serialization we expect for a message
filled with SetAllFieldsAndExtensions(). (Specifically, ensures that the
serialization is in canonical, tag-number order).
"""
my_extension_int = unittest_pb2.my_extension_int
my_extension_string = unittest_pb2.my_extension_string
expected_strings = []
message = unittest_pb2.TestFieldOrderings()
message.my_int = 1 # Field 1.
expected_strings.append(message.SerializeToString())
message.Clear()
message.Extensions[my_extension_int] = 23 # Field 5.
expected_strings.append(message.SerializeToString())
message.Clear()
message.my_string = 'foo' # Field 11.
expected_strings.append(message.SerializeToString())
message.Clear()
message.Extensions[my_extension_string] = 'bar' # Field 50.
expected_strings.append(message.SerializeToString())
message.Clear()
message.my_float = 1.0
expected_strings.append(message.SerializeToString())
message.Clear()
expected = b('').join(expected_strings)
if expected != serialized:
raise ValueError('Expected %r, found %r' % (expected, serialized))
def AtEnd(self):
"""Checks the end of the text was reached.
Returns:
True iff the end was reached.
"""
return self.token == b('')
def ParseFloat(text):
"""Parse a floating point number.
Args:
text: Text to parse.
Returns:
The number parsed.
Raises:
ValueError: If a floating point number couldn't be parsed.
"""
try:
# Assume Python compatible syntax.
return float(text)
except ValueError:
# Check alternative spellings.
if _FLOAT_INFINITY.match(text):
if text[0] == '-':
return float('-inf')
else:
return float('inf')
elif _FLOAT_NAN.match(text):
return float('nan')
else:
# assume '1.0f' format
try:
return float(text.rstrip(b('f')))
except ValueError:
raise ValueError('Couldn\'t parse float: %s' % text)
def AtEnd(self):
"""Checks the end of the text was reached.
Returns:
True iff the end was reached.
"""
return self.token == b('')
def testConsumeIntegers(self):
# This test only tests the failures in the integer parsing methods as well
# as the '0' special cases.
int64_max = (1 << 63) - 1
uint32_max = (1 << 32) - 1
text = string_to_bytestr('-1 %d %d' % (uint32_max + 1, int64_max + 1))
tokenizer = text_format._Tokenizer(text)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint64)
self.assertEqual(-1, tokenizer.ConsumeInt32())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt32)
self.assertEqual(uint32_max + 1, tokenizer.ConsumeInt64())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt64)
self.assertEqual(int64_max + 1, tokenizer.ConsumeUint64())
self.assertTrue(tokenizer.AtEnd())
text = b('-0 -0 0 0')
tokenizer = text_format._Tokenizer(text)
self.assertEqual(0, tokenizer.ConsumeUint32())
self.assertEqual(0, tokenizer.ConsumeUint64())
self.assertEqual(0, tokenizer.ConsumeUint32())
self.assertEqual(0, tokenizer.ConsumeUint64())
self.assertTrue(tokenizer.AtEnd())
def testCopyToProto_FileDescriptor(self):
UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII = (
b(
"""
name: 'google/protobuf/unittest_import.proto'
package: 'protobuf_unittest_import'
dependency: 'google/protobuf/unittest_import_public.proto'
message_type: <
name: 'ImportMessage'
field: <
name: 'd'
number: 1
label: 1 # Optional
type: 5 # TYPE_INT32
>
>
"""
)
+ b(
"""enum_type: <
name: 'ImportEnum'
value: <
name: 'IMPORT_FOO'
number: 7
>
value: <
name: 'IMPORT_BAR'
number: 8
>
value: <
name: 'IMPORT_BAZ'
number: 9
>
>
options: <
java_package: 'com.google.protobuf.test'
optimize_for: 1 # SPEED
>
public_dependency: 0
"""
)
)
self._InternalTestCopyToProto(
unittest_import_pb2.DESCRIPTOR, descriptor_pb2.FileDescriptorProto, UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII
)
def _PopLine(self):
while len(self._current_line) <= self._column:
if not self._lines:
self._current_line = b('')
return
self._line += 1
self._column = 0
self._current_line = self._lines.popleft()
def _PopLine(self):
while len(self._current_line) <= self._column:
if not self._lines:
self._current_line = b('')
return
self._line += 1
self._column = 0
self._current_line = self._lines.popleft()
def testPrintRawUtf8String(self):
message = unittest_pb2.TestAllTypes()
message.repeated_string.append(u('\u00fc\ua71f'))
text = text_format.MessageToString(message, as_utf8=True)
self.CompareToGoldenText(
text, b('repeated_string: "\303\274\352\234\237"\n'))
parsed_message = unittest_pb2.TestAllTypes()
text_format.Merge(text, parsed_message)
self.assertEqual(message, parsed_message)
def testMergeGoldenExtensions(self):
golden_text = b('\n').join(
self.ReadGolden('text_format_unittest_extensions_data.txt'))
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Merge(golden_text, parsed_message)
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.assertEqual(message, parsed_message)
def InnerDecode(buffer, pos):
# We expect a 64-bit value in little-endian byte order. Bit 1 is the sign
# bit, bits 2-12 represent the exponent, and bits 13-64 are the significand.
new_pos = pos + 8
double_bytes = buffer[pos:new_pos]
# If this value has all its exponent bits set and at least one significand
# bit set, it's not a number. In Python 2.4, struct.unpack will treat it
# as inf or -inf. To avoid that, we treat it specially.
if ((double_bytes[7] in b('\x7F\xFF')) and (double_bytes[6] >= 240)
and (double_bytes[0:7] != b('\x00\x00\x00\x00\x00\x00\xF0'))):
return (_NAN, new_pos)
# Note that we expect someone up-stack to catch struct.error and convert
# it to _DecodeError -- this way we don't have to set up exception-
# handling blocks every time we parse one value.
result = local_unpack('<d', double_bytes)[0]
return (result, new_pos)
def testMergeMessageSet(self):
message = unittest_pb2.TestAllTypes()
text = (b('repeated_uint64: 1\n') + b('repeated_uint64: 2\n'))
text_format.Merge(text, message)
self.assertEqual(1, message.repeated_uint64[0])
self.assertEqual(2, message.repeated_uint64[1])
message = unittest_mset_pb2.TestMessageSetContainer()
text = (b('message_set {\n') +
b(' [protobuf_unittest.TestMessageSetExtension1] {\n') +
b(' i: 23\n') + b(' }\n') +
b(' [protobuf_unittest.TestMessageSetExtension2] {\n') +
b(' str: \"foo\"\n') + b(' }\n') + b('}\n'))
text_format.Merge(text, message)
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
self.assertEqual(23, message.message_set.Extensions[ext1].i)
self.assertEqual('foo', message.message_set.Extensions[ext2].str)
def testDifferentCustomOptionTypes(self):
kint32min = -2 ** 31
kint64min = -2 ** 63
kint32max = 2 ** 31 - 1
kint64max = 2 ** 63 - 1
kuint32max = 2 ** 32 - 1
kuint64max = 2 ** 64 - 1
message_descriptor = unittest_custom_options_pb2.CustomOptionMinIntegerValues.DESCRIPTOR
message_options = message_descriptor.GetOptions()
self.assertEqual(False, message_options.Extensions[unittest_custom_options_pb2.bool_opt])
self.assertEqual(kint32min, message_options.Extensions[unittest_custom_options_pb2.int32_opt])
self.assertEqual(kint64min, message_options.Extensions[unittest_custom_options_pb2.int64_opt])
self.assertEqual(0, message_options.Extensions[unittest_custom_options_pb2.uint32_opt])
self.assertEqual(0, message_options.Extensions[unittest_custom_options_pb2.uint64_opt])
self.assertEqual(kint32min, message_options.Extensions[unittest_custom_options_pb2.sint32_opt])
self.assertEqual(kint64min, message_options.Extensions[unittest_custom_options_pb2.sint64_opt])
self.assertEqual(0, message_options.Extensions[unittest_custom_options_pb2.fixed32_opt])
self.assertEqual(0, message_options.Extensions[unittest_custom_options_pb2.fixed64_opt])
self.assertEqual(kint32min, message_options.Extensions[unittest_custom_options_pb2.sfixed32_opt])
self.assertEqual(kint64min, message_options.Extensions[unittest_custom_options_pb2.sfixed64_opt])
message_descriptor = unittest_custom_options_pb2.CustomOptionMaxIntegerValues.DESCRIPTOR
message_options = message_descriptor.GetOptions()
self.assertEqual(True, message_options.Extensions[unittest_custom_options_pb2.bool_opt])
self.assertEqual(kint32max, message_options.Extensions[unittest_custom_options_pb2.int32_opt])
self.assertEqual(kint64max, message_options.Extensions[unittest_custom_options_pb2.int64_opt])
self.assertEqual(kuint32max, message_options.Extensions[unittest_custom_options_pb2.uint32_opt])
self.assertEqual(kuint64max, message_options.Extensions[unittest_custom_options_pb2.uint64_opt])
self.assertEqual(kint32max, message_options.Extensions[unittest_custom_options_pb2.sint32_opt])
self.assertEqual(kint64max, message_options.Extensions[unittest_custom_options_pb2.sint64_opt])
self.assertEqual(kuint32max, message_options.Extensions[unittest_custom_options_pb2.fixed32_opt])
self.assertEqual(kuint64max, message_options.Extensions[unittest_custom_options_pb2.fixed64_opt])
self.assertEqual(kint32max, message_options.Extensions[unittest_custom_options_pb2.sfixed32_opt])
self.assertEqual(kint64max, message_options.Extensions[unittest_custom_options_pb2.sfixed64_opt])
message_descriptor = unittest_custom_options_pb2.CustomOptionOtherValues.DESCRIPTOR
message_options = message_descriptor.GetOptions()
self.assertEqual(-100, message_options.Extensions[unittest_custom_options_pb2.int32_opt])
self.assertAlmostEqual(12.3456789, message_options.Extensions[unittest_custom_options_pb2.float_opt], 6)
self.assertAlmostEqual(1.234567890123456789, message_options.Extensions[unittest_custom_options_pb2.double_opt])
self.assertEqual('Hello, "World"', message_options.Extensions[unittest_custom_options_pb2.string_opt])
self.assertEqual(b("Hello\0World"), message_options.Extensions[unittest_custom_options_pb2.bytes_opt])
dummy_enum = unittest_custom_options_pb2.DummyMessageContainingEnum
self.assertEqual(
dummy_enum.TEST_OPTION_ENUM_TYPE2, message_options.Extensions[unittest_custom_options_pb2.enum_opt]
)
message_descriptor = unittest_custom_options_pb2.SettingRealsFromPositiveInts.DESCRIPTOR
message_options = message_descriptor.GetOptions()
self.assertAlmostEqual(12, message_options.Extensions[unittest_custom_options_pb2.float_opt], 6)
self.assertAlmostEqual(154, message_options.Extensions[unittest_custom_options_pb2.double_opt])
message_descriptor = unittest_custom_options_pb2.SettingRealsFromNegativeInts.DESCRIPTOR
message_options = message_descriptor.GetOptions()
self.assertAlmostEqual(-12, message_options.Extensions[unittest_custom_options_pb2.float_opt], 6)
self.assertAlmostEqual(-154, message_options.Extensions[unittest_custom_options_pb2.double_opt])
def InnerDecode(buffer, pos):
# We expect a 64-bit value in little-endian byte order. Bit 1 is the sign
# bit, bits 2-12 represent the exponent, and bits 13-64 are the significand.
new_pos = pos + 8
double_bytes = buffer[pos:new_pos]
# If this value has all its exponent bits set and at least one significand
# bit set, it's not a number. In Python 2.4, struct.unpack will treat it
# as inf or -inf. To avoid that, we treat it specially.
if ((double_bytes[7] in b('\x7F\xFF'))
and (double_bytes[6] >= 240)
and (double_bytes[0:7] != b('\x00\x00\x00\x00\x00\x00\xF0'))):
return (_NAN, new_pos)
# Note that we expect someone up-stack to catch struct.error and convert
# it to _DecodeError -- this way we don't have to set up exception-
# handling blocks every time we parse one value.
result = local_unpack('<d', double_bytes)[0]
return (result, new_pos)
def BoolEncoder(field_number, is_repeated, is_packed):
"""Returns an encoder for a boolean field."""
false_byte = b('\x00')
true_byte = b('\x01')
if is_packed:
tag_bytes = TagBytes(field_number,
wire_format.WIRETYPE_LENGTH_DELIMITED)
local_EncodeVarint = _EncodeVarint
def EncodePackedField(write, value):
write(tag_bytes)
local_EncodeVarint(write, len(value))
for element in value:
if element:
write(true_byte)
else:
write(false_byte)
return EncodePackedField
elif is_repeated:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
def EncodeRepeatedField(write, value):
for element in value:
write(tag_bytes)
if element:
write(true_byte)
else:
write(false_byte)
return EncodeRepeatedField
else:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
def EncodeField(write, value):
write(tag_bytes)
if value:
return write(true_byte)
return write(false_byte)
return EncodeField
def testMergeBadExtension(self):
message = unittest_pb2.TestAllExtensions()
text = b('[unknown_extension]: 8\n')
self.assertRaisesWithMessage(
text_format.ParseError,
'1:2 : Extension "unknown_extension" not registered.',
text_format.Merge, text, message)
message = unittest_pb2.TestAllTypes()
self.assertRaisesWithMessage(text_format.ParseError, (
'1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
'extensions.'), text_format.Merge, text, message)
def testConsumeByteString(self):
text = b('"string1\'')
tokenizer = text_format._Tokenizer(text)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
text = b('string1"')
tokenizer = text_format._Tokenizer(text)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
text = b('\n"\\xt"')
tokenizer = text_format._Tokenizer(text)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
text = b('\n"\\"')
tokenizer = text_format._Tokenizer(text)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
text = b('\n"\\x"')
tokenizer = text_format._Tokenizer(text)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
def testPrintMessageSet(self):
message = unittest_mset_pb2.TestMessageSetContainer()
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(
text_format.MessageToString(message),
b('message_set {\n') +
b(' [protobuf_unittest.TestMessageSetExtension1] {\n') +
b(' i: 23\n') + b(' }\n') +
b(' [protobuf_unittest.TestMessageSetExtension2] {\n') +
b(' str: \"foo\"\n') + b(' }\n') + b('}\n'))
def _MergeScalarField(tokenizer, message, field):
"""Merges a single protocol message scalar field into a message.
Args:
tokenizer: A tokenizer to parse the field value.
message: A protocol message to record the data.
field: The descriptor of the field to be merged.
Raises:
ParseError: In case of ASCII parsing problems.
RuntimeError: On runtime errors.
"""
tokenizer.Consume(b(':'))
value = None
if field.type in (descriptor.FieldDescriptor.TYPE_INT32,
descriptor.FieldDescriptor.TYPE_SINT32,
descriptor.FieldDescriptor.TYPE_SFIXED32):
value = tokenizer.ConsumeInt32()
elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,
descriptor.FieldDescriptor.TYPE_SINT64,
descriptor.FieldDescriptor.TYPE_SFIXED64):
value = tokenizer.ConsumeInt64()
elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,
descriptor.FieldDescriptor.TYPE_FIXED32):
value = tokenizer.ConsumeUint32()
elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,
descriptor.FieldDescriptor.TYPE_FIXED64):
value = tokenizer.ConsumeUint64()
elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,
descriptor.FieldDescriptor.TYPE_DOUBLE):
value = tokenizer.ConsumeFloat()
elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:
value = tokenizer.ConsumeBool()
elif field.type == descriptor.FieldDescriptor.TYPE_STRING:
value = tokenizer.ConsumeString()
elif field.type == descriptor.FieldDescriptor.TYPE_BYTES:
value = tokenizer.ConsumeByteString()
elif field.type == descriptor.FieldDescriptor.TYPE_ENUM:
value = tokenizer.ConsumeEnum(field)
else:
raise RuntimeError('Unknown field type %d' % field.type)
if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
if field.is_extension:
message.Extensions[field].append(value)
else:
getattr(message, field.name).append(value)
else:
if field.is_extension:
message.Extensions[field] = value
else:
setattr(message, field.name, value)
def _ConsumeSingleByteString(self):
"""Consume one token of a string literal.
String literals (whether bytes or text) can come in multiple adjacent
tokens which are automatically concatenated, like in C or Python. This
method only consumes one token.
"""
text = self.token
if len(text) < 1 or text[0:1] not in (b('\''), b('"')):
raise self._ParseError("Expected string, got: %s" % text)
if len(text) < 2 or text[-1] != text[0]:
raise self._ParseError("String missing ending quote, got: %s" % text)
try:
result = _CUnescape(text[1:-1])
except ValueError:
_, e, _ = sys.exc_info()
raise self._ParseError(str(e))
self.NextToken()
return result
def _MergeScalarField(tokenizer, message, field):
"""Merges a single protocol message scalar field into a message.
Args:
tokenizer: A tokenizer to parse the field value.
message: A protocol message to record the data.
field: The descriptor of the field to be merged.
Raises:
ParseError: In case of ASCII parsing problems.
RuntimeError: On runtime errors.
"""
tokenizer.Consume(b(':'))
value = None
if field.type in (descriptor.FieldDescriptor.TYPE_INT32,
descriptor.FieldDescriptor.TYPE_SINT32,
descriptor.FieldDescriptor.TYPE_SFIXED32):
value = tokenizer.ConsumeInt32()
elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,
descriptor.FieldDescriptor.TYPE_SINT64,
descriptor.FieldDescriptor.TYPE_SFIXED64):
value = tokenizer.ConsumeInt64()
elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,
descriptor.FieldDescriptor.TYPE_FIXED32):
value = tokenizer.ConsumeUint32()
elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,
descriptor.FieldDescriptor.TYPE_FIXED64):
value = tokenizer.ConsumeUint64()
elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,
descriptor.FieldDescriptor.TYPE_DOUBLE):
value = tokenizer.ConsumeFloat()
elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:
value = tokenizer.ConsumeBool()
elif field.type == descriptor.FieldDescriptor.TYPE_STRING:
value = tokenizer.ConsumeString()
elif field.type == descriptor.FieldDescriptor.TYPE_BYTES:
value = tokenizer.ConsumeByteString()
elif field.type == descriptor.FieldDescriptor.TYPE_ENUM:
value = tokenizer.ConsumeEnum(field)
else:
raise RuntimeError('Unknown field type %d' % field.type)
if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
if field.is_extension:
message.Extensions[field].append(value)
else:
getattr(message, field.name).append(value)
else:
if field.is_extension:
message.Extensions[field] = value
else:
setattr(message, field.name, value)
def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False):
"""Print a single field name/value pair. For repeated fields, the value
should be a single element."""
out.write(b(' ') * indent)
if field.is_extension:
out.write(b('['))
if (field.containing_type.GetOptions().message_set_wire_format
and field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
and field.message_type == field.extension_scope
and field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL):
out.write(string_to_bytestr(field.message_type.full_name))
else:
out.write(string_to_bytestr(field.full_name))
out.write(b(']'))
elif field.type == descriptor.FieldDescriptor.TYPE_GROUP:
# For groups, use the capitalized name.
out.write(string_to_bytestr(field.message_type.name))
else:
out.write(string_to_bytestr(field.name))
if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
# The colon is optional in this case, but our cross-language golden files
# don't include it.
out.write(b(': '))
PrintFieldValue(field, value, out, indent, as_utf8, as_one_line)
if as_one_line:
out.write(b(' '))
else:
out.write(b('\n'))
def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False):
"""Print a single field name/value pair. For repeated fields, the value
should be a single element."""
out.write(b(' ') * indent)
if field.is_extension:
out.write(b('['))
if (field.containing_type.GetOptions().message_set_wire_format and
field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
field.message_type == field.extension_scope and
field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL):
out.write(string_to_bytestr(field.message_type.full_name))
else:
out.write(string_to_bytestr(field.full_name))
out.write(b(']'))
elif field.type == descriptor.FieldDescriptor.TYPE_GROUP:
# For groups, use the capitalized name.
out.write(string_to_bytestr(field.message_type.name))
else:
out.write(string_to_bytestr(field.name))
if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
# The colon is optional in this case, but our cross-language golden files
# don't include it.
out.write(b(': '))
PrintFieldValue(field, value, out, indent, as_utf8, as_one_line)
if as_one_line:
out.write(b(' '))
else:
out.write(b('\n'))
def ConsumeByteString(self):
"""Consumes a byte array value.
Returns:
The array parsed (as a string).
Raises:
ParseError: If a byte array value couldn't be consumed.
"""
list = [self._ConsumeSingleByteString()]
while len(self.token) > 0 and ord(self.token[0:1]) in (ord('\''), ord('"')):
list.append(self._ConsumeSingleByteString())
return b("").join(list)
def testCopyToProto_FileDescriptor(self):
UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII = (b("""
name: 'google/protobuf/unittest_import.proto'
package: 'protobuf_unittest_import'
dependency: 'google/protobuf/unittest_import_public.proto'
message_type: <
name: 'ImportMessage'
field: <
name: 'd'
number: 1
label: 1 # Optional
type: 5 # TYPE_INT32
>
>
""") + b("""enum_type: <
name: 'ImportEnum'
value: <
name: 'IMPORT_FOO'
number: 7
>
value: <
name: 'IMPORT_BAR'
number: 8
>
value: <
name: 'IMPORT_BAZ'
number: 9
>
>
options: <
java_package: 'com.google.protobuf.test'
optimize_for: 1 # SPEED
>
public_dependency: 0
"""))
self._InternalTestCopyToProto(unittest_import_pb2.DESCRIPTOR,
descriptor_pb2.FileDescriptorProto,
UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII)
def testCopyToProto_AllExtensions(self):
TEST_EMPTY_MESSAGE_WITH_EXTENSIONS_ASCII = b("""
name: 'TestEmptyMessageWithExtensions'
extension_range: <
start: 1
end: 536870912
>
""")
self._InternalTestCopyToProto(
unittest_pb2.TestEmptyMessageWithExtensions.DESCRIPTOR,
descriptor_pb2.DescriptorProto,
TEST_EMPTY_MESSAGE_WITH_EXTENSIONS_ASCII)
def _CEscape(byte_array, as_utf8):
def escape(c):
o = bytes_as_num(c)
if o == 10: return b(r"\n") # optional escape
if o == 13: return b(r"\r") # optional escape
if o == 9: return b(r"\t") # optional escape
if o == 39: return b(r"\'") # optional escape
if o == 34: return b(r'\"') # necessary escape
if o == 92: return b(r"\\") # necessary escape
# necessary escapes
if not as_utf8 and (o >= 127 or o < 32): return string_to_bytes("\\%03o" % o)
return num_as_byte(c)
return b("").join([escape(c) for c in byte_array])
def InnerDecode(buffer, pos):
# We expect a 32-bit value in little-endian byte order. Bit 1 is the sign
# bit, bits 2-9 represent the exponent, and bits 10-32 are the significand.
new_pos = pos + 4
float_bytes = buffer[pos:new_pos]
# If this value has all its exponent bits set, then it's non-finite.
# In Python 2.4, struct.unpack will convert it to a finite 64-bit value.
# To avoid that, we parse it specially.
if ((float_bytes[3] in b('\x7F\xFF'))
and (float_bytes[2] >= 128)):
# If at least one significand bit is set...
if float_bytes[0:3] != b('\x00\x00\x80'):
return (_NAN, new_pos)
# If sign bit is set...
if float_bytes[3] == b('\xFF'):
return (_NEG_INF, new_pos)
return (_POS_INF, new_pos)
# Note that we expect someone up-stack to catch struct.error and convert
# it to _DecodeError -- this way we don't have to set up exception-
# handling blocks every time we parse one value.
result = local_unpack('<f', float_bytes)[0]
return (result, new_pos)
def testCopyToProto_AllExtensions(self):
TEST_EMPTY_MESSAGE_WITH_EXTENSIONS_ASCII = b(
"""
name: 'TestEmptyMessageWithExtensions'
extension_range: <
start: 1
end: 536870912
>
"""
)
self._InternalTestCopyToProto(
unittest_pb2.TestEmptyMessageWithExtensions.DESCRIPTOR,
descriptor_pb2.DescriptorProto,
TEST_EMPTY_MESSAGE_WITH_EXTENSIONS_ASCII,
)
def testCopyToProto_ForeignNestedMessage(self):
TEST_FOREIGN_NESTED_ASCII = b(
"""
name: 'TestForeignNested'
field: <
name: 'foreign_nested'
number: 1
label: 1 # Optional
type: 11 # TYPE_MESSAGE
type_name: '.protobuf_unittest.TestAllTypes.NestedMessage'
>
"""
)
self._InternalTestCopyToProto(
unittest_pb2.TestForeignNested.DESCRIPTOR, descriptor_pb2.DescriptorProto, TEST_FOREIGN_NESTED_ASCII
)
def testCopyToProto_NestedMessage(self):
TEST_NESTED_MESSAGE_ASCII = b(
"""
name: 'NestedMessage'
field: <
name: 'bb'
number: 1
label: 1 # Optional
type: 5 # TYPE_INT32
>
"""
)
self._InternalTestCopyToProto(
unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR,
descriptor_pb2.DescriptorProto,
TEST_NESTED_MESSAGE_ASCII,
)
def escape(c):
o = bytes_as_num(c)
if o == 10: return b(r"\n") # optional escape
if o == 13: return b(r"\r") # optional escape
if o == 9: return b(r"\t") # optional escape
if o == 39: return b(r"\'") # optional escape
if o == 34: return b(r'\"') # necessary escape
if o == 92: return b(r"\\") # necessary escape
# necessary escapes
if not as_utf8 and (o >= 127 or o < 32): return string_to_bytes("\\%03o" % o)
return num_as_byte(c)
def testCopyToProto_Options(self):
TEST_DEPRECATED_FIELDS_ASCII = b(
"""
name: 'TestDeprecatedFields'
field: <
name: 'deprecated_int32'
number: 1
label: 1 # Optional
type: 5 # TYPE_INT32
options: <
deprecated: true
>
>
"""
)
self._InternalTestCopyToProto(
unittest_pb2.TestDeprecatedFields.DESCRIPTOR, descriptor_pb2.DescriptorProto, TEST_DEPRECATED_FIELDS_ASCII
)
def NextToken(self):
"""Reads the next meaningful token."""
self._previous_line = self._line
self._previous_column = self._column
self._column += len(self.token)
self._SkipWhitespace()
if not self._lines and len(self._current_line) <= self._column:
self.token = b('')
return
match = self._TOKEN_BYTES.match(self._current_line, self._column)
if match:
token = match.group(0)
self.token = token
else:
self.token = self._current_line[self._column:self._column+1]
def testCopyToProto_ServiceDescriptor(self):
TEST_SERVICE_ASCII = b(
"""
name: 'TestService'
method: <
name: 'Foo'
input_type: '.protobuf_unittest.FooRequest'
output_type: '.protobuf_unittest.FooResponse'
>
method: <
name: 'Bar'
input_type: '.protobuf_unittest.BarRequest'
output_type: '.protobuf_unittest.BarResponse'
>
"""
)
self._InternalTestCopyToProto(
unittest_pb2.TestService.DESCRIPTOR, descriptor_pb2.ServiceDescriptorProto, TEST_SERVICE_ASCII
)
def testCopyToProto_ForeignEnum(self):
TEST_FOREIGN_ENUM_ASCII = b(
"""
name: 'ForeignEnum'
value: <
name: 'FOREIGN_FOO'
number: 4
>
value: <
name: 'FOREIGN_BAR'
number: 5
>
value: <
name: 'FOREIGN_BAZ'
number: 6
>
"""
)
self._InternalTestCopyToProto(
unittest_pb2._FOREIGNENUM, descriptor_pb2.EnumDescriptorProto, TEST_FOREIGN_ENUM_ASCII
)
def ParseBool(text):
"""Parse a boolean value.
Args:
text: Text to parse.
Returns:
Boolean values parsed
Raises:
ValueError: If text is not a valid boolean.
"""
if text in (b('true'), b('t'), b('1')):
return True
elif text in (b('false'), b('f'), b('0')):
return False
else:
raise ValueError('Expected "true" or "false".')
def testCopyToProto_SeveralExtensions(self):
TEST_MESSAGE_WITH_SEVERAL_EXTENSIONS_ASCII = b(
"""
name: 'TestMultipleExtensionRanges'
extension_range: <
start: 42
end: 43
>
extension_range: <
start: 4143
end: 4244
>
extension_range: <
start: 65536
end: 536870912
>
"""
)
self._InternalTestCopyToProto(
unittest_pb2.TestMultipleExtensionRanges.DESCRIPTOR,
descriptor_pb2.DescriptorProto,
TEST_MESSAGE_WITH_SEVERAL_EXTENSIONS_ASCII,
)