def testPackTag(self): field_number = 0xabc tag_type = 2 self.assertEqual((field_number << 3) | tag_type, wire_format.PackTag(field_number, tag_type)) PackTag = wire_format.PackTag # Number too high. self.assertRaises(message.EncodeError, PackTag, field_number, 6) # Number too low. self.assertRaises(message.EncodeError, PackTag, field_number, -1)
def testUnpackTag(self): # Test field numbers that will require various varint sizes. for expected_field_number in (1, 15, 16, 2047, 2048): for expected_wire_type in range(6): # Highest-numbered wiretype is 5. field_number, wire_type = wire_format.UnpackTag( wire_format.PackTag(expected_field_number, expected_wire_type)) self.assertEqual(expected_field_number, field_number) self.assertEqual(expected_wire_type, wire_type) self.assertRaises(TypeError, wire_format.UnpackTag, None) self.assertRaises(TypeError, wire_format.UnpackTag, 'abc') self.assertRaises(TypeError, wire_format.UnpackTag, 0.0) self.assertRaises(TypeError, wire_format.UnpackTag, object())
def testReadGroupInto_WrongFieldNumberInEndGroupTag(self): expected_field_number = 10 field_number = expected_field_number + 1 num_bytes = 5 d = self.ReadGroupInto_FailureTestHelper(bytes_read=num_bytes) self.mock_stream.SkipBytes(num_bytes) # Wrong field number, right wire type. self.mock_stream.ReadVarUInt32().AndReturn( wire_format.PackTag(field_number, wire_format.WIRETYPE_END_GROUP)) self.mox.ReplayAll() self.assertRaises(message.DecodeError, d.ReadGroupInto, expected_field_number, self.mock_message) self.mox.VerifyAll()
def testReadGroupInto_NoEndGroupTag(self): field_number = expected_field_number = 10 num_bytes = 5 d = self.ReadGroupInto_FailureTestHelper(bytes_read=num_bytes) self.mock_stream.SkipBytes(num_bytes) # Right field number, wrong wire type. self.mock_stream.ReadVarUInt32().AndReturn( wire_format.PackTag(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)) self.mox.ReplayAll() self.assertRaises(message.DecodeError, d.ReadGroupInto, expected_field_number, self.mock_message) self.mox.VerifyAll()
def testReadGroupInto_Success(self): # Test both the empty and nonempty cases. for num_bytes in (5, 0): field_number = expected_field_number = 10 d = decoder.Decoder('') d._stream = self.mock_stream sub_buffer = object() self.mock_stream.GetSubBuffer().AndReturn(sub_buffer) self.mock_message.MergeFromString(sub_buffer).AndReturn(num_bytes) self.mock_stream.SkipBytes(num_bytes) self.mock_stream.ReadVarUInt32().AndReturn(wire_format.PackTag( field_number, wire_format.WIRETYPE_END_GROUP)) self.mox.ReplayAll() d.ReadGroupInto(expected_field_number, self.mock_message) self.mox.VerifyAll() self.mox.ResetAll()
def TagBytes(field_number, wire_type): """Encode the given tag and return the bytes. Only called at startup.""" return _VarintBytes(wire_format.PackTag(field_number, wire_type))
def _TagSize(field_number): """Returns the number of bytes required to serialize a tag with this field number.""" # Just pass in type 0, since the type won't affect the tag+type size. return _VarintSize(wire_format.PackTag(field_number, 0))
def _MergeField(self, tokenizer, message): """Merges a single protocol message field into a message. Args: tokenizer: A tokenizer to parse the field name and values. message: A protocol message to record the data. Raises: ParseError: In case of text parsing problems. """ message_descriptor = message.DESCRIPTOR if (message_descriptor.full_name == text_format._ANY_FULL_TYPE_NAME and tokenizer.TryConsume('[')): type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer) tokenizer.Consume(']') tokenizer.TryConsume(':') if tokenizer.TryConsume('<'): expanded_any_end_token = '>' else: tokenizer.Consume('{') expanded_any_end_token = '}' expanded_any_sub_message = text_format._BuildMessageFromTypeName( packed_type_name, self.descriptor_pool) if not expanded_any_sub_message: raise text_format.ParseError( 'Type %s not found in descriptor pool' % packed_type_name) while not tokenizer.TryConsume(expanded_any_end_token): if tokenizer.AtEnd(): raise tokenizer.ParseErrorPreviousToken( 'Expected "%s".' % (expanded_any_end_token, )) self._MergeField(tokenizer, expanded_any_sub_message) message.Pack(expanded_any_sub_message, type_url_prefix=type_url_prefix) return if tokenizer.TryConsume('['): name = [tokenizer.ConsumeIdentifier()] while tokenizer.TryConsume('.'): name.append(tokenizer.ConsumeIdentifier()) name = '.'.join(name) if not message_descriptor.is_extendable: raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" does not have extensions.' % message_descriptor.full_name) # pylint: disable=protected-access field = message.Extensions._FindExtensionByName(name) # pylint: enable=protected-access if not field: if self.allow_unknown_extension: field = None else: raise tokenizer.ParseErrorPreviousToken( 'Extension "%s" not registered. ' 'Did you import the _pb2 module which defines it? ' 'If you are trying to place the extension in the MessageSet ' 'field of another message that is in an Any or MessageSet field, ' 'that message\'s _pb2 module must be imported as well' % name) elif message_descriptor != field.containing_type: raise tokenizer.ParseErrorPreviousToken( 'Extension "%s" does not extend message type "%s".' % (name, message_descriptor.full_name)) tokenizer.Consume(']') else: name = tokenizer.ConsumeIdentifierOrNumber() if self.allow_field_number and name.isdigit(): number = text_format.ParseInteger(name, True, True) field = message_descriptor.fields_by_number.get(number, None) if not field and message_descriptor.is_extendable: field = message.Extensions._FindExtensionByNumber(number) else: field = message_descriptor.fields_by_name.get(name, None) # Group names are expected to be capitalized as they appear in the # .proto file, which actually matches their type names, not their field # names. if not field: field = message_descriptor.fields_by_name.get( name.lower(), None) if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP: field = None if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and field.message_type.name != name): field = None if not field and not self.allow_unknown_field: raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" has no field named "%s".' % (message_descriptor.full_name, name)) if field: if not self._allow_multiple_scalars and field.containing_oneof: # Check if there's a different field set in this oneof. # Note that we ignore the case if the same field was set before, and we # apply _allow_multiple_scalars to non-scalar fields as well. which_oneof = message.WhichOneof(field.containing_oneof.name) if which_oneof is not None and which_oneof != field.name: raise tokenizer.ParseErrorPreviousToken( 'Field "%s" is specified along with field "%s", another member ' 'of oneof "%s" for message type "%s".' % (field.name, which_oneof, field.containing_oneof.name, message_descriptor.full_name)) if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: tokenizer.TryConsume(':') merger = self._MergeMessageField else: tokenizer.Consume(':') merger = self._MergeScalarField if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and tokenizer.TryConsume('[')): # Short repeated format, e.g. "foo: [1, 2, 3]" if not tokenizer.TryConsume(']'): while True: merger(tokenizer, message, field) if tokenizer.TryConsume(']'): break tokenizer.Consume(',') else: merger(tokenizer, message, field) else: # Proto field is unknown. assert (self.allow_unknown_extension or self.allow_unknown_field) if not name.isdigit(): # Got a field name we don't recognize text_format._SkipFieldContents(tokenizer) else: # We have an int for a name, which means an unknown field we can parse. field_number = int(name) if message._unknown_fields == (): message._unknown_fields = [] if tokenizer.TryConsume('('): # Has a defined wire type -- varint, fixed32 or fixed64. # We are going to treat the fixed types as floating-point numbers, # since that's what they usually are. wire_type = { 'f64': wire_format.WIRETYPE_FIXED64, 'f32': wire_format.WIRETYPE_FIXED32, 'int': wire_format.WIRETYPE_VARINT }[tokenizer.ConsumeIdentifier()] tokenizer.Consume(')') tokenizer.Consume(':') if wire_type == wire_format.WIRETYPE_VARINT: data = tokenizer.ConsumeInteger() field_bytes = encode_varint(data) elif wire_type == wire_format.WIRETYPE_FIXED32: field_bytes = struct.pack('<f', tokenizer.ConsumeFloat()) data = struct.unpack('<I', field_bytes) else: field_bytes = struct.pack('<d', tokenizer.ConsumeFloat()) data = struct.unpack('<Q', field_bytes) tag_bytes = struct.pack( 'B', wire_format.PackTag(field_number, wire_type)) message.UnknownFields()._add(field_number, wire_type, data) message._unknown_fields.append((tag_bytes, field_bytes)) elif tokenizer.TryConsume(':'): # String tag_bytes = struct.pack( 'B', wire_format.PackTag(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)) string = tokenizer.ConsumeString() string_encoded = string.encode('utf-8') message.UnknownFields()._add( field_number, wire_format.WIRETYPE_LENGTH_DELIMITED, string_encoded) message._unknown_fields.append( (tag_bytes, encode_varint(len(string_encoded)) + string_encoded)) elif tokenizer.TryConsume('<'): # Group group = Empty() while not tokenizer.TryConsume('>'): self._MergeField(tokenizer, group) message.UnknownFields()._add(field_number, wire_format.WIRETYPE_START_GROUP, group.UnknownFields()) tag_start = struct.pack( 'B', wire_format.PackTag(field_number, wire_format.WIRETYPE_START_GROUP)) tag_end = struct.pack( 'B', wire_format.PackTag(field_number, wire_format.WIRETYPE_END_GROUP)) message._unknown_fields.append( (tag_start, group.SerializeToString() + tag_end)) elif tokenizer.TryConsume('{'): # Message nested_message = Empty() while not tokenizer.TryConsume('}'): self._MergeField(tokenizer, nested_message) serialized = nested_message.SerializeToString() message.UnknownFields()._add( field_number, wire_format.WIRETYPE_LENGTH_DELIMITED, serialized) tag_bytes = struct.pack( 'B', wire_format.PackTag(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)) message._unknown_fields.append( (tag_bytes, encode_varint(len(serialized)) + serialized)) else: raise Exception( 'Encountered unexpected field data, should not happen')
def PackTag(self, field_number, wire_type): return wire_format.PackTag(field_number, wire_type)
def _TagSize(field_number): return _VarintSize(wire_format.PackTag(field_number, 0))
def TagBytes(field_number, wire_type): return _VarintBytes(wire_format.PackTag(field_number, wire_type))
def _AppendTag(self, field_number, wire_type): """Appends a tag containing field number and wire type information.""" self._stream.AppendVarUInt32( wire_format.PackTag(field_number, wire_type))