def __init__(self, msg): def InternalAdd(field_number, wire_type, data): unknown_field = UnknownField(field_number, wire_type, data) self._values.append(unknown_field) self._values = [] msg_des = msg.DESCRIPTOR # pylint: disable=protected-access unknown_fields = msg._unknown_fields if (msg_des.has_options and msg_des.GetOptions().message_set_wire_format): local_decoder = decoder.UnknownMessageSetItemDecoder() for _, buffer in unknown_fields: (field_number, data) = local_decoder(memoryview(buffer)) InternalAdd(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED, data) else: for tag_bytes, buffer in unknown_fields: # pylint: disable=protected-access (tag, _) = decoder._DecodeVarint(tag_bytes, 0) field_number, wire_type = wire_format.UnpackTag(tag) if field_number == 0: raise RuntimeError('Field number 0 is illegal.') (data, _) = decoder._DecodeUnknownField(memoryview(buffer), 0, wire_type) InternalAdd(field_number, wire_type, data)
def testUnpackTag(self): # Test field numbers that will require various varint sizes. for expected_field_number in (1, 15, 16, 2047, 2048): for expected_wire_type in range(6): # Highest-numbered wiretype is 5. field_number, wire_type = wire_format.UnpackTag( wire_format.PackTag(expected_field_number, expected_wire_type)) self.assertEqual(expected_field_number, field_number) self.assertEqual(expected_wire_type, wire_type) self.assertRaises(TypeError, wire_format.UnpackTag, None) self.assertRaises(TypeError, wire_format.UnpackTag, 'abc') self.assertRaises(TypeError, wire_format.UnpackTag, 0.0) self.assertRaises(TypeError, wire_format.UnpackTag, object())
def _DecodeUnknownFieldSet(buffer, pos, end_pos=None): """Decode UnknownFieldSet. Returns the UnknownFieldSet and new position.""" unknown_field_set = containers.UnknownFieldSet() while end_pos is None or pos < end_pos: (tag_bytes, pos) = ReadTag(buffer, pos) (tag, _) = _DecodeVarint(tag_bytes, 0) field_number, wire_type = wire_format.UnpackTag(tag) if wire_type == wire_format.WIRETYPE_END_GROUP: break (data, pos) = _DecodeUnknownField(buffer, pos, wire_type) # pylint: disable=protected-access unknown_field_set._add(field_number, wire_type, data) return (unknown_field_set, pos)
def decode_message(buf, typedef=None, pos=0, end=None, group=False): """Decode a protobuf message with no length delimiter""" if end is None: end = len(buf) if typedef is None: typedef = {} else: # Don't want to accidentally modify the original typedef = copy.deepcopy(typedef) output = {} while pos < end: # Read in a field tag, pos = decoder._DecodeVarint(buf, pos) try: field_number, wire_type = wire_format.UnpackTag(tag) except Exception as exc: raise ( ValueError, 'Could not read valid tag at pos %d. Ensure it is a valid protobuf message: %s' % (pos - len(tag), exc), sys.exc_info()[2]) # Convert to str field_number = str(field_number) orig_field_number = field_number field_typedef = None if field_number in typedef: field_typedef = typedef[field_number] else: field_typedef = {} field_typedef['type'] = types.wire_type_defaults[wire_type] field_type = field_typedef['type'] # If field_type is None, its either an unsupported wire type, length delim or group # length delim we have to try and decode first field_out = None if field_type is None: if wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED: out, field_type = decode_guess(buf, pos) if field_type == 'message': field_out, message_typedef, pos = out field_typedef['message_typedef'] = message_typedef else: field_out, pos = out elif wire_type == wire_format.WIRETYPE_END_GROUP: # TODO Should probably match the field_number to START_GROUP if not group: raise ValueError("Found END_GROUP before START_GROUP") # exit out return output, typedef, pos else: raise ValueError( "Could not find default type for wiretype: %d" % wire_type) else: if field_type == 'message': #TODO probably big enough to factor out message_typedef = None # Check for a anonymous type if 'message_typedef' in field_typedef: message_typedef = field_typedef['message_typedef'] # Check for type defined by message type name elif 'message_type_name' in field_typedef: message_typedef = types.messages[ field_typedef['message_type_name']] try: field_out, message_typedef, pos = decode_lendelim_message( buf, message_typedef, pos) # Save type definition field_typedef['message_typedef'] = message_typedef except Exception as exc: # If this is the root message just fail if pos == 0: raise exc if field_out is None and 'alt_typedefs' in field_typedef: # check for an alternative type definition for alt_field_number, alt_typedef in field_typedef[ 'alt_typedefs'].items(): try: field_out, message_typedef, pos = decode_lendelim_message( buf, alt_typedef, pos) except Exception as exc: pass if field_out is not None: # Found working typedef field_typedef['alt_typedefs'][ alt_field_number] = message_typedef field_number = field_number + "-" + alt_field_number break if field_out is None: # Still no typedef, try anonymous, and let the error propogate if it fails field_out, message_typedef, pos = decode_lendelim_message( buf, {}, pos) if 'alt_typedefs' in field_typedef: # get the next higher alt field number alt_field_number = str( max(map(int, field_typedef['alt_typedefs'].keys())) + 1) else: field_typedef['alt_typedefs'] = {} alt_field_number = '1' field_typedef['alt_typedefs'][ alt_field_number] = message_typedef field_number = field_number + "-" + alt_field_number elif field_type == 'group': group_typedef = None # Check for a anonymous type if 'group_typedef' in field_typedef: group_typedef = field_typedef['group_typedef'] field_out, group_typedef, pos = decode_group( buf, group_typedef, pos) # Save type definition field_typedef['group_typedef'] = group_typedef else: # Verify wiretype matches if types.wiretypes[field_type] != wire_type: raise ValueError( "Invalid wiretype for field number %s. %s is not wiretype %s" % (field_number, field_type, wire_type)) # Simple type, just look up the decoder field_out, pos = types.decoders[field_type](buf, pos) field_typedef['type'] = field_type if 'name' not in field_typedef: field_typedef['name'] = '' field_key = field_number if '-' not in field_number and 'name' in field_typedef and field_typedef[ 'name'] != '': field_key = field_typedef['name'] # Deal with repeats if field_key in output: if isinstance(field_out, list): if isinstance(output[field_number], list): output[field_key] += field_out else: output[field_key] = field_out.append(output[field_key]) else: if isinstance(output[field_number], list): output[field_key].append(field_out) else: output[field_key] = [output[field_key], field_out] else: output[field_key] = field_out typedef[orig_field_number] = field_typedef if pos > end: raise decoder._DecodeError("Invalid Message Length") # Should never hit here as a group if group: raise ValueError("Got START_GROUP with no END_GROUP.") return output, typedef, pos
def ReadFieldNumberAndWireType(self): """Reads a tag from the wire. Returns a (field_number, wire_type) pair.""" tag_and_type = self.ReadUInt32() return wire_format.UnpackTag(tag_and_type)
def _group_by_number(buf, pos, end, path): """Parse through the whole message and return buffers based on wire type. This forces us to parse the whole message at once, but I think we're doing that anyway. Returns a dictionary like: { "2": (<wiretype>, [<data>]) } """ output_map = {} while pos < end: # Read in a field try: if six.PY2: tag, pos = decoder._DecodeVarint(str(buf), pos) else: tag, pos = decoder._DecodeVarint(buf, pos) except (IndexError, decoder._DecodeError) as exc: six.raise_from( DecoderException( "Error decoding length from buffer: %r..." % (binascii.hexlify(buf[pos:pos + 8])), path=path, ), exc, ) field_number, wire_type = wire_format.UnpackTag(tag) # We want field numbers as strings everywhere field_number = str(field_number) path = path[:] + [field_number] if field_number in output_map and output_map[field_number][ 0] != wire_type: """This should never happen""" raise DecoderException( "Field %s has mistmatched wiretypes. Previous: %s Now: %s" % (field_number, output_map[field_number][0], wire_type), path=path, ) length = None if wire_type == wire_format.WIRETYPE_VARINT: # We actually have to read in the whole varint to figure out it's size _, new_pos = varint.decode_varint(buf, pos) length = new_pos - pos elif wire_type == wire_format.WIRETYPE_FIXED32: length = 4 elif wire_type == wire_format.WIRETYPE_FIXED64: length = 8 elif wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED: # Read the length from the start of the message # add on the length of the length tag as well bytes_length, new_pos = varint.decode_varint(buf, pos) length = bytes_length + (new_pos - pos) elif wire_type in [ wire_format.WIRETYPE_START_GROUP, wire_format.WIRETYPE_END_GROUP, ]: raise DecoderException("GROUP wire types not supported", path=path) else: raise DecoderException("Got unkown wire type: %d" % wire_type, path=path) if pos + length > end: raise DecoderException( "Decoded length for field %s goes over end: %d > %d" % (field_number, pos + length, end), path=path, ) field_buf = buf[pos:pos + length] if field_number in output_map: output_map[field_number][1].append(field_buf) else: output_map[field_number] = (wire_type, [field_buf]) pos += length return output_map, pos
def read_field_num(self): tag, self._curr_cursor = _DecodeSignedVarint32(self._data, self._curr_cursor) return wire_format.UnpackTag(tag)[0]
def decode_message(buf, typedef=None, pos=0, end=None, group=False, depth=0, path=None): """Decode a protobuf message with no length delimiter""" if end is None: end = len(buf) if typedef is None: typedef = {} else: # Don't want to accidentally modify the original typedef = copy.deepcopy(typedef) if path is None: path = [] output = {} while pos < end: # Read in a field try: if six.PY2: tag, pos = decoder._DecodeVarint(str(buf), pos) else: tag, pos = decoder._DecodeVarint(buf, pos) except (IndexError, decoder._DecodeError) as exc: six.raise_from(DecoderException( "Error decoding length from buffer: %r..." % (binascii.hexlify(buf[pos : pos+8]))), exc) field_number, wire_type = wire_format.UnpackTag(tag) # Convert to str field_number = str(field_number) orig_field_number = field_number field_path = path[:] field_path.append(field_number) if wire_type not in blackboxprotobuf.lib.types.wire_type_defaults: raise DecoderException('%d is not a valid wire type at pos %d.' % (wire_type, pos), field_path) field_typedef = None if field_number in typedef: field_typedef = typedef[field_number] else: field_typedef = {} field_typedef['type'] = blackboxprotobuf.lib.types.wire_type_defaults[wire_type] field_type = field_typedef['type'] # If field_type is None, its either an unsupported wire type, length delim or group # length delim we have to try and decode first field_out = None if field_type is None: if wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED: out, field_type = decode_guess(buf, pos, depth=depth, path=field_path) if field_type == 'message': field_out, message_typedef, pos = out field_typedef['message_typedef'] = message_typedef else: field_out, pos = out elif wire_type == wire_format.WIRETYPE_END_GROUP: # TODO Should probably match the field_number to START_GROUP if not group: raise DecoderException( "Found END_GROUP before START_GROUP", field_path) # exit out return output, typedef, pos else: raise DecoderException("Could not find default type for wiretype: %d" % wire_type, field_path) else: if field_type == 'message': #TODO probably big enough to factor out message_typedef = None # Check for a anonymous type if 'message_typedef' in field_typedef: message_typedef = field_typedef['message_typedef'] # Check for type defined by message type name elif 'message_type_name' in field_typedef: message_typedef = blackboxprotobuf.lib.known_messages[ field_typedef['message_type_name']] try: field_out, message_typedef, pos = decode_lendelim_message( buf, message_typedef, pos, path=field_path) # Save type definition field_typedef['message_typedef'] = message_typedef except DecoderException as exc: # If this is the root message just fail if pos == 0: six.reraise(*sys.exc_info()) logging.debug( ("Encountered exception when decoding message at %s " "with known typdef. Trying alt typedefs and then " "anonymous. Exception: \n%s"), "->".join(map(str, field_path)), str(exc)) if field_out is None and 'alt_typedefs' in field_typedef: # check for an alternative type definition for alt_field_number, alt_typedef in field_typedef['alt_typedefs'].items(): try: field_out, message_typedef, pos = decode_lendelim_message( buf, alt_typedef, pos, path=field_path) except DecoderException as exc: logging.debug( ("Encountered exception when decoding message at %s with alt_typedef %s. Trying anonymous decoding next. Exception:\n%s"), "->".join(map(str, field_path)), str(alt_field_number), str(exc)) if field_out is not None: # Found working typedef field_typedef['alt_typedefs'][alt_field_number] = message_typedef field_number = field_number + "-" + alt_field_number break if field_out is None: # Still no typedef, try anonymous, and let the error propogate if it fails field_out, message_typedef, pos = \ decode_lendelim_message(buf, {}, pos, path=field_path) if 'alt_typedefs' in field_typedef: # get the next higher alt field number alt_field_number = str( max(map(int, field_typedef['alt_typedefs'].keys())) + 1) else: field_typedef['alt_typedefs'] = {} alt_field_number = '1' field_typedef['alt_typedefs'][alt_field_number] = message_typedef field_number = field_number + "-" + alt_field_number elif field_type == 'group': group_typedef = None # Check for a anonymous type if 'group_typedef' in field_typedef: group_typedef = field_typedef['group_typedef'] field_out, group_typedef, pos = \ decode_group(buf, group_typedef, pos, depth=depth, path=field_path) # Save type definition field_typedef['group_typedef'] = group_typedef else: # Verify wiretype matches if blackboxprotobuf.lib.types.wiretypes[field_type] != wire_type: raise DecoderException( "Invalid wiretype for field number %s. %s is not wiretype %s" % (field_number, field_type, wire_type), field_path) # Simple type, just look up the decoder try: field_out, pos = blackboxprotobuf.lib.types.decoders[field_type](buf, pos) except DecoderException as exc: exc.set_path(field_path) six.reraise(*sys.exc_info()) field_typedef['type'] = field_type if 'name' not in field_typedef: field_typedef['name'] = '' field_key = field_number if '-' not in field_number and 'name' in field_typedef and field_typedef['name'] != '': field_key = field_typedef['name'] # Deal with repeats if field_key in output: if isinstance(field_out, list): if isinstance(output[field_number], list): output[field_key] += field_out else: output[field_key] = field_out.append(output[field_key]) else: if isinstance(output[field_number], list): output[field_key].append(field_out) else: output[field_key] = [output[field_key], field_out] else: output[field_key] = field_out typedef[orig_field_number] = field_typedef if pos > end: raise DecoderException( "Field sizes are greater than designated length. pos: %d end_pos: %d" % (pos, end)) # Should never hit here as a group if group: raise DecoderException("Got START_GROUP with no END_GROUP.") return output, typedef, pos