def __init__(self, msg):
def InternalAdd(field_number, wire_type, data):
unknown_field = UnknownField(field_number, wire_type, data)
self._values.append(unknown_field)
self._values = []
msg_des = msg.DESCRIPTOR
# pylint: disable=protected-access
unknown_fields = msg._unknown_fields
if (msg_des.has_options
and msg_des.GetOptions().message_set_wire_format):
local_decoder = decoder.UnknownMessageSetItemDecoder()
for _, buffer in unknown_fields:
(field_number, data) = local_decoder(memoryview(buffer))
InternalAdd(field_number,
wire_format.WIRETYPE_LENGTH_DELIMITED, data)
else:
for tag_bytes, buffer in unknown_fields:
# pylint: disable=protected-access
(tag, _) = decoder._DecodeVarint(tag_bytes, 0)
field_number, wire_type = wire_format.UnpackTag(tag)
if field_number == 0:
raise RuntimeError('Field number 0 is illegal.')
(data,
_) = decoder._DecodeUnknownField(memoryview(buffer), 0,
wire_type)
InternalAdd(field_number, wire_type, data)
def testUnpackTag(self):
# Test field numbers that will require various varint sizes.
for expected_field_number in (1, 15, 16, 2047, 2048):
for expected_wire_type in range(6): # Highest-numbered wiretype is 5.
field_number, wire_type = wire_format.UnpackTag(
wire_format.PackTag(expected_field_number, expected_wire_type))
self.assertEqual(expected_field_number, field_number)
self.assertEqual(expected_wire_type, wire_type)
self.assertRaises(TypeError, wire_format.UnpackTag, None)
self.assertRaises(TypeError, wire_format.UnpackTag, 'abc')
self.assertRaises(TypeError, wire_format.UnpackTag, 0.0)
self.assertRaises(TypeError, wire_format.UnpackTag, object())
def _DecodeUnknownFieldSet(buffer, pos, end_pos=None):
"""Decode UnknownFieldSet. Returns the UnknownFieldSet and new position."""
unknown_field_set = containers.UnknownFieldSet()
while end_pos is None or pos < end_pos:
(tag_bytes, pos) = ReadTag(buffer, pos)
(tag, _) = _DecodeVarint(tag_bytes, 0)
field_number, wire_type = wire_format.UnpackTag(tag)
if wire_type == wire_format.WIRETYPE_END_GROUP:
break
(data, pos) = _DecodeUnknownField(buffer, pos, wire_type)
# pylint: disable=protected-access
unknown_field_set._add(field_number, wire_type, data)
return (unknown_field_set, pos)
def decode_message(buf, typedef=None, pos=0, end=None, group=False):
"""Decode a protobuf message with no length delimiter"""
if end is None:
end = len(buf)
if typedef is None:
typedef = {}
else:
# Don't want to accidentally modify the original
typedef = copy.deepcopy(typedef)
output = {}
while pos < end:
# Read in a field
tag, pos = decoder._DecodeVarint(buf, pos)
try:
field_number, wire_type = wire_format.UnpackTag(tag)
except Exception as exc:
raise (
ValueError,
'Could not read valid tag at pos %d. Ensure it is a valid protobuf message: %s'
% (pos - len(tag), exc), sys.exc_info()[2])
# Convert to str
field_number = str(field_number)
orig_field_number = field_number
field_typedef = None
if field_number in typedef:
field_typedef = typedef[field_number]
else:
field_typedef = {}
field_typedef['type'] = types.wire_type_defaults[wire_type]
field_type = field_typedef['type']
# If field_type is None, its either an unsupported wire type, length delim or group
# length delim we have to try and decode first
field_out = None
if field_type is None:
if wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED:
out, field_type = decode_guess(buf, pos)
if field_type == 'message':
field_out, message_typedef, pos = out
field_typedef['message_typedef'] = message_typedef
else:
field_out, pos = out
elif wire_type == wire_format.WIRETYPE_END_GROUP:
# TODO Should probably match the field_number to START_GROUP
if not group:
raise ValueError("Found END_GROUP before START_GROUP")
# exit out
return output, typedef, pos
else:
raise ValueError(
"Could not find default type for wiretype: %d" % wire_type)
else:
if field_type == 'message':
#TODO probably big enough to factor out
message_typedef = None
# Check for a anonymous type
if 'message_typedef' in field_typedef:
message_typedef = field_typedef['message_typedef']
# Check for type defined by message type name
elif 'message_type_name' in field_typedef:
message_typedef = types.messages[
field_typedef['message_type_name']]
try:
field_out, message_typedef, pos = decode_lendelim_message(
buf, message_typedef, pos)
# Save type definition
field_typedef['message_typedef'] = message_typedef
except Exception as exc:
# If this is the root message just fail
if pos == 0:
raise exc
if field_out is None and 'alt_typedefs' in field_typedef:
# check for an alternative type definition
for alt_field_number, alt_typedef in field_typedef[
'alt_typedefs'].items():
try:
field_out, message_typedef, pos = decode_lendelim_message(
buf, alt_typedef, pos)
except Exception as exc:
pass
if field_out is not None:
# Found working typedef
field_typedef['alt_typedefs'][
alt_field_number] = message_typedef
field_number = field_number + "-" + alt_field_number
break
if field_out is None:
# Still no typedef, try anonymous, and let the error propogate if it fails
field_out, message_typedef, pos = decode_lendelim_message(
buf, {}, pos)
if 'alt_typedefs' in field_typedef:
# get the next higher alt field number
alt_field_number = str(
max(map(int, field_typedef['alt_typedefs'].keys()))
+ 1)
else:
field_typedef['alt_typedefs'] = {}
alt_field_number = '1'
field_typedef['alt_typedefs'][
alt_field_number] = message_typedef
field_number = field_number + "-" + alt_field_number
elif field_type == 'group':
group_typedef = None
# Check for a anonymous type
if 'group_typedef' in field_typedef:
group_typedef = field_typedef['group_typedef']
field_out, group_typedef, pos = decode_group(
buf, group_typedef, pos)
# Save type definition
field_typedef['group_typedef'] = group_typedef
else:
# Verify wiretype matches
if types.wiretypes[field_type] != wire_type:
raise ValueError(
"Invalid wiretype for field number %s. %s is not wiretype %s"
% (field_number, field_type, wire_type))
# Simple type, just look up the decoder
field_out, pos = types.decoders[field_type](buf, pos)
field_typedef['type'] = field_type
if 'name' not in field_typedef:
field_typedef['name'] = ''
field_key = field_number
if '-' not in field_number and 'name' in field_typedef and field_typedef[
'name'] != '':
field_key = field_typedef['name']
# Deal with repeats
if field_key in output:
if isinstance(field_out, list):
if isinstance(output[field_number], list):
output[field_key] += field_out
else:
output[field_key] = field_out.append(output[field_key])
else:
if isinstance(output[field_number], list):
output[field_key].append(field_out)
else:
output[field_key] = [output[field_key], field_out]
else:
output[field_key] = field_out
typedef[orig_field_number] = field_typedef
if pos > end:
raise decoder._DecodeError("Invalid Message Length")
# Should never hit here as a group
if group:
raise ValueError("Got START_GROUP with no END_GROUP.")
return output, typedef, pos
def ReadFieldNumberAndWireType(self):
"""Reads a tag from the wire. Returns a (field_number, wire_type) pair."""
tag_and_type = self.ReadUInt32()
return wire_format.UnpackTag(tag_and_type)
def _group_by_number(buf, pos, end, path):
"""Parse through the whole message and return buffers based on wire type.
This forces us to parse the whole message at once, but I think we're
doing that anyway.
Returns a dictionary like:
{
"2": (<wiretype>, [<data>])
}
"""
output_map = {}
while pos < end:
# Read in a field
try:
if six.PY2:
tag, pos = decoder._DecodeVarint(str(buf), pos)
else:
tag, pos = decoder._DecodeVarint(buf, pos)
except (IndexError, decoder._DecodeError) as exc:
six.raise_from(
DecoderException(
"Error decoding length from buffer: %r..." %
(binascii.hexlify(buf[pos:pos + 8])),
path=path,
),
exc,
)
field_number, wire_type = wire_format.UnpackTag(tag)
# We want field numbers as strings everywhere
field_number = str(field_number)
path = path[:] + [field_number]
if field_number in output_map and output_map[field_number][
0] != wire_type:
"""This should never happen"""
raise DecoderException(
"Field %s has mistmatched wiretypes. Previous: %s Now: %s" %
(field_number, output_map[field_number][0], wire_type),
path=path,
)
length = None
if wire_type == wire_format.WIRETYPE_VARINT:
# We actually have to read in the whole varint to figure out it's size
_, new_pos = varint.decode_varint(buf, pos)
length = new_pos - pos
elif wire_type == wire_format.WIRETYPE_FIXED32:
length = 4
elif wire_type == wire_format.WIRETYPE_FIXED64:
length = 8
elif wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED:
# Read the length from the start of the message
# add on the length of the length tag as well
bytes_length, new_pos = varint.decode_varint(buf, pos)
length = bytes_length + (new_pos - pos)
elif wire_type in [
wire_format.WIRETYPE_START_GROUP,
wire_format.WIRETYPE_END_GROUP,
]:
raise DecoderException("GROUP wire types not supported", path=path)
else:
raise DecoderException("Got unkown wire type: %d" % wire_type,
path=path)
if pos + length > end:
raise DecoderException(
"Decoded length for field %s goes over end: %d > %d" %
(field_number, pos + length, end),
path=path,
)
field_buf = buf[pos:pos + length]
if field_number in output_map:
output_map[field_number][1].append(field_buf)
else:
output_map[field_number] = (wire_type, [field_buf])
pos += length
return output_map, pos
def read_field_num(self):
tag, self._curr_cursor = _DecodeSignedVarint32(self._data,
self._curr_cursor)
return wire_format.UnpackTag(tag)[0]
def decode_message(buf, typedef=None, pos=0, end=None, group=False, depth=0, path=None):
"""Decode a protobuf message with no length delimiter"""
if end is None:
end = len(buf)
if typedef is None:
typedef = {}
else:
# Don't want to accidentally modify the original
typedef = copy.deepcopy(typedef)
if path is None:
path = []
output = {}
while pos < end:
# Read in a field
try:
if six.PY2:
tag, pos = decoder._DecodeVarint(str(buf), pos)
else:
tag, pos = decoder._DecodeVarint(buf, pos)
except (IndexError, decoder._DecodeError) as exc:
six.raise_from(DecoderException(
"Error decoding length from buffer: %r..." %
(binascii.hexlify(buf[pos : pos+8]))), exc)
field_number, wire_type = wire_format.UnpackTag(tag)
# Convert to str
field_number = str(field_number)
orig_field_number = field_number
field_path = path[:]
field_path.append(field_number)
if wire_type not in blackboxprotobuf.lib.types.wire_type_defaults:
raise DecoderException('%d is not a valid wire type at pos %d.' % (wire_type, pos), field_path)
field_typedef = None
if field_number in typedef:
field_typedef = typedef[field_number]
else:
field_typedef = {}
field_typedef['type'] = blackboxprotobuf.lib.types.wire_type_defaults[wire_type]
field_type = field_typedef['type']
# If field_type is None, its either an unsupported wire type, length delim or group
# length delim we have to try and decode first
field_out = None
if field_type is None:
if wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED:
out, field_type = decode_guess(buf, pos, depth=depth, path=field_path)
if field_type == 'message':
field_out, message_typedef, pos = out
field_typedef['message_typedef'] = message_typedef
else:
field_out, pos = out
elif wire_type == wire_format.WIRETYPE_END_GROUP:
# TODO Should probably match the field_number to START_GROUP
if not group:
raise DecoderException( "Found END_GROUP before START_GROUP", field_path)
# exit out
return output, typedef, pos
else:
raise DecoderException("Could not find default type for wiretype: %d" % wire_type, field_path)
else:
if field_type == 'message':
#TODO probably big enough to factor out
message_typedef = None
# Check for a anonymous type
if 'message_typedef' in field_typedef:
message_typedef = field_typedef['message_typedef']
# Check for type defined by message type name
elif 'message_type_name' in field_typedef:
message_typedef = blackboxprotobuf.lib.known_messages[
field_typedef['message_type_name']]
try:
field_out, message_typedef, pos = decode_lendelim_message(
buf, message_typedef, pos, path=field_path)
# Save type definition
field_typedef['message_typedef'] = message_typedef
except DecoderException as exc:
# If this is the root message just fail
if pos == 0:
six.reraise(*sys.exc_info())
logging.debug(
("Encountered exception when decoding message at %s "
"with known typdef. Trying alt typedefs and then "
"anonymous. Exception: \n%s"),
"->".join(map(str, field_path)), str(exc))
if field_out is None and 'alt_typedefs' in field_typedef:
# check for an alternative type definition
for alt_field_number, alt_typedef in field_typedef['alt_typedefs'].items():
try:
field_out, message_typedef, pos = decode_lendelim_message(
buf, alt_typedef, pos, path=field_path)
except DecoderException as exc:
logging.debug(
("Encountered exception when decoding message at %s with alt_typedef %s. Trying anonymous decoding next. Exception:\n%s"),
"->".join(map(str, field_path)),
str(alt_field_number),
str(exc))
if field_out is not None:
# Found working typedef
field_typedef['alt_typedefs'][alt_field_number] = message_typedef
field_number = field_number + "-" + alt_field_number
break
if field_out is None:
# Still no typedef, try anonymous, and let the error propogate if it fails
field_out, message_typedef, pos = \
decode_lendelim_message(buf, {}, pos, path=field_path)
if 'alt_typedefs' in field_typedef:
# get the next higher alt field number
alt_field_number = str(
max(map(int, field_typedef['alt_typedefs'].keys()))
+ 1)
else:
field_typedef['alt_typedefs'] = {}
alt_field_number = '1'
field_typedef['alt_typedefs'][alt_field_number] = message_typedef
field_number = field_number + "-" + alt_field_number
elif field_type == 'group':
group_typedef = None
# Check for a anonymous type
if 'group_typedef' in field_typedef:
group_typedef = field_typedef['group_typedef']
field_out, group_typedef, pos = \
decode_group(buf, group_typedef, pos, depth=depth, path=field_path)
# Save type definition
field_typedef['group_typedef'] = group_typedef
else:
# Verify wiretype matches
if blackboxprotobuf.lib.types.wiretypes[field_type] != wire_type:
raise DecoderException(
"Invalid wiretype for field number %s. %s is not wiretype %s"
% (field_number, field_type, wire_type), field_path)
# Simple type, just look up the decoder
try:
field_out, pos = blackboxprotobuf.lib.types.decoders[field_type](buf, pos)
except DecoderException as exc:
exc.set_path(field_path)
six.reraise(*sys.exc_info())
field_typedef['type'] = field_type
if 'name' not in field_typedef:
field_typedef['name'] = ''
field_key = field_number
if '-' not in field_number and 'name' in field_typedef and field_typedef['name'] != '':
field_key = field_typedef['name']
# Deal with repeats
if field_key in output:
if isinstance(field_out, list):
if isinstance(output[field_number], list):
output[field_key] += field_out
else:
output[field_key] = field_out.append(output[field_key])
else:
if isinstance(output[field_number], list):
output[field_key].append(field_out)
else:
output[field_key] = [output[field_key], field_out]
else:
output[field_key] = field_out
typedef[orig_field_number] = field_typedef
if pos > end:
raise DecoderException(
"Field sizes are greater than designated length. pos: %d end_pos: %d" % (pos, end))
# Should never hit here as a group
if group:
raise DecoderException("Got START_GROUP with no END_GROUP.")
return output, typedef, pos
