def decode_string(value, pos):
"""Decode varint for length and then the bytes"""
length, pos = varint.decode_varint(value, pos)
end = pos+length
try:
return value[pos:end].decode('utf-8', 'backslashreplace'), end
except TypeError as exc:
six.raise_from(DecoderException("Error decoding UTF-8 string %s" % value[pos:end]), exc)
def decode_bytes(value, pos):
"""Decode varint for the length and then returns that number of bytes"""
length, pos = varint.decode_varint(value, pos)
end = pos+length
try:
return value[pos:end], end
except IndexError as exc:
six.raise_from(DecoderException(
("Error decoding bytes. Decoded length %d is longer than bytes"
" available %d") % (length, len(value)-pos)), exc)
def length_wrapper(buf, pos):
"""Decode repeat values prefixed with the length"""
length, pos = varint.decode_varint(buf, pos)
end = pos+length
output = []
while pos < end:
value, pos = wrapped_decoder(buf, pos)
output.append(value)
if pos > end:
raise decoder._DecodeError("Invalid Packed Field Length")
return output, pos
def decode_string(value, pos):
"""Decode varint for length and then the bytes"""
length, pos = varint.decode_varint(value, pos)
end = pos + length
try:
# backslash escaping isn't reversible easily
return value[pos:end].decode("utf-8"), end
except (TypeError, UnicodeDecodeError) as exc:
six.raise_from(
DecoderException("Error decoding UTF-8 string %s" %
value[pos:end]), exc)
def length_wrapper(buf, pos):
"""Decode repeat values prefixed with the length"""
length, pos = varint.decode_varint(buf, pos)
end = pos + length
output = []
while pos < end:
value, pos = wrapped_decoder(buf, pos)
output.append(value)
if pos > end:
raise DecoderException(
("Error decoding packed field. Packed length larger than"
" buffer: decoded = %d, left = %d") %
(length, len(buf) - pos))
return output, pos
def decode_lendelim_message(buf,
config,
typedef=None,
pos=0,
depth=0,
path=None):
"""Read in the length and use it as the end"""
length, pos = varint.decode_varint(buf, pos)
ret = decode_message(buf,
config,
typedef,
pos,
pos + length,
depth=depth,
path=path)
return ret
def test_decode_varint(buf, pos):
try:
varint.decode_varint(buf, pos)
except BlackboxProtobufException as exc:
assert not isinstance(exc, EncoderException)
pass
def decode_lendelim_message(buf, typedef=None, pos=0):
"""Read in the length and use it as the end"""
length, pos = varint.decode_varint(buf, pos)
ret = decode_message(buf, typedef, pos, pos+length)
return ret
def decode_bytes(value, pos):
"""Decode varint for length and then the bytes"""
length, pos = varint.decode_varint(value, pos)
end = pos+length
return value[pos:end], end
def decode_str(value, pos):
"""Decode varint for length and then the string"""
length, pos = varint.decode_varint(value, pos)
end = pos + length
return value[pos:end].decode('utf-8', 'backslashreplace'), end
def test_varint_inverse(x):
encoded = varint.encode_varint(x)
decoded,pos = varint.decode_varint(encoded,0)
assert pos == len(encoded)
assert decoded == x
def _group_by_number(buf, pos, end, path):
"""Parse through the whole message and return buffers based on wire type.
This forces us to parse the whole message at once, but I think we're
doing that anyway.
Returns a dictionary like:
{
"2": (<wiretype>, [<data>])
}
"""
output_map = {}
while pos < end:
# Read in a field
try:
if six.PY2:
tag, pos = decoder._DecodeVarint(str(buf), pos)
else:
tag, pos = decoder._DecodeVarint(buf, pos)
except (IndexError, decoder._DecodeError) as exc:
six.raise_from(
DecoderException(
"Error decoding length from buffer: %r..." %
(binascii.hexlify(buf[pos:pos + 8])),
path=path,
),
exc,
)
field_number, wire_type = wire_format.UnpackTag(tag)
# We want field numbers as strings everywhere
field_number = str(field_number)
path = path[:] + [field_number]
if field_number in output_map and output_map[field_number][
0] != wire_type:
"""This should never happen"""
raise DecoderException(
"Field %s has mistmatched wiretypes. Previous: %s Now: %s" %
(field_number, output_map[field_number][0], wire_type),
path=path,
)
length = None
if wire_type == wire_format.WIRETYPE_VARINT:
# We actually have to read in the whole varint to figure out it's size
_, new_pos = varint.decode_varint(buf, pos)
length = new_pos - pos
elif wire_type == wire_format.WIRETYPE_FIXED32:
length = 4
elif wire_type == wire_format.WIRETYPE_FIXED64:
length = 8
elif wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED:
# Read the length from the start of the message
# add on the length of the length tag as well
bytes_length, new_pos = varint.decode_varint(buf, pos)
length = bytes_length + (new_pos - pos)
elif wire_type in [
wire_format.WIRETYPE_START_GROUP,
wire_format.WIRETYPE_END_GROUP,
]:
raise DecoderException("GROUP wire types not supported", path=path)
else:
raise DecoderException("Got unkown wire type: %d" % wire_type,
path=path)
if pos + length > end:
raise DecoderException(
"Decoded length for field %s goes over end: %d > %d" %
(field_number, pos + length, end),
path=path,
)
field_buf = buf[pos:pos + length]
if field_number in output_map:
output_map[field_number][1].append(field_buf)
else:
output_map[field_number] = (wire_type, [field_buf])
pos += length
return output_map, pos
def decode_bytes(value, pos):
"""Decode varint for the length and then returns that number of bytes"""
length, pos = varint.decode_varint(value, pos)
end = pos + length
return value[pos:end], end
