def MergeFromString(self, serialized):
length = len(serialized)
try:
if self._InternalParse(serialized, 0, length) != length:
# The only reason _InternalParse would return early is if it
# encountered an end-group tag.
raise message_mod.DecodeError('Unexpected end-group tag.')
except IndexError:
raise message_mod.DecodeError('Truncated message.')
except struct.error, e:
raise message_mod.DecodeError(e)
def MergeFromString(self, serialized):
length = len(serialized)
try:
if self._InternalParse(serialized, 0, length) != length:
# The only reason _InternalParse would return early is if it
# encountered an end-group tag.
raise message_mod.DecodeError('Unexpected end-group tag.')
except (IndexError, TypeError):
# Now ord(buf[p:p+1]) == ord('') gets TypeError.
raise message_mod.DecodeError('Truncated message.')
except struct.error as e:
raise message_mod.DecodeError(e)
return length # Return this for legacy reasons.
def ReadBytes(self, size):
if size < 0:
raise message.DecodeError('Negative size %d' % size)
s = self._buffer[self._pos:self._pos + size].tostring()
self._pos += len(
s) # Only advance by the number of bytes actually read.
return s
def GetSubBuffer(self, size=None):
if size is None:
return self._buffer[self._pos:].tostring()
else:
if size < 0:
raise message.DecodeError('Negative size %d' % size)
return self._buffer[self._pos:self._pos + size].tostring()
def GetSubBuffer(self, size=None):
"""Returns a sequence-like object that represents a portion of our
underlying sequence.
Position 0 in the returned object corresponds to self.Position()
in this stream.
If size is specified, then the returned object ends after the
next "size" bytes in this stream. If size is not specified,
then the returned object ends at the end of this stream.
We guarantee that the returned object R supports the Python buffer
interface (and thus that the call buffer(R) will work).
Note that the returned buffer is read-only.
The intended use for this method is for nested-message and nested-group
deserialization, where we want to make a recursive MergeFromString()
call on the portion of the original sequence that contains the serialized
nested message. (And we'd like to do so without making unnecessary string
copies).
REQUIRES: size is nonnegative.
"""
# Note that buffer() doesn't perform any actual string copy.
if size is None:
return self._buffer[self._pos:]
else:
if size < 0:
raise message.DecodeError('Negative size %d' % size)
return self._buffer[self._pos:self._pos + size]
def _ReadVarintHelper(self):
result = 0
shift = 0
while 1:
if shift >= 64:
raise message.DecodeError(
'Too many bytes when decoding varint.')
try:
b = self._buffer[self._pos]
except IndexError:
raise message.DecodeError('Truncated varint.')
self._pos += 1
result |= ((b & 0x7f) << shift)
shift += 7
if not (b & 0x80):
return result
def Get8(self):
"""Get the next 8 bits from the buffer."""
if self.idx >= self.limit:
raise message.DecodeError('truncated')
c = self.buf[self.idx]
self.idx += 1
return c
def ReadLittleEndian64(self):
try:
i = struct.unpack(wire_format.FORMAT_UINT64_LITTLE_ENDIAN,
self._buffer[self._pos:self._pos + 8])
self._pos += 8
return i[0] # unpack() result is a 1-element tuple.
except struct.error, e:
raise message.DecodeError(e)
def ReadVarUInt32(self):
"""Reads a varint from the stream, interprets this varint
as an unsigned, 32-bit integer, and returns the integer.
"""
i = self.ReadVarUInt64()
if i > wire_format.UINT32_MAX:
raise message.DecodeError('Value out of range for uint32: %d' % i)
return i
def ReadVarUInt64(self):
"""Reads a varint from the stream, interprets this varint
as an unsigned, 64-bit integer, and returns the integer.
"""
i = self._ReadVarintHelper()
if not 0 <= i <= wire_format.UINT64_MAX:
raise message.DecodeError('Value out of range for uint64: %d' % i)
return i
def Get16(self):
"""Get the next 16 bits from the buffer."""
if self.idx + 2 > self.limit:
raise message.DecodeError('truncated')
c = self.buf[self.idx]
d = self.buf[self.idx + 1]
self.idx += 2
return (c << 8) | d
def SkipBytes(self, num_bytes):
"""Skip num_bytes bytes ahead, or go to the end of the stream, whichever
comes first.
REQUIRES: num_bytes is nonnegative.
"""
if num_bytes < 0:
raise message.DecodeError('Negative num_bytes %d' % num_bytes)
self._pos += num_bytes
self._pos = min(self._pos, len(self._buffer))
def Get32(self):
"""Get the next 32 bits from the buffer."""
if self.idx + 4 > self.limit:
raise message.DecodeError('truncated')
c = int(self.buf[self.idx])
d = self.buf[self.idx + 1]
e = self.buf[self.idx + 2]
f = self.buf[self.idx + 3]
self.idx += 4
return (c << 24) | (d << 16) | (e << 8) | f
def ReadBytes(self, size):
"""Reads up to 'size' bytes from the stream, stopping early
only if we reach the end of the stream. Returns the bytes read
as a string.
"""
if size < 0:
raise message.DecodeError('Negative size %d' % size)
s = (self._buffer[self._pos:self._pos + size])
self._pos += len(
s) # Only advance by the number of bytes actually read.
return s
def ReadLittleEndian64(self):
"""Interprets the next 8 bytes of the stream as a little-endian
encoded, unsiged 64-bit integer, and returns that integer.
"""
try:
i = struct.unpack(wire_format.FORMAT_UINT64_LITTLE_ENDIAN,
self._buffer[self._pos:self._pos + 8])
self._pos += 8
return i[0] # unpack() result is a 1-element tuple.
except struct.error, e:
raise message.DecodeError(e)
def GetFloat(self):
"""Decode the next bits as a floating point value."""
if self.idx + 4 > self.limit:
raise message.DecodeError('truncated')
a = self.buf[self.idx:self.idx + 4]
self.idx += 4
if a[0] & 0x80:
a[0] ^= 0x80
else:
a = [x ^ 0xFF for x in a]
return struct.unpack('>f', ToBytes(array.array('B', a)))[0]
def GetDouble(self):
"""Decode the next bits as a double value."""
if self.idx + 8 > self.limit:
raise message.DecodeError('truncated')
a = self.buf[self.idx:self.idx + 8]
self.idx += 8
if a[0] & 0x80:
a[0] ^= 0x80
else:
a = [x ^ 0xFF for x in a]
return struct.unpack('>d', ToBytes(array.array('B', a)))[0]
def read_tensor_summary(path):
with tf.gfile.Open(path, 'rb') as summary_file:
summary_string = summary_file.read()
if not summary_string:
raise message.DecodeError('Empty summary.')
summary_proto = summary_pb2.Summary()
summary_proto.ParseFromString(summary_string)
tensor_proto = summary_proto.value[0].tensor
array = tensor_util.make_ndarray(tensor_proto)
return array
def _ReadVarintHelper(self):
"""Helper for the various varint-reading methods above.
Reads an unsigned, varint-encoded integer from the stream and
returns this integer.
Does no bounds checking except to ensure that we read at most as many bytes
as could possibly be present in a varint-encoded 64-bit number.
"""
result = 0
shift = 0
while 1:
if shift >= 64:
raise message.DecodeError(
'Too many bytes when decoding varint.')
try:
b = self._buffer[self._pos]
except IndexError:
raise message.DecodeError('Truncated varint.')
self._pos += 1
result |= ((b & 0x7f) << shift)
shift += 7
if not (b & 0x80):
return result
def Get64(self):
"""Get the next 64 bits from the buffer."""
if self.idx + 8 > self.limit:
raise message.DecodeError('truncated')
c = int(self.buf[self.idx])
d = int(self.buf[self.idx + 1])
e = int(self.buf[self.idx + 2])
f = int(self.buf[self.idx + 3])
g = int(self.buf[self.idx + 4])
h = self.buf[self.idx + 5]
i = self.buf[self.idx + 6]
j = self.buf[self.idx + 7]
self.idx += 8
return ((c << 56) | (d << 48) | (e << 40) | (f << 32) | (g << 24)
| (h << 16) | (i << 8) | j)
def ReadMessageInto(self, msg):
"""Calls msg.MergeFromString() to merge
length-delimited serialized message data into |msg|.
REQUIRES: The decoder must be positioned at the serialized "length"
prefix to a length-delmiited serialized message.
POSTCONDITION: The decoder is positioned just after the
serialized message, and we have merged those serialized
contents into |msg|.
"""
length = self._stream.ReadVarUInt32()
sub_buffer = self._stream.GetSubBuffer(length)
num_bytes_used = msg.MergeFromString(sub_buffer)
if num_bytes_used != length:
raise message.DecodeError(
'Submessage told to deserialize from %d-byte encoding, '
'but used only %d bytes' % (length, num_bytes_used))
self._stream.SkipBytes(num_bytes_used)
def _ConvertToUnicode(memview):
"""Convert byte to unicode."""
byte_str = memview.tobytes()
try:
value = local_unicode(byte_str, 'utf-8')
except UnicodeDecodeError as e:
# add more information to the error message and re-raise it.
e.reason = '%s in field: %s' % (e, key.full_name)
raise
if is_strict_utf8 and six.PY2 and sys.maxunicode > _UCS2_MAXUNICODE:
# Only do the check for python2 ucs4 when is_strict_utf8 enabled
if _SURROGATE_PATTERN.search(value):
reason = ('String field %s contains invalid UTF-8 data when parsing'
'a protocol buffer: surrogates not allowed. Use'
'the bytes type if you intend to send raw bytes.') % (
key.full_name)
raise message.DecodeError(reason)
return value
def ReadVarint32(self):
i = self.ReadVarint64()
if not wire_format.INT32_MIN <= i <= wire_format.INT32_MAX:
raise message.DecodeError('Value out of range for int32: %d' % i)
return int(i)
def GetVarInt32(self):
"""Decode the next bits as a varint and check it's a 32 bit value."""
result = self.GetVarInt64()
if result >= 0x80000000 or result < -0x80000000:
raise message.DecodeError('corrupted')
return result
def GetVarUint64(self):
"""Decode the next bits as a varint and check it's non-negative value."""
result = self.GetVarInt64()
if result < 0:
raise message.DecodeError('corrupted')
return result
def GetBoolean(self):
"""Decode the next bits as a boolean value."""
b = self.Get8()
if b != 0 and b != 1:
raise message.DecodeError('corrupted')
return b
def SkipBytes(self, num_bytes):
if num_bytes < 0:
raise message.DecodeError('Negative num_bytes %d' % num_bytes)
self._pos += num_bytes
self._pos = min(self._pos, len(self._buffer))
def _Skip(self, n):
if self.idx + n > self.limit:
raise message.DecodeError('truncated')
self.idx += n
return
def ReadVarUInt32(self):
i = self.ReadVarUInt64()
if i > wire_format.UINT32_MAX:
raise message.DecodeError('Value out of range for uint32: %d' % i)
return i
def ReadVarUInt64(self):
i = self._ReadVarintHelper()
if not 0 <= i <= wire_format.UINT64_MAX:
raise message.DecodeError('Value out of range for uint64: %d' % i)
return i
