def data_received(self, data):
"""asyncio.Protocol data received callback. Decodes JSON data into
Swirl messages and dispatches them to the handle() coroutine.
Args:
data (bytes): Data received.
"""
self._buf += data
while True:
# Check if buf is long enough for length field
if len(self._buf) < 2:
break
# Read length field
length = int.from_bytes(self._buf[0:2], 'big')
# Check if buf is long enough for message
if len(self._buf) < 2+length:
break
# Extract serialized message
message_data = self._buf[2:2+length]
self._buf = self._buf[2+length:]
# Deserialize message
message = swirl_pb3.SwirlServerMessage()
try:
message.ParseFromString(message_data)
except google.protobuf.message.DecodeError as e:
logger.debug("[swirl_client] Error decoding server message: %s", str(e))
return
self._loop.create_task(self.handle(message))
def validate_reaction(message_name):
"""Receives a serialized Reaction protobuf and runs validations."""
message = message_helpers.create_message(message_name)
message.ParseFromString(flask.request.get_data())
options = validations.ValidationOptions(require_provenance=True)
output = validations.validate_message(message,
raise_on_error=False,
options=options)
return json.dumps({'errors': output.errors, 'warnings': output.warnings})
def deserialize(cls, data):
message = openxc_pb2.VehicleMessage()
try:
message.ParseFromString(data)
except google.protobuf.message.DecodeError as e:
pass
except UnicodeDecodeError as e:
LOG.warn("Unable to parse protobuf: %s", e)
else:
return cls._protobuf_to_dict(message)
def testNotANumberPacked(self, message_module):
golden_data = (b'\xA2\x06\x04\x00\x00\xC0\x7F'
b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF8\x7F')
golden_message = message_module.TestPackedTypes()
golden_message.ParseFromString(golden_data)
self.assertTrue(isnan(golden_message.packed_float[0]))
self.assertTrue(isnan(golden_message.packed_double[0]))
serialized = golden_message.SerializeToString()
message = message_module.TestPackedTypes()
message.ParseFromString(serialized)
self.assertTrue(isnan(message.packed_float[0]))
self.assertTrue(isnan(message.packed_double[0]))
def deserialize(self, data):
message = vonxc_pb2.RadioVehicleMessage()
print "vonxc returnvalue ", binascii.hexlify(bytearray(data))
try:
message.ParseFromString(data)
except google.protobuf.message.DecodeError as e:
print "decode error"
pass
except UnicodeDecodeError as e:
print("Unable to parse protobuf: %s", e)
#print "decode error"
else:
return message
def validate_reaction(message_name):
"""Receives a serialized Reaction protobuf and runs validations."""
message = message_helpers.create_message(message_name)
message.ParseFromString(flask.request.get_data())
if message == type(message)():
# Do not try to validate empty messages.
return json.dumps({"errors": [], "warnings": []})
options = validations.ValidationOptions(require_provenance=True)
output = validations.validate_message(message,
raise_on_error=False,
options=options)
errors = list(map(_adjust_error, output.errors))
warnings = list(map(_adjust_error, output.warnings))
return json.dumps({"errors": errors, "warnings": warnings})
def testExtremeDoubleValues(self, message_module):
message = message_module.TestAllTypes()
# Most positive exponent, no significand bits set.
kMostPosExponentNoSigBits = math.pow(2, 1023)
message.optional_double = kMostPosExponentNoSigBits
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == kMostPosExponentNoSigBits)
# Most positive exponent, one significand bit set.
kMostPosExponentOneSigBit = 1.5 * math.pow(2, 1023)
message.optional_double = kMostPosExponentOneSigBit
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == kMostPosExponentOneSigBit)
# Repeat last two cases with values of same magnitude, but negative.
message.optional_double = -kMostPosExponentNoSigBits
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == -kMostPosExponentNoSigBits)
message.optional_double = -kMostPosExponentOneSigBit
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == -kMostPosExponentOneSigBit)
# Most negative exponent, no significand bits set.
kMostNegExponentNoSigBits = math.pow(2, -1023)
message.optional_double = kMostNegExponentNoSigBits
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == kMostNegExponentNoSigBits)
# Most negative exponent, one significand bit set.
kMostNegExponentOneSigBit = 1.5 * math.pow(2, -1023)
message.optional_double = kMostNegExponentOneSigBit
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == kMostNegExponentOneSigBit)
# Repeat last two cases with values of the same magnitude, but negative.
message.optional_double = -kMostNegExponentNoSigBits
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == -kMostNegExponentNoSigBits)
message.optional_double = -kMostNegExponentOneSigBit
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == -kMostNegExponentOneSigBit)
def testExtremeFloatValues(self):
message = unittest_pb2.TestAllTypes()
# Most positive exponent, no significand bits set.
kMostPosExponentNoSigBits = math.pow(2, 127)
message.optional_float = kMostPosExponentNoSigBits
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_float == kMostPosExponentNoSigBits)
# Most positive exponent, one significand bit set.
kMostPosExponentOneSigBit = 1.5 * math.pow(2, 127)
message.optional_float = kMostPosExponentOneSigBit
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_float == kMostPosExponentOneSigBit)
# Repeat last two cases with values of same magnitude, but negative.
message.optional_float = -kMostPosExponentNoSigBits
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_float == -kMostPosExponentNoSigBits)
message.optional_float = -kMostPosExponentOneSigBit
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_float == -kMostPosExponentOneSigBit)
# Most negative exponent, no significand bits set.
kMostNegExponentNoSigBits = math.pow(2, -127)
message.optional_float = kMostNegExponentNoSigBits
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_float == kMostNegExponentNoSigBits)
# Most negative exponent, one significand bit set.
kMostNegExponentOneSigBit = 1.5 * math.pow(2, -127)
message.optional_float = kMostNegExponentOneSigBit
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_float == kMostNegExponentOneSigBit)
# Repeat last two cases with values of the same magnitude, but negative.
message.optional_float = -kMostNegExponentNoSigBits
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_float == -kMostNegExponentNoSigBits)
message.optional_float = -kMostNegExponentOneSigBit
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_float == -kMostNegExponentOneSigBit)
def handle_stream(self, stream, address):
logging.debug("Opened a stream from %s", address)
while True:
try:
message_bytes = yield packet.read_packet_from_stream(stream)
message = Urination()
message.ParseFromString(message_bytes)
logging.debug("Server received message from %s:\n%s", address,
message)
except tornado.iostream.StreamClosedError:
logging.info("Closed stream from %s", address)
return
except google.protobuf.message.DecodeError:
logging.warning(
"Broken message read from %s "
"terminating connection!", address)
stream.close()
return
def _parse_protobuf_message(self, message_buffer):
parsed_message = None
remainder = message_buffer
message_data = ""
# 1. decode a varint from the top of the stream
# 2. using that as the length, if there's enough in the buffer, try and
# decode try and decode a VehicleMessage after the varint
# 3. if it worked, great, we're oriented in the stream - continue
# 4. if either couldn't be parsed, skip to the next byte and repeat
while parsed_message is None and len(message_buffer) > 1:
message_length, message_start = _DecodeVarint(message_buffer, 0)
# sanity check to make sure we didn't parse some huge number that's
# clearly not the length prefix
if message_length > self.MAX_PROTOBUF_MESSAGE_LENGTH:
message_buffer = message_buffer[1:]
continue
if message_start + message_length >= len(message_buffer):
break
message_data = message_buffer[message_start:message_start +
message_length]
remainder = message_buffer[message_start + message_length:]
message = openxc_pb2.VehicleMessage()
try:
message.ParseFromString(message_data)
except google.protobuf.message.DecodeError as e:
message_buffer = message_buffer[1:]
except UnicodeDecodeError as e:
LOG.warn("Unable to parse protobuf: %s", e)
else:
parsed_message = self._protobuf_to_dict(message)
if parsed_message is None:
message_buffer = message_buffer[1:]
bytes_received = 0
if parsed_message is not None:
bytes_received = len(message_data)
return parsed_message, remainder, bytes_received
def testNotANumber(self, message_module):
golden_data = (b'\x5D\x00\x00\xC0\x7F'
b'\x61\x00\x00\x00\x00\x00\x00\xF8\x7F'
b'\xCD\x02\x00\x00\xC0\x7F'
b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF8\x7F')
golden_message = message_module.TestAllTypes()
golden_message.ParseFromString(golden_data)
self.assertTrue(isnan(golden_message.optional_float))
self.assertTrue(isnan(golden_message.optional_double))
self.assertTrue(isnan(golden_message.repeated_float[0]))
self.assertTrue(isnan(golden_message.repeated_double[0]))
# The protocol buffer may serialize to any one of multiple different
# representations of a NaN. Rather than verify a specific representation,
# verify the serialized string can be converted into a correctly
# behaving protocol buffer.
serialized = golden_message.SerializeToString()
message = message_module.TestAllTypes()
message.ParseFromString(serialized)
self.assertTrue(isnan(message.optional_float))
self.assertTrue(isnan(message.optional_double))
self.assertTrue(isnan(message.repeated_float[0]))
self.assertTrue(isnan(message.repeated_double[0]))
def FromFile(path: pathlib.Path,
message: ProtocolBuffer,
assume_filename: typing.Optional[typing.Union[
str, pathlib.Path]] = None,
uninitialized_okay: bool = False) -> ProtocolBuffer:
"""Read a protocol buffer from a file.
This method uses attempts to guess the encoding from the path suffix,
supporting binary, text, and json formatted messages. The mapping of suffixes
to formatting is, in order:
*.txt.gz: Gzipped text.
*.txt: Text.
*.pbtxt.gz: Gzipped text.
*.pbtxt: Text.
*.json.gz: Gzipped JSON.
*.json: JSON.
*.gz: Gzipped encoded string.
*: Encoded string.
Args:
path: Path to the proto file.
message: A message instance to read into.
assume_filename: For the purpose of determining the encoding from the file
extension, use this name rather than the true path.
uninitialized_okay: If True, do not require that decoded messages be
initialized. If False, DecodeError is raised.
Returns:
The parsed message (same as the message argument).
Raises:
FileNotFoundError: If the path does not exist.
IsADirectoryError: If the path is a directory.
DecodeError: If the file cannot be decoded to the given message type, or if
after decoding, the message is not initialized and uninitialized_okay is
False.
"""
if not path.is_file():
if path.is_dir():
raise IsADirectoryError(f"Path is a directory: '{path}'")
else:
raise FileNotFoundError(f"File not found: '{path}'")
suffixes = pathlib.Path(
assume_filename).suffixes if assume_filename else path.suffixes
if suffixes and suffixes[-1] == '.gz':
suffixes.pop()
open_function = gzip.open
else:
open_function = open
suffix = suffixes[-1] if suffixes else ''
try:
with open_function(path, 'rb') as f:
if suffix == '.txt' or suffix == '.pbtxt':
# Allow uninitialized fields here because we will catch the error later,
# allowing us to report the path of the proto.
FromString(f.read().decode('utf-8'),
message,
uninitialized_okay=True)
elif suffix == '.json':
google.protobuf.json_format.Parse(f.read(), message)
else:
message.ParseFromString(f.read())
except (google.protobuf.text_format.ParseError,
google.protobuf.json_format.ParseError) as e:
# The exception raised during parsing depends on the message format. Catch
# them all under a single DecodeError exception type.
raise DecodeError(e)
if not uninitialized_okay and not message.IsInitialized():
raise DecodeError(f"Required fields not set: '{path}'")
return message