def test_none_serialization_roundtrip(self):
# Change default from JSON to None so we test the correct serializer
serialization.registry.set_default(None)
text_data = "The Quick Brown Fox Jumps Over The Lazy Dog"
content_type, content_encoding, payload = serialization.dumps(
text_data)
self.assertIsInstance(payload, bytes)
self.assertEqual(content_type, serialization.CONTENT_TYPE_TEXT)
self.assertEqual(content_encoding, "utf-8")
recovered_data = serialization.loads(payload,
content_type=content_type,
content_encoding=content_encoding)
self.assertEqual(text_data, recovered_data)
binary_data = text_data.encode()
content_type, content_encoding, payload = serialization.dumps(
binary_data)
self.assertIsInstance(payload, bytes)
self.assertEqual(content_type, serialization.CONTENT_TYPE_DATA)
self.assertEqual(content_encoding, "binary")
recovered_data = serialization.loads(payload,
content_type=content_type,
content_encoding=content_encoding)
self.assertEqual(binary_data, recovered_data)
# check exception is raised when bytes are not passed in
with self.assertRaises(Exception) as cm:
content_type, content_encoding, payload = serialization.dumps({})
self.assertIn("Can only serialize bytes", str(cm.exception))
def test_dumps_with_unspecified_name_or_type(self):
content_type, content_encoding, _payload = serialization.dumps(b"")
self.assertEqual(content_type, serialization.CONTENT_TYPE_DATA)
self.assertEqual(content_encoding, "binary")
content_type, content_encoding, _payload = serialization.dumps("")
self.assertEqual(content_type, serialization.CONTENT_TYPE_TEXT)
self.assertEqual(content_encoding, "utf-8")
test_value = {"a": "a_string", "b": 42}
content_type, content_encoding, _payload = serialization.dumps(
test_value)
self.assertEqual(content_type, serialization.CONTENT_TYPE_JSON)
self.assertEqual(content_encoding, "utf-8")
def send(self,
data: bytes,
*,
peer_id: bytes = None,
type_identifier: int = 0,
**kwargs):
""" Send a message to one or more peers.
:param data: a bytes object containing the message payload.
:param peer_id: The unique peer identity to send this message to. If
no peer_id is specified then send to all peers. For a client
endpoint, which typically has a single peer, this argument can
conveniently be left unspecified.
:param type_identifier: An optional parameter specifying the message
type identifier. If supplied this integer value will be encoded
into the message frame header.
"""
if not self._protocol:
logger.error(f"No protocol to send message with!")
return
_content_type, _content_encoding, data = serialization.dumps(
data, self.serialization_name)
if not isinstance(data, bytes):
logger.error(
f"data must be bytes - can't send message. data={data}")
return
self._protocol.send(data, type_identifier=type_identifier, **kwargs)
def test_text_serialization_roundtrip(self):
text_data = "The Quick Brown Fox Jumps Over The Lazy Dog"
content_type, content_encoding, payload = serialization.dumps(
text_data, "text")
self.assertIsInstance(payload, bytes)
self.assertEqual(content_type, serialization.CONTENT_TYPE_TEXT)
self.assertEqual(content_encoding, "utf-8")
recovered_data = serialization.loads(payload,
content_type=content_type,
content_encoding=content_encoding)
self.assertEqual(text_data, recovered_data)
def test_yaml_serialization_roundtrip(self):
yaml_data = {
"string": "The quick brown fox jumps over the lazy dog",
"int": 10,
"float": 3.14159265,
"unicode": "Thé quick brown fox jumps over thé lazy dog",
"list": ["george", "jerry", "elaine", "cosmo"],
}
content_type, content_encoding, payload = serialization.dumps(
yaml_data, "yaml")
self.assertIsInstance(payload, bytes)
self.assertEqual(content_type, serialization.CONTENT_TYPE_YAML)
self.assertEqual(content_encoding, "utf-8")
recovered_data = serialization.loads(payload,
content_type=content_type,
content_encoding=content_encoding)
self.assertEqual(yaml_data, recovered_data)
def test_protobuf_serialization_roundtrip(self):
from position_pb2 import Position
protobuf_data = Position(latitude=130.0,
longitude=-30.0,
altitude=50.0,
status=Position.SIMULATED)
serializer = serialization.registry.get_serializer("protobuf")
type_identifier = serializer.registry.register_message(
Position, type_identifier=1)
content_type, content_encoding, payload = serialization.dumps(
protobuf_data, "protobuf", type_identifier=type_identifier)
self.assertIsInstance(payload, bytes)
self.assertEqual(content_type, serialization.CONTENT_TYPE_PROTOBUF)
self.assertEqual(content_encoding, "binary")
recovered_data = serialization.loads(
payload,
content_type=content_type,
content_encoding=content_encoding,
type_identifier=type_identifier,
)
self.assertEqual(protobuf_data, recovered_data)
def encode_payload(
data: Any,
*,
content_type: str = None,
compression: str = None,
headers: dict = None,
type_identifier: int = None,
) -> Tuple[bytes, Optional[str], str]:
""" Prepare a message payload.
:param data: The message data to encode.
:param content_type: A string specifying the message content type. By
default the value is None. This field determines the data
serialization format.
:param compression: An optional string specifying the compression strategy
to use. It can be provided using the convenience name or the mime-type.
If compression is defined then headers must also be supplied as
compression is passed as an attribute in message headers.
:param headers: A dict of headers that will be associated with the
message.
:param type_identifier: An integer that uniquely identifies a
registered message.
:returns: A three-item tuple containing the serialized data as bytes
a string specifying the content type (e.g., `application/json`) and
a string specifying the content encoding, (e.g. `utf-8`).
"""
# Some content-types require additional information to be passed to
# help decode the message payload. This is achieved by adding
# information to the message headers.
# Google Protocol buffer decoders require awareness of the object type
# being decoded (referred to as a symbol). The symbol id is added to
# the headers so that it can be used on the receiving side.
if content_type == CONTENT_TYPE_PROTOBUF:
serializer = registry.get_serializer(CONTENT_TYPE_PROTOBUF)
if not isinstance(headers, dict):
raise Exception("Headers must be supplied when using protobuf")
headers["x-type-id"] = serializer.registry.get_id_for_object(data)
# Avro decoders require awareness of the schema that describes the object.
# This information is added to the headers so that it can be used on the
# receiving side.
elif content_type == CONTENT_TYPE_AVRO:
if type_identifier is None:
raise Exception("No Avro id specified!")
if not isinstance(headers, dict):
raise Exception("Headers must be supplied when using Avro")
headers["x-type-id"] = type_identifier
serialization_name = registry.type_to_name[content_type]
try:
content_type, content_encoding, payload = dumps(
data, serialization_name, type_identifier=type_identifier
)
except Exception as exc:
raise Exception(f"Error serializing payload to {content_type}: {exc}") from None
if compression:
if not isinstance(headers, dict):
raise Exception("Headers must be supplied when using compression")
try:
headers["compression"], payload = compress(payload, compression)
except Exception as exc:
raise Exception(
f"Error compressing payload using {compression}: {exc}"
) from None
return payload, content_type, content_encoding
def test_dumps_with_invalid_name_or_type(self):
with self.assertRaises(Exception) as cm:
serialization.dumps(b"", "invalid")
self.assertIn("Invalid serializer", str(cm.exception))
def test_avro_serialization_roundtrip(self):
# Add schema to serializer schema registry
message_schema = {
"namespace":
"unittest.serialization",
"type":
"record",
"name":
"Test",
"fields": [
{
"name": "string",
"type": "string"
},
{
"name": "int",
"type": ["int", "null"]
},
{
"name": "float",
"type": ["float", "null"]
},
{
"name": "unicode",
"type": ["string", "null"]
},
{
"name": "list",
"type": {
"type": "array",
"items": "string"
}
},
],
}
serializer = serialization.registry.get_serializer("avro")
type_identifier = serializer.registry.register_message(
message_schema, type_identifier=1)
avro_data = {
"string": "The quick brown fox jumps over the lazy dog",
"int": 10,
"float": 3.14159265,
"unicode": "Thé quick brown fox jumps over thé lazy dog",
"list": ["george", "jerry", "elaine", "cosmo"],
}
content_type, content_encoding, payload = serialization.dumps(
avro_data, "avro", type_identifier=type_identifier)
self.assertIsInstance(payload, bytes)
self.assertEqual(content_type, serialization.CONTENT_TYPE_AVRO)
self.assertEqual(content_encoding, "binary")
recovered_data = serialization.loads(
payload,
content_type=content_type,
content_encoding=content_encoding,
type_identifier=type_identifier,
)
self.assertEqual(avro_data["string"], recovered_data["string"])
self.assertEqual(avro_data["int"], recovered_data["int"])
self.assertAlmostEqual(avro_data["float"],
recovered_data["float"],
places=6)
self.assertEqual(avro_data["unicode"], recovered_data["unicode"])
self.assertEqual(avro_data["list"], recovered_data["list"])
