def test_none_serialization_roundtrip(self): # Change default from JSON to None so we test the correct serializer serialization.registry.set_default(None) text_data = "The Quick Brown Fox Jumps Over The Lazy Dog" content_type, content_encoding, payload = serialization.dumps( text_data) self.assertIsInstance(payload, bytes) self.assertEqual(content_type, serialization.CONTENT_TYPE_TEXT) self.assertEqual(content_encoding, "utf-8") recovered_data = serialization.loads(payload, content_type=content_type, content_encoding=content_encoding) self.assertEqual(text_data, recovered_data) binary_data = text_data.encode() content_type, content_encoding, payload = serialization.dumps( binary_data) self.assertIsInstance(payload, bytes) self.assertEqual(content_type, serialization.CONTENT_TYPE_DATA) self.assertEqual(content_encoding, "binary") recovered_data = serialization.loads(payload, content_type=content_type, content_encoding=content_encoding) self.assertEqual(binary_data, recovered_data) # check exception is raised when bytes are not passed in with self.assertRaises(Exception) as cm: content_type, content_encoding, payload = serialization.dumps({}) self.assertIn("Can only serialize bytes", str(cm.exception))
def test_dumps_with_unspecified_name_or_type(self): content_type, content_encoding, _payload = serialization.dumps(b"") self.assertEqual(content_type, serialization.CONTENT_TYPE_DATA) self.assertEqual(content_encoding, "binary") content_type, content_encoding, _payload = serialization.dumps("") self.assertEqual(content_type, serialization.CONTENT_TYPE_TEXT) self.assertEqual(content_encoding, "utf-8") test_value = {"a": "a_string", "b": 42} content_type, content_encoding, _payload = serialization.dumps( test_value) self.assertEqual(content_type, serialization.CONTENT_TYPE_JSON) self.assertEqual(content_encoding, "utf-8")
def send(self, data: bytes, *, peer_id: bytes = None, type_identifier: int = 0, **kwargs): """ Send a message to one or more peers. :param data: a bytes object containing the message payload. :param peer_id: The unique peer identity to send this message to. If no peer_id is specified then send to all peers. For a client endpoint, which typically has a single peer, this argument can conveniently be left unspecified. :param type_identifier: An optional parameter specifying the message type identifier. If supplied this integer value will be encoded into the message frame header. """ if not self._protocol: logger.error(f"No protocol to send message with!") return _content_type, _content_encoding, data = serialization.dumps( data, self.serialization_name) if not isinstance(data, bytes): logger.error( f"data must be bytes - can't send message. data={data}") return self._protocol.send(data, type_identifier=type_identifier, **kwargs)
def test_text_serialization_roundtrip(self): text_data = "The Quick Brown Fox Jumps Over The Lazy Dog" content_type, content_encoding, payload = serialization.dumps( text_data, "text") self.assertIsInstance(payload, bytes) self.assertEqual(content_type, serialization.CONTENT_TYPE_TEXT) self.assertEqual(content_encoding, "utf-8") recovered_data = serialization.loads(payload, content_type=content_type, content_encoding=content_encoding) self.assertEqual(text_data, recovered_data)
def test_yaml_serialization_roundtrip(self): yaml_data = { "string": "The quick brown fox jumps over the lazy dog", "int": 10, "float": 3.14159265, "unicode": "Thé quick brown fox jumps over thé lazy dog", "list": ["george", "jerry", "elaine", "cosmo"], } content_type, content_encoding, payload = serialization.dumps( yaml_data, "yaml") self.assertIsInstance(payload, bytes) self.assertEqual(content_type, serialization.CONTENT_TYPE_YAML) self.assertEqual(content_encoding, "utf-8") recovered_data = serialization.loads(payload, content_type=content_type, content_encoding=content_encoding) self.assertEqual(yaml_data, recovered_data)
def test_protobuf_serialization_roundtrip(self): from position_pb2 import Position protobuf_data = Position(latitude=130.0, longitude=-30.0, altitude=50.0, status=Position.SIMULATED) serializer = serialization.registry.get_serializer("protobuf") type_identifier = serializer.registry.register_message( Position, type_identifier=1) content_type, content_encoding, payload = serialization.dumps( protobuf_data, "protobuf", type_identifier=type_identifier) self.assertIsInstance(payload, bytes) self.assertEqual(content_type, serialization.CONTENT_TYPE_PROTOBUF) self.assertEqual(content_encoding, "binary") recovered_data = serialization.loads( payload, content_type=content_type, content_encoding=content_encoding, type_identifier=type_identifier, ) self.assertEqual(protobuf_data, recovered_data)
def encode_payload( data: Any, *, content_type: str = None, compression: str = None, headers: dict = None, type_identifier: int = None, ) -> Tuple[bytes, Optional[str], str]: """ Prepare a message payload. :param data: The message data to encode. :param content_type: A string specifying the message content type. By default the value is None. This field determines the data serialization format. :param compression: An optional string specifying the compression strategy to use. It can be provided using the convenience name or the mime-type. If compression is defined then headers must also be supplied as compression is passed as an attribute in message headers. :param headers: A dict of headers that will be associated with the message. :param type_identifier: An integer that uniquely identifies a registered message. :returns: A three-item tuple containing the serialized data as bytes a string specifying the content type (e.g., `application/json`) and a string specifying the content encoding, (e.g. `utf-8`). """ # Some content-types require additional information to be passed to # help decode the message payload. This is achieved by adding # information to the message headers. # Google Protocol buffer decoders require awareness of the object type # being decoded (referred to as a symbol). The symbol id is added to # the headers so that it can be used on the receiving side. if content_type == CONTENT_TYPE_PROTOBUF: serializer = registry.get_serializer(CONTENT_TYPE_PROTOBUF) if not isinstance(headers, dict): raise Exception("Headers must be supplied when using protobuf") headers["x-type-id"] = serializer.registry.get_id_for_object(data) # Avro decoders require awareness of the schema that describes the object. # This information is added to the headers so that it can be used on the # receiving side. elif content_type == CONTENT_TYPE_AVRO: if type_identifier is None: raise Exception("No Avro id specified!") if not isinstance(headers, dict): raise Exception("Headers must be supplied when using Avro") headers["x-type-id"] = type_identifier serialization_name = registry.type_to_name[content_type] try: content_type, content_encoding, payload = dumps( data, serialization_name, type_identifier=type_identifier ) except Exception as exc: raise Exception(f"Error serializing payload to {content_type}: {exc}") from None if compression: if not isinstance(headers, dict): raise Exception("Headers must be supplied when using compression") try: headers["compression"], payload = compress(payload, compression) except Exception as exc: raise Exception( f"Error compressing payload using {compression}: {exc}" ) from None return payload, content_type, content_encoding
def test_dumps_with_invalid_name_or_type(self): with self.assertRaises(Exception) as cm: serialization.dumps(b"", "invalid") self.assertIn("Invalid serializer", str(cm.exception))
def test_avro_serialization_roundtrip(self): # Add schema to serializer schema registry message_schema = { "namespace": "unittest.serialization", "type": "record", "name": "Test", "fields": [ { "name": "string", "type": "string" }, { "name": "int", "type": ["int", "null"] }, { "name": "float", "type": ["float", "null"] }, { "name": "unicode", "type": ["string", "null"] }, { "name": "list", "type": { "type": "array", "items": "string" } }, ], } serializer = serialization.registry.get_serializer("avro") type_identifier = serializer.registry.register_message( message_schema, type_identifier=1) avro_data = { "string": "The quick brown fox jumps over the lazy dog", "int": 10, "float": 3.14159265, "unicode": "Thé quick brown fox jumps over thé lazy dog", "list": ["george", "jerry", "elaine", "cosmo"], } content_type, content_encoding, payload = serialization.dumps( avro_data, "avro", type_identifier=type_identifier) self.assertIsInstance(payload, bytes) self.assertEqual(content_type, serialization.CONTENT_TYPE_AVRO) self.assertEqual(content_encoding, "binary") recovered_data = serialization.loads( payload, content_type=content_type, content_encoding=content_encoding, type_identifier=type_identifier, ) self.assertEqual(avro_data["string"], recovered_data["string"]) self.assertEqual(avro_data["int"], recovered_data["int"]) self.assertAlmostEqual(avro_data["float"], recovered_data["float"], places=6) self.assertEqual(avro_data["unicode"], recovered_data["unicode"]) self.assertEqual(avro_data["list"], recovered_data["list"])