コード例 #1
0
    def test_basic_sr_avro_encoder_decode_readers_schema(self, **kwargs):
        schemaregistry_fully_qualified_namespace = kwargs.pop("schemaregistry_fully_qualified_namespace")
        sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace)
        schemaregistry_group = kwargs.pop("schemaregistry_group")
        sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register=True)

        schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}"""

        dict_content = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"}
        encoded_message_content = sr_avro_encoder.encode(dict_content, schema=schema_str)
        content_type = encoded_message_content["content_type"]
        encoded_content = encoded_message_content["content"]

        # readers_schema with removed field
        readers_schema_remove_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]}]}"""
        encoded_content_dict = {"content": encoded_content, "content_type": content_type}
        decoded_content = sr_avro_encoder.decode(encoded_content_dict, readers_schema=readers_schema_remove_field)
        assert decoded_content["name"] == u"Ben"
        assert decoded_content["favorite_number"] == 7

        # readers_schema with extra field with default
        readers_schema_extra_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}, {"name":"favorite_city","type":["string","null"], "default": "Redmond"}]}"""
        decoded_content = sr_avro_encoder.decode(encoded_content_dict, readers_schema=readers_schema_extra_field)
        assert decoded_content["name"] == u"Ben"
        assert decoded_content["favorite_number"] == 7
        assert decoded_content["favorite_color"] == "red"
        assert decoded_content["favorite_city"] == "Redmond"

        # readers_schema with changed name results in error
        readers_schema_change_name = """{"namespace":"fakeexample.avro","type":"record","name":"fake_user","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}"""
        with pytest.raises(InvalidSchemaError) as e:
            decoded_content = sr_avro_encoder.decode(encoded_content_dict, readers_schema=readers_schema_change_name)
        assert "Incompatible schemas" in e.value.message
        assert "schema_id" in e.value.details
        assert "schema_definition" in e.value.details

        # invalid readers_schema
        invalid_schema = {
            "name":"User",
            "type":"record",
            "namespace":"example.avro",
            "fields":[{"name":"name","type":"string"}]
        }
        invalid_schema_string = "{}".format(invalid_schema)
        with pytest.raises(InvalidSchemaError) as e:
            decoded_content = sr_avro_encoder.decode(encoded_content_dict, readers_schema=invalid_schema_string)
        assert "Invalid schema" in e.value.message
        assert "schema_id" in e.value.details
        assert "schema_definition" in e.value.details
コード例 #2
0
    def test_encode_record(self, **kwargs):
        schemaregistry_fully_qualified_namespace = kwargs.pop("schemaregistry_fully_qualified_namespace")
        schemaregistry_group = kwargs.pop("schemaregistry_group")
        sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace)
        sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register=True)

        # add below to schema fields later if needed
        # {"name":"example.innerrec","type":"record","fields":[{"name":"a","type":"int"}]},
        # {"name":"innerenum","type":"enum","symbols":["FOO", "BAR"]},
        # {"name":"innerarray","type":"array","items":"int"},
        # {"name":"innermap","type":"map","values":"int"},
        # {"name":"innerfixed","type":"fixed","size":74}
        schema_record = """{
            "name":"User",
            "namespace":"example.avro.populatedrecord",
            "type":"record",
            "fields":[
                {"name":"name","type":"string"},
                {"name":"age","type":"int"},
                {"name":"married","type":"boolean"},
                {"name":"height","type":"float"},
                {"name":"randb","type":"bytes"}
            ]
        }"""
        content = {
            "name": u"Ben",
            "age": 3,
            "married": False,
            "height": 13.5,
            "randb": b"\u00FF"
        }

        encoded_message_content = sr_avro_encoder.encode(content, schema=schema_record)
        decoded_content = sr_avro_encoder.decode(encoded_message_content)
        assert decoded_content == content 
コード例 #3
0
    def test_basic_sr_avro_encoder_without_auto_register_schemas(self, **kwargs):
        schemaregistry_fully_qualified_namespace = kwargs.pop("schemaregistry_fully_qualified_namespace")
        schemaregistry_group = kwargs.pop("schemaregistry_group")
        sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace)
        sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group)

        schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}"""
        schema = avro.schema.parse(schema_str)

        dict_content = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"}
        encoded_message_content = sr_avro_encoder.encode(dict_content, schema=schema_str)
        content_type = encoded_message_content["content_type"]
        encoded_content = encoded_message_content["content"]

        assert content_type.split("+")[0] == 'avro/binary'
        schema_id = sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro").id
        assert content_type.split("+")[1] == schema_id

        encoded_content_dict = {"content": encoded_content, "content_type": content_type}
        decoded_content = sr_avro_encoder.decode(encoded_content_dict)
        assert decoded_content["name"] == u"Ben"
        assert decoded_content["favorite_number"] == 7
        assert decoded_content["favorite_color"] == u"red"

        sr_avro_encoder.close()
コード例 #4
0
    def test_basic_sr_avro_encoder_decode_readers_schema(
            self, schemaregistry_fully_qualified_namespace,
            schemaregistry_group, **kwargs):
        sr_client = self.create_basic_client(
            SchemaRegistryClient,
            fully_qualified_namespace=schemaregistry_fully_qualified_namespace)
        sr_avro_encoder = AvroEncoder(client=sr_client,
                                      group_name=schemaregistry_group,
                                      auto_register_schemas=True)

        schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}"""

        dict_data = {
            "name": u"Ben",
            "favorite_number": 7,
            "favorite_color": u"red"
        }
        encoded_metadata = sr_avro_encoder.encode(dict_data, schema=schema_str)
        content_type = encoded_metadata["content_type"]
        encoded_data = encoded_metadata["data"]

        # readers_schema with removed field
        readers_schema_remove_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]}]}"""
        encoded_data_dict = {
            "data": encoded_data,
            "content_type": content_type
        }
        decoded_data = sr_avro_encoder.decode(
            encoded_data_dict, readers_schema=readers_schema_remove_field)
        assert decoded_data["name"] == u"Ben"
        assert decoded_data["favorite_number"] == 7

        # readers_schema with extra field with default
        readers_schema_extra_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}, {"name":"favorite_city","type":["string","null"], "default": "Redmond"}]}"""
        decoded_data = sr_avro_encoder.decode(
            encoded_data_dict, readers_schema=readers_schema_extra_field)
        assert decoded_data["name"] == u"Ben"
        assert decoded_data["favorite_number"] == 7
        assert decoded_data["favorite_color"] == "red"
        assert decoded_data["favorite_city"] == "Redmond"

        # readers_schema with changed name results in error
        readers_schema_change_name = """{"namespace":"fakeexample.avro","type":"record","name":"fake_user","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}"""
        with pytest.raises(SchemaDecodeError):
            decoded_data = sr_avro_encoder.decode(
                encoded_data_dict, readers_schema=readers_schema_change_name)
コード例 #5
0
    def test_basic_sr_avro_encoder_with_request_options(self, **kwargs):
        schemaregistry_fully_qualified_namespace = kwargs.pop("schemaregistry_fully_qualified_namespace")
        schemaregistry_group = kwargs.pop("schemaregistry_group")
        sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace)
        sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register=True)

        schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}"""

        dict_content = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"}
        with pytest.raises(TypeError) as e:
            encoded_message_content = sr_avro_encoder.encode(dict_content, schema=schema_str, request_options={"fake_kwarg": True})
        assert 'request() got an unexpected keyword' in str(e.value)
        encoded_message_content = sr_avro_encoder.encode(dict_content, schema=schema_str)
        content_type = encoded_message_content["content_type"]
        encoded_content = encoded_message_content["content"]

        encoded_content_dict = {"content": encoded_content, "content_type": content_type}
        with pytest.raises(TypeError) as e:
            decoded_content = sr_avro_encoder.decode(encoded_content_dict, request_options={"fake_kwarg": True})
        assert 'request() got an unexpected keyword' in str(e.value)
コード例 #6
0
    def test_basic_sr_avro_encoder_with_auto_register_schemas(
            self, schemaregistry_fully_qualified_namespace,
            schemaregistry_group, **kwargs):
        sr_client = self.create_basic_client(
            SchemaRegistryClient,
            fully_qualified_namespace=schemaregistry_fully_qualified_namespace)
        sr_avro_encoder = AvroEncoder(client=sr_client,
                                      group_name=schemaregistry_group,
                                      auto_register_schemas=True)

        schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}"""
        schema = avro.schema.parse(schema_str)

        dict_data = {
            "name": u"Ben",
            "favorite_number": 7,
            "favorite_color": u"red"
        }
        encoded_metadata = sr_avro_encoder.encode(dict_data, schema=schema_str)
        content_type = encoded_metadata["content_type"]
        encoded_data = encoded_metadata["data"]

        assert content_type.split("+")[0] == 'avro/binary'
        schema_id = sr_client.get_schema_properties(schemaregistry_group,
                                                    schema.fullname,
                                                    str(schema), "Avro").id
        assert content_type.split("+")[1] == schema_id

        encoded_data_dict = {
            "data": encoded_data,
            "content_type": content_type
        }
        decoded_data = sr_avro_encoder.decode(encoded_data_dict)
        assert decoded_data["name"] == u"Ben"
        assert decoded_data["favorite_number"] == 7
        assert decoded_data["favorite_color"] == u"red"

        # check that AvroEncoder won't work with message types that don't follow protocols
        class BadExample:
            def __init__(self, not_data):
                self.not_data = not_data

        with pytest.raises(
                SchemaEncodeError) as e:  # caught avro SchemaParseError
            sr_avro_encoder.encode({"name": u"Ben"},
                                   schema=schema_str,
                                   message_type=BadExample)
        assert "subtype of the MessageType" in (str(e.value))

        bad_ex = BadExample('fake')
        with pytest.raises(
                SchemaDecodeError) as e:  # caught avro SchemaParseError
            sr_avro_encoder.decode(message=bad_ex)
        assert "subtype of the MessageType" in (str(e.value))

        # check that AvroEncoder will work with message types that follow protocols
        class GoodExample:
            def __init__(self, data: bytes, content_type: str, **kwargs):
                self.data = data
                self.content_type = content_type
                self.extra = kwargs.pop('extra', None)

            def __message_data__(self):
                return {"data": self.data, "content_type": self.content_type}

        def good_callback(data: bytes, content_type: str, **kwargs):
            return GoodExample(data, content_type, **kwargs)

        good_ex_obj = sr_avro_encoder.encode(dict_data,
                                             schema=schema_str,
                                             message_type=GoodExample,
                                             extra='val')
        good_ex_callback = sr_avro_encoder.encode(dict_data,
                                                  schema=schema_str,
                                                  message_type=good_callback,
                                                  extra='val')
        decoded_data_obj = sr_avro_encoder.decode(message=good_ex_obj)
        decoded_data_callback = sr_avro_encoder.decode(
            message=good_ex_callback)

        assert decoded_data_obj["name"] == u"Ben"
        assert decoded_data_obj["favorite_number"] == 7
        assert decoded_data_obj["favorite_color"] == u"red"
        assert decoded_data_callback["name"] == u"Ben"
        assert decoded_data_callback["favorite_number"] == 7
        assert decoded_data_callback["favorite_color"] == u"red"

        sr_avro_encoder.close()
コード例 #7
0
    def test_basic_sr_avro_encoder_with_auto_register_schemas(self, **kwargs):
        schemaregistry_fully_qualified_namespace = kwargs.pop("schemaregistry_fully_qualified_namespace")
        schemaregistry_group = kwargs.pop("schemaregistry_group")
        sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace)
        sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register=True)

        schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}"""
        schema = avro.schema.parse(schema_str)

        dict_content = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"}
        encoded_message_content = sr_avro_encoder.encode(dict_content, schema=schema_str)
        content_type = encoded_message_content["content_type"]
        encoded_content = encoded_message_content["content"]

        # wrong data type
        dict_content_bad = {"name": u"Ben", "favorite_number": 7, "favorite_color": 7}
        with pytest.raises(InvalidContentError) as e:
            encoded_message_content = sr_avro_encoder.encode(dict_content_bad, schema=schema_str)
        assert "schema_id" in e.value.details

        assert content_type.split("+")[0] == 'avro/binary'
        schema_id = sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro").id
        assert content_type.split("+")[1] == schema_id

        encoded_content_dict = {"content": encoded_content, "content_type": content_type}
        decoded_content = sr_avro_encoder.decode(encoded_content_dict)
        assert decoded_content["name"] == u"Ben"
        assert decoded_content["favorite_number"] == 7
        assert decoded_content["favorite_color"] == u"red"

        # bad content type
        encoded_content_dict["content_type"] = 'a+b+c'
        with pytest.raises(InvalidContentError) as e:
            decoded_content = sr_avro_encoder.decode(encoded_content_dict)

        # check that AvroEncoder won't work with message types that don't follow protocols
        class BadExample:
            def __init__(self, not_content):
                self.not_content = not_content

        with pytest.raises(TypeError) as e:
            sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_str, message_type=BadExample) 
        assert "subtype of the MessageType" in (str(e.value))

        bad_ex = BadExample('fake')
        with pytest.raises(TypeError) as e:    # caught TypeError
            sr_avro_encoder.decode(message=bad_ex) 
        assert "subtype of the MessageType" in (str(e.value))

        # check that AvroEncoder will work with message types that follow protocols
        class GoodExample:
            def __init__(self, content, **kwargs):
                self.content = content
                self.content_type = None
                self.extra = kwargs.pop('extra', None)
            
            @classmethod
            def from_message_content(cls, content: bytes, content_type: str, **kwargs):
                ge = cls(content)
                ge.content_type = content_type
                return ge

            def __message_content__(self):
                return {"content": self.content, "content_type": self.content_type}

        good_ex_obj = sr_avro_encoder.encode(dict_content, schema=schema_str, message_type=GoodExample, extra='val') 
        decoded_content_obj = sr_avro_encoder.decode(message=good_ex_obj)

        assert decoded_content_obj["name"] == u"Ben"
        assert decoded_content_obj["favorite_number"] == 7
        assert decoded_content_obj["favorite_color"] == u"red"

        sr_avro_encoder.close()