def get_json_serializer(self,
schema_str,
schema_registry_client,
to_dict=None,
conf=None):
return JSONSerializer(schema_str, schema_registry_client, to_dict,
conf)
def test_json_record_serialization_incompatible(kafka_cluster, load_file):
"""
Tests Serializer validation functionality.
product.json from:
https://json-schema.org/learn/getting-started-step-by-step.html
Args:
kafka_cluster (KafkaClusterFixture): cluster fixture
load_file (callable(str)): JSON Schema file reader
"""
topic = kafka_cluster.create_topic("serialization-json")
sr = kafka_cluster.schema_registry({'url': 'http://localhost:8081'})
schema_str = load_file("product.json")
value_serializer = JSONSerializer(sr, schema_str)
producer = kafka_cluster.producer(value_serializer=value_serializer)
record = {
"contractorId": 1,
"contractorName": "David Davidson",
"contractRate": 1250,
"trades": ["mason"]
}
with pytest.raises(SerializationError,
match=r"(.*) is a required property"):
producer.produce(topic, value=record, partition=0)
def make_serializer(topic, to_dict, schema_registry_url):
"""Create an AvroSerializer from a topic."""
client = SchemaRegistryClient({'url': schema_registry_url})
return JSONSerializer(
schema_str=_get_schema(client, topic),
schema_registry_client=client,
to_dict=to_dict,
conf={'auto.register.schemas': True},
)
def test_json_record_serialization_custom(kafka_cluster, load_file):
"""
Ensures to_dict and from_dict hooks are properly applied by the serializer.
Args:
kafka_cluster (KafkaClusterFixture): cluster fixture
load_file (callable(str)): JSON Schema file reader
"""
topic = kafka_cluster.create_topic("serialization-json")
sr = kafka_cluster.schema_registry({'url': 'http://localhost:8081'})
schema_str = load_file("product.json")
value_serializer = JSONSerializer(sr,
schema_str,
to_dict=_testProduct_to_dict)
value_deserializer = JSONDeserializer(schema_str,
from_dict=_testProduct_from_dict)
producer = kafka_cluster.producer(value_serializer=value_serializer)
record = _TestProduct(product_id=1,
name="The ice sculpture",
price=12.50,
tags=["cold", "ice"],
dimensions={
"length": 7.0,
"width": 12.0,
"height": 9.5
},
location={
"latitude": -78.75,
"longitude": 20.4
})
producer.produce(topic, value=record, partition=0)
producer.flush()
consumer = kafka_cluster.consumer(value_deserializer=value_deserializer)
consumer.assign([TopicPartition(topic, 0)])
msg = consumer.poll()
actual = msg.value()
assert all([
getattr(actual, attribute) == getattr(record, attribute)
for attribute in vars(record)
])
def test_json_record_serialization(kafka_cluster, load_file):
"""
Tests basic JsonSerializer and JsonDeserializer basic functionality.
product.json from:
https://json-schema.org/learn/getting-started-step-by-step.html
Args:
kafka_cluster (KafkaClusterFixture): cluster fixture
load_file (callable(str)): JSON Schema file reader
"""
topic = kafka_cluster.create_topic("serialization-json")
sr = kafka_cluster.schema_registry({'url': 'http://localhost:8081'})
schema_str = load_file("product.json")
value_serializer = JSONSerializer(sr, schema_str)
value_deserializer = JSONDeserializer(schema_str)
producer = kafka_cluster.producer(value_serializer=value_serializer)
record = {
"productId": 1,
"productName": "An ice sculpture",
"price": 12.50,
"tags": ["cold", "ice"],
"dimensions": {
"length": 7.0,
"width": 12.0,
"height": 9.5
},
"warehouseLocation": {
"latitude": -78.75,
"longitude": 20.4
}
}
producer.produce(topic, value=record, partition=0)
producer.flush()
consumer = kafka_cluster.consumer(value_deserializer=value_deserializer)
consumer.assign([TopicPartition(topic, 0)])
msg = consumer.poll()
actual = msg.value()
assert all([actual[k] == v for k, v in record.items()])
def test_json_record_serialization_no_title(kafka_cluster, load_file):
"""
Ensures ValueError raise if JSON Schema definition lacks Title annotation.
Args:
kafka_cluster (KafkaClusterFixture): cluster fixture
load_file (callable(str)): JSON Schema file reader
"""
sr = kafka_cluster.schema_registry({'url': 'http://localhost:8081'})
schema_str = load_file('not_title.json')
with pytest.raises(ValueError,
match="Missing required JSON schema annotation title"):
JSONSerializer(sr, schema_str)
def test_json_record_deserialization_mismatch(kafka_cluster, load_file):
"""
Ensures to_dict and from_dict hooks are properly applied by the serializer.
Args:
kafka_cluster (KafkaClusterFixture): cluster fixture
load_file (callable(str)): JSON Schema file reader
"""
topic = kafka_cluster.create_topic("serialization-json")
sr = kafka_cluster.schema_registry({'url': 'http://localhost:8081'})
schema_str = load_file("contractor.json")
schema_str2 = load_file("product.json")
value_serializer = JSONSerializer(sr, schema_str)
value_deserializer = JSONDeserializer(schema_str2)
producer = kafka_cluster.producer(value_serializer=value_serializer)
record = {
"contractorId": 2,
"contractorName": "Magnus Edenhill",
"contractRate": 30,
"trades": ["pickling"]
}
producer.produce(topic, value=record, partition=0)
producer.flush()
consumer = kafka_cluster.consumer(value_deserializer=value_deserializer)
consumer.assign([TopicPartition(topic, 0)])
with pytest.raises(
ConsumeError,
match=r"(.*) is a required property \(KafkaError code {}\)".format(
KafkaError._VALUE_DESERIALIZATION)):
consumer.poll()
def main(args):
topic = args.topic
schema_str = """
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "User",
"description": "A Confluent Kafka Python User",
"type": "object",
"properties": {
"name": {
"description": "User's name",
"type": "string"
},
"favorite_number": {
"description": "User's favorite number",
"type": "number",
"exclusiveMinimum": 0
},
"favorite_color": {
"description": "User's favorite color",
"type": "string"
}
},
"required": [ "name", "favorite_number", "favorite_color" ]
}
"""
schema_registry_conf = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
json_serializer = JSONSerializer(schema_registry_client, schema_str,
user_to_dict)
producer_conf = {
'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': json_serializer
}
producer = SerializingProducer(producer_conf)
print("Producing user records to topic {}. ^C to exit.".format(topic))
while True:
# Serve on_delivery callbacks from previous calls to produce()
producer.poll(0.0)
try:
user_name = input("Enter name: ")
user_address = input("Enter address: ")
user_favorite_number = int(input("Enter favorite number: "))
user_favorite_color = input("Enter favorite color: ")
user = User(name=user_name,
address=user_address,
favorite_color=user_favorite_color,
favorite_number=user_favorite_number)
producer.produce(topic=topic,
key=str(uuid4()),
value=user,
on_delivery=delivery_report)
except KeyboardInterrupt:
break
except ValueError:
print("Invalid input, discarding record...")
continue
print("\nFlushing records...")
producer.flush()
"address": {
"type": "string"
},
"keyword": {
"type": "string"
},
"position": {
"type": "integer"
},
},
}
schema_client = SchemaRegistryClient({"url": s.schema_server})
json_serializer = JSONSerializer(dumps(message_schema),
schema_client,
conf={"auto.register.schemas": False})
string_serializer = StringSerializer()
key_context = SerializationContext(s.registrations_topic, MessageField.KEY)
value_context = SerializationContext(s.registrations_topic, MessageField.VALUE)
con = psycopg2.connect(
database=s.db_database,
user=s.db_user,
password=s.db_password,
host=s.db_server,
port=s.db_port,
)
from .models import UserProducer
schema_registry_conf = {'url': SCHEMA_REGISTRY_URL}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
def delivery_report(err, msg):
if err is not None:
print("Delivery failed for User record {}: {}".format(msg.key(), err))
return
print(
'User record {} successfully produced to {} [{}] at offset {}'.format(
msg.key(), msg.topic(), msg.partition(), msg.offset()))
json_serializer = JSONSerializer(USER_SCHEMA, schema_registry_client,
user_to_dict)
producer_conf = {
'bootstrap.servers': 'localhost:9092',
'key.serializer': StringSerializer('utf_8'),
'value.serializer': json_serializer
}
producer = SerializingProducer(producer_conf)
def send(username, data, token):
user = UserProducer(username=username, data=data, token=token)
producer.produce(topic=USER_TOPIC,
key=str(uuid4()),
value=user,
on_delivery=delivery_report)
def MyProducer(config):
schema_str = """
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "covid",
"type": "object",
"properties": {
"ID": {
"type": "string"
},
"Country": {
"type": "string"
},
"CountryCode": {
"type": "string"
},
"Slug": {
"type": "string"
},
"NewConfirmed": {
"type": "integer"
},
"TotalConfirmed": {
"type": "integer"
},
"NewDeaths": {
"type": "integer"
},
"TotalDeaths": {
"type": "integer"
},
"NewRecovered": {
"type": "integer"
},
"TotalRecovered": {
"type": "integer"
},
"Date": {
"type": "string"
},
"Premium": {
"type": "object"
}
},
"required": [
"ID",
"Country",
"CountryCode",
"Slug",
"NewConfirmed",
"TotalConfirmed",
"NewDeaths",
"TotalDeaths",
"NewRecovered",
"TotalRecovered",
"Date",
"Premium"
]
}
"""
schema_registry_conf = {
'url': config["schema.registry.url"],
'basic.auth.user.info': config['basic.auth.user.info']
}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
jsons = JSONSerializer(schema_str, schema_registry_client,
lambda f, ctx: f)
producer_conf = {
'bootstrap.servers': config['bootstrap.servers'],
'key.serializer': StringSerializer('utf_8'),
'value.serializer': jsons,
'security.protocol': config["security.protocol"],
"sasl.mechanisms": config["sasl.mechanisms"],
"sasl.username": config["sasl.username"],
"sasl.password": config["sasl.password"]
}
return SerializingProducer(producer_conf)