def __init__(self, driver, nameSalt): self.driver = driver self.fileName = "travis_correct_confluent_protobuf_protobuf" self.topic = self.fileName + nameSalt self.sensor = sensor_pb2.SensorReading() self.sensor.dateTime = 1234 self.sensor.reading = 321.321 self.sensor.device.deviceID = "555-4321" self.sensor.device.enabled = True self.sensor.float_val = 4321.4321 self.sensor.int32_val = (1 << 31) - 1 self.sensor.sint32_val = (1 << 31) - 1 self.sensor.sint64_val = (1 << 63) - 1 self.sensor.uint32_val = (1 << 32) - 1 self.sensor.bytes_val = b'\xDE\xAD' self.sensor.double_array_val.extend([1 / 3, 32.21, 434324321]) self.sensor.uint64_val = (1 << 64) - 1 self.schema_registry_client = SchemaRegistryClient( {'url': driver.schemaRegistryAddress}) self.keyProtobufSerializer = ProtobufSerializer( sensor_pb2.SensorReading, self.schema_registry_client) self.valueProtobufSerializer = ProtobufSerializer( sensor_pb2.SensorReading, self.schema_registry_client) producer_conf = { 'bootstrap.servers': driver.kafkaAddress, 'key.serializer': self.keyProtobufSerializer, 'value.serializer': self.valueProtobufSerializer } self.protobufProducer = SerializingProducer(producer_conf)
def _make_serializer(self): return { SchemaType.AVRO: AvroSerializer(self.sr_client, AVRO_SCHEMA), SchemaType.PROTOBUF: ProtobufSerializer(ProtobufPayloadClass, self.sr_client) }[self.schema_type]
def test_protobuf_deserializer_type_mismatch(kafka_cluster): """ Ensures an Exception is raised when deserializing an unexpected type. """ pb2_1 = PublicTestProto_pb2.TestMessage pb2_2 = metadata_proto_pb2.HDFSOptions sr = kafka_cluster.schema_registry() topic = kafka_cluster.create_topic("serialization-proto-refs") serializer = ProtobufSerializer(pb2_1, sr) deserializer = ProtobufDeserializer(pb2_2) producer = kafka_cluster.producer(key_serializer=serializer) consumer = kafka_cluster.consumer(key_deserializer=deserializer) consumer.assign([TopicPartition(topic, 0)]) def dr(err, msg): print("dr msg {} {}".format(msg.key(), msg.value())) producer.produce(topic, key=pb2_1(test_string='abc', test_bool=True, test_bytes=b'def'), partition=0) producer.flush() with pytest.raises(ConsumeError, match="Error parsing message"): consumer.poll()
def getConfigs(): sr_client_props = { 'url': '<CCLOUD_SR_DNS>', 'basic.auth.user.info': '<CCLOUD_SR_KEY>:<CCLOUD_SR_SECRET>' } sr_client = SchemaRegistryClient(sr_client_props) value_serializer = ProtobufSerializer(FoodPreferences_pb2.PersonFood, sr_client) configs = { 'bootstrap.servers': '<CCLOUD_DNS>', 'security.protocol': 'SASL_SSL', 'sasl.mechanism': 'PLAIN', 'sasl.username': '******', 'sasl.password': '******', 'client.id': 'pythonProduction', 'compression.type': 'zstd', 'retries': '10', 'linger.ms': '5', 'key.serializer': StringSerializer('utf_8'), 'value.serializer': value_serializer } return configs
def test_protobuf_message_serialization(kafka_cluster, pb2, data): """ Validates that we get the same message back that we put in. """ topic = kafka_cluster.create_topic("serialization-proto") sr = kafka_cluster.schema_registry() value_serializer = ProtobufSerializer(pb2, sr, {'use.deprecated.format': False}) value_deserializer = ProtobufDeserializer(pb2, {'use.deprecated.format': False}) producer = kafka_cluster.producer(value_serializer=value_serializer) consumer = kafka_cluster.consumer(value_deserializer=value_deserializer) consumer.assign([TopicPartition(topic, 0)]) expect = pb2(**data) producer.produce(topic, value=expect, partition=0) producer.flush() msg = consumer.poll() actual = msg.value() assert [getattr(expect, k) == getattr(actual, k) for k in data.keys()]
def __protobuf_serializer(self): schema_registry_conf = {'url': self.config['schemaregistry.url']} schema_registry_client = SchemaRegistryClient(schema_registry_conf) _proto_conf = { 'auto.register.schemas': self.config['auto.register.schemas'], } return ProtobufSerializer(self.config['proto_msg_type'], schema_registry_client, conf=_proto_conf)
def test_protobuf_reference_registration(kafka_cluster, pb2, expected_refs): """ Registers multiple messages with dependencies then queries the Schema Registry to ensure the references match up. """ sr = kafka_cluster.schema_registry() topic = kafka_cluster.create_topic("serialization-proto-refs") serializer = ProtobufSerializer(pb2, sr) producer = kafka_cluster.producer(key_serializer=serializer) producer.produce(topic, key=pb2(), partition=0) producer.flush() registered_refs = sr.get_schema(serializer._schema_id).references assert expected_refs.sort() == [ref.name for ref in registered_refs].sort()
def test_protobuf_serializer_type_mismatch(kafka_cluster): """ Ensures an Exception is raised when deserializing an unexpected type. """ pb2_1 = TestProto_pb2.TestMessage pb2_2 = NestedTestProto_pb2.NestedMessage sr = kafka_cluster.schema_registry() topic = kafka_cluster.create_topic("serialization-proto-refs") serializer = ProtobufSerializer(pb2_1, sr) producer = kafka_cluster.producer(key_serializer=serializer) with pytest.raises(KafkaException, match=r"message must be of type <class" r" 'TestProto_pb2.TestMessage'\> not \<class" r" 'NestedTestProto_pb2.NestedMessage'\>"): producer.produce(topic, key=pb2_2())
def main(args): topic = args.topic schema_registry_conf = {'url': args.schema_registry} schema_registry_client = SchemaRegistryClient(schema_registry_conf) protobuf_serializer = ProtobufSerializer(user_pb2.User, schema_registry_client, {'use.deprecated.format': True}) producer_conf = { 'bootstrap.servers': args.bootstrap_servers, 'key.serializer': StringSerializer('utf_8'), 'value.serializer': protobuf_serializer } producer = SerializingProducer(producer_conf) print("Producing user records to topic {}. ^C to exit.".format(topic)) while True: # Serve on_delivery callbacks from previous calls to produce() producer.poll(0.0) try: user_name = input("Enter name: ") user_favorite_number = int(input("Enter favorite number: ")) user_favorite_color = input("Enter favorite color: ") user = user_pb2.User(name=user_name, favorite_color=user_favorite_color, favorite_number=user_favorite_number) producer.produce(topic=topic, partition=0, key=str(uuid4()), value=user, on_delivery=delivery_report) except (KeyboardInterrupt, EOFError): break except ValueError: print("Invalid input, discarding record...") continue print("\nFlushing records...") producer.flush()
from uuid import uuid4 # Protobuf generated class; resides at ./meal_pb2.py # Create it by running # protoc -I=. --python_out=. ./meal.proto import meal_pb2 from confluent_kafka import SerializingProducer from confluent_kafka.serialization import StringSerializer from confluent_kafka.schema_registry import SchemaRegistryClient from confluent_kafka.schema_registry.protobuf import ProtobufSerializer topic = 'meal' schema_registry_client = SchemaRegistryClient({'url': 'http://t620.lan:8081'}) protobuf_serializer = ProtobufSerializer(meal_pb2.Meal, schema_registry_client) producer_conf = { 'bootstrap.servers': 't620.lan:9092', 'key.serializer': StringSerializer('utf_8'), 'value.serializer': protobuf_serializer } producer = SerializingProducer(producer_conf) producer.poll(0.0) mybeer = meal_pb2.Meal.DrinkItems(drink_name="beer") mywine = meal_pb2.Meal.DrinkItems(drink_name="wine") meal = meal_pb2.Meal(name='pizza', drink=[mybeer, mywine])
def get_protobuf_serializer(self, msg_type, schema_registry_client, conf=None): return ProtobufSerializer(msg_type, schema_registry_client, conf)
from uuid import uuid4 epic = "CHART:CS.D.EURUSD.MINI.IP:1MINUTE" config_file = 'librdkafka.config' topic = "eurusd" conf = ccloud_lib.read_ccloud_config(config_file) # Create topic if needed ccloud_lib.create_topic(conf, topic) delivered_records = 0 schema_registry_conf = { 'url': conf['schema.registry.url'], 'basic.auth.user.info': conf['schema.registry.basic.auth.user.info'] } schema_registry_client = SchemaRegistryClient(schema_registry_conf) protobuf_serializer = ProtobufSerializer(lightbringer_pb2.CandlePrice, schema_registry_client) producer_conf = { 'bootstrap.servers': conf['bootstrap.servers'], 'sasl.mechanisms': conf['sasl.mechanisms'], 'security.protocol': conf['security.protocol'], 'sasl.username': conf['sasl.username'], 'sasl.password': conf['sasl.password'], 'key.serializer': StringSerializer('utf_8'), 'value.serializer': protobuf_serializer } producer = SerializingProducer(producer_conf) print("Producing user records to topic {}. ^C to exit.".format(topic)) #/////////////////////////////////////////////// def delivery_report(err, msg):