class AvroProducer(Producer): """ Kafka Producer client which does avro schema encoding to messages. Handles schema registration, Message serialization. Constructor takes below parameters @:param: config: dict object with config parameters containing url for schema registry (schema.registry.url). @:param: default_key_schema: Optional avro schema for key @:param: default_value_schema: Optional avro schema for value """ def __init__(self, config, default_key_schema=None, default_value_schema=None): if ('schema.registry.url' not in config.keys()): raise ValueError("Missing parameter: schema.registry.url") schem_registry_url = config["schema.registry.url"] del config["schema.registry.url"] super(AvroProducer, self).__init__(config) self._serializer = MessageSerializer( CachedSchemaRegistryClient(url=schem_registry_url)) self._key_schema = default_key_schema self._value_schema = default_value_schema def produce(self, **kwargs): """ Sends message to kafka by encoding with specified avro schema @:param: topic: topic name @:param: value: A dictionary object @:param: value_schema : Avro schema for value @:param: key: A dictionary object @:param: key_schema : Avro schema for key @:exception: SerializerError """ # get schemas from kwargs if defined key_schema = kwargs.pop('key_schema', self._key_schema) value_schema = kwargs.pop('value_schema', self._value_schema) topic = kwargs.pop('topic', None) if not topic: raise ClientError("Topic name not specified.") value = kwargs.pop('value', None) key = kwargs.pop('key', None) if value: if value_schema: value = self._serializer.encode_record_with_schema( topic, value_schema, value) else: raise SerializerError("Avro schema required for value") if key: if key_schema: key = self._serializer.encode_record_with_schema( topic, key_schema, key, True) else: raise SerializerError("Avro schema required for key") super(AvroProducer, self).produce(topic, value, key, **kwargs)
class AvroProducer(object): def __init__(self, config, default_key_schema=None, default_value_schema=None, schema_registry=None): schema_registry_url = config.pop("schema.registry.url", None) schema_registry_ca_location = config.pop( "schema.registry.ssl.ca.location", None) schema_registry_certificate_location = config.pop( "schema.registry.ssl.certificate.location", None) schema_registry_key_location = config.pop( "schema.registry.ssl.key.location", None) if schema_registry is None: if schema_registry_url is None: raise ValueError("Missing parameter: schema.registry.url") schema_registry = CachedSchemaRegistryClient( url=schema_registry_url, ca_location=schema_registry_ca_location, cert_location=schema_registry_certificate_location, key_location=schema_registry_key_location) elif schema_registry_url is not None: raise ValueError( "Cannot pass schema_registry along with schema.registry.url config" ) self.producer = Producer(config) self._serializer = MessageSerializer(schema_registry) self._key_schema = default_key_schema self._value_schema = default_value_schema def flush(self): self.producer.flush() def produce(self, **kwargs): key_schema = kwargs.pop('key_schema', self._key_schema) value_schema = kwargs.pop('value_schema', self._value_schema) topic = kwargs.pop('topic', None) if not topic: raise ClientError("Topic name not specified.") value = kwargs.pop('value', None) key = kwargs.pop('key', None) if value is not None: if value_schema: value = self._serializer.encode_record_with_schema( topic, value_schema, value) else: raise ValueSerializerError("Avro schema required for values") if key is not None: if key_schema: key = self._serializer.encode_record_with_schema( topic, key_schema, key, True) self.producer.produce(topic, value, key, **kwargs)
def produce(config, topic, input_messages): """ produce initiate sending a message to Kafka, call the produce method passing in the input_messages key/value and and callback Parameters ---------- topic: str topic where the input message publish too input_messages: dict a key/value input messages config: dict the config values that needed by the produce """ if topic is None: logger.debug('Required topic field must be set') raise ValueError() if len(input_messages) <= 0: logger.debug('Required data field must not be empty.') raise ValueError() bootstrap_servers, schema_registry = producer_config(config) producer = Producer(bootstrap_servers) admin_client = AdminClient(bootstrap_servers) topics = admin_client.list_topics().topics #Just to show what's available print(topics) if not topics: print('Not Topics') raise RuntimeError() sr = CachedSchemaRegistryClient(schema_registry) ser = MessageSerializer(sr) # get schema id, schema, version = sr.get_latest_schema(topic + "-value") if schema: print('In If Schema') for key, value in input_messages.items(): if validate_uuid4(key): print('In validate in For loop') serializedMessage = ser.encode_record_with_schema( topic, schema, value) producer.produce(topic=topic, key=key, value=serializedMessage, callback=acked) # producer.flush() # bad idea, it limits throughput to the broker round trip time producer.poll(1) else: print('In Else of For Loop') logger.error('Invalid UUID String: ', key) else: print('Schema not found for topic name: ', topic) print('In Else Schema') sys.exit(1)
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id, schema): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded_msg, decoded_schema = self.ms.decode_message(message) self.assertTrue(decoded_msg) self.assertEqual(decoded_msg, expected) self.assertEqual(decoded_schema, schema) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id, basic) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id( adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id, adv) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id, basic) def test_decode_none(self): """"null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def hash_func(self): return hash(str(self))
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) def test_decode_none(self): """"null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def hash_func(self): return hash(str(self))
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer # Make RecordSchema and PrimitiveSchema hashable schema.RecordSchema.__hash__ = self.hash_func schema.PrimitiveSchema.__hash__ = self.hash_func self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = util.parse_schema_from_string(data_gen.ADVANCED_SCHEMA) basic = util.parse_schema_from_string(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id( adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = util.parse_schema_from_string(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) def hash_func(self): return hash(str(self))
def test_select(started_cluster): # type: (ClickHouseCluster) -> None schema_registry_client = CachedSchemaRegistryClient( 'http://localhost:{}'.format(started_cluster.schema_registry_port)) serializer = MessageSerializer(schema_registry_client) schema = avro.schema.make_avsc_object({ 'name': 'test_record', 'type': 'record', 'fields': [{ 'name': 'value', 'type': 'long' }] }) buf = io.BytesIO() for x in range(0, 3): message = serializer.encode_record_with_schema('test_subject', schema, {'value': x}) buf.write(message) data = buf.getvalue() instance = started_cluster.instances["dummy"] # type: ClickHouseInstance schema_registry_url = "http://{}:{}".format( started_cluster.schema_registry_host, 8081) run_query(instance, "create table avro_data(value Int64) engine = Memory()") settings = {'format_avro_schema_registry_url': schema_registry_url} run_query(instance, "insert into avro_data format AvroConfluent", data, settings) stdout = run_query(instance, "select * from avro_data") assert list(map(str.split, stdout.splitlines())) == [ ["0"], ["1"], ["2"], ]
class AvroProducer(Producer): """ Kafka Producer client which does avro schema encoding to messages. Handles schema registration, Message serialization. Constructor takes below parameters. :param dict config: Config parameters containing url for schema registry (``schema.registry.url``) and the standard Kafka client configuration (``bootstrap.servers`` et.al). :param str default_key_schema: Optional default avro schema for key :param str default_value_schema: Optional default avro schema for value """ def __init__(self, config, default_key_schema=None, default_value_schema=None, schema_registry=None): schema_registry_url = config.pop("schema.registry.url", None) schema_registry_ca_location = config.pop( "schema.registry.ssl.ca.location", None) schema_registry_certificate_location = config.pop( "schema.registry.ssl.certificate.location", None) schema_registry_key_location = config.pop( "schema.registry.ssl.key.location", None) if schema_registry is None: if schema_registry_url is None: raise ValueError("Missing parameter: schema.registry.url") schema_registry = CachedSchemaRegistryClient( url=schema_registry_url, ca_location=schema_registry_ca_location, cert_location=schema_registry_certificate_location, key_location=schema_registry_key_location) elif schema_registry_url is not None: raise ValueError( "Cannot pass schema_registry along with schema.registry.url config" ) super(AvroProducer, self).__init__(config) self._serializer = MessageSerializer(schema_registry) self._key_schema = default_key_schema self._value_schema = default_value_schema def produce(self, **kwargs): """ Asynchronously sends message to Kafka by encoding with specified or default avro schema. :param str topic: topic name :param object value: An object to serialize :param str value_schema: Avro schema for value :param object key: An object to serialize :param str key_schema: Avro schema for key Plus any other parameters accepted by confluent_kafka.Producer.produce :raises SerializerError: On serialization failure :raises BufferError: If producer queue is full. :raises KafkaException: For other produce failures. """ # get schemas from kwargs if defined key_schema = kwargs.pop('key_schema', self._key_schema) value_schema = kwargs.pop('value_schema', self._value_schema) topic = kwargs.pop('topic', None) if not topic: raise ClientError("Topic name not specified.") value = kwargs.pop('value', None) key = kwargs.pop('key', None) if value is not None: if value_schema: value = self._serializer.encode_record_with_schema( topic, value_schema, value) else: raise ValueSerializerError("Avro schema required for values") if key is not None: if key_schema: key = self._serializer.encode_record_with_schema( topic, key_schema, key, True) else: raise KeySerializerError("Avro schema required for key") super(AvroProducer, self).produce(topic, value, key, **kwargs)
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id( adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) def test_decode_none(self): """"null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def test_decode_with_schema(self): topic = 'test_specific' schema_v1 = avro.loads( data_gen.load_schema_file('evolution_schema_v1.avsc')) schema_v2 = avro.loads( data_gen.load_schema_file('evolution_schema_v2.avsc')) dsv1 = SpecificRecordMessageDeserializer(self.client, value_schema=schema_v1) dsv2 = SpecificRecordMessageDeserializer(self.client, value_schema=schema_v2) record_v1 = {"name": "suzyq", "age": 27} record_v2 = dict(record_v1) record_v2['gender'] = 'NONE' encoded_v1 = self.ms.encode_record_with_schema(topic, schema_v1, record_v1) decoded_v1_v1 = dsv1.decode_message(encoded_v1, is_key=False) self.assertDictEqual(record_v1, decoded_v1_v1) decoded_v1_v2 = dsv2.decode_message(encoded_v1, is_key=False) self.assertDictEqual(record_v2, decoded_v1_v2) encoded_v2 = self.ms.encode_record_with_schema(topic, schema_v2, record_v2) decoded_v2_v2 = dsv2.decode_message(encoded_v2, is_key=False) self.assertDictEqual(record_v2, decoded_v2_v2) decoded_v2_v1 = dsv1.decode_message(encoded_v2, is_key=False) self.assertDictEqual(record_v1, decoded_v2_v1) def hash_func(self): return hash(str(self))
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id( adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) @skipIf(version_info < (3, ), 'unittest.mock.patch not available in Python 2') def test_encode_record_with_schema_sets_writers_cache_once(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' self.client.register(subject, basic) records = data_gen.BASIC_ITEMS with patch.object(self.ms, "_get_encoder_func") as encoder_func_mock: for record in records: self.ms.encode_record_with_schema(topic, basic, record) encoder_func_mock.assert_called_once_with(basic) def test_decode_none(self): """null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def hash_func(self): return hash(str(self))
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) def test_decode_none(self): """"null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def test__get_subject_for_key_with_topic_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.key_subject_name_strategy_func = topic_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True) expected = "topic-key" self.assertEqual(expected, subject) def test__get_subject_for_key_with_record_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.key_subject_name_strategy_func = record_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True) expected = "python.test.basic.basic" self.assertEqual(expected, subject) def test__get_subject_for_key_with_topic_record_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.key_subject_name_strategy_func = topic_record_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True) expected = "topic-python.test.basic.basic" self.assertEqual(expected, subject) def test__get_subject_for_value_with_topic_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.value_subject_name_strategy_func = topic_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False) expected = "topic-value" self.assertEqual(expected, subject) def test__get_subject_for_value_with_record_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.value_subject_name_strategy_func = record_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False) expected = "python.test.basic.basic" self.assertEqual(expected, subject) def test__get_subject_for_value_with_topic_record_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.value_subject_name_strategy_func = topic_record_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False) expected = "topic-python.test.basic.basic" self.assertEqual(expected, subject) def hash_func(self): return hash(str(self))
class AvroProducer(Producer): """ Kafka Producer client which does avro schema encoding to messages. Handles schema registration, Message serialization. Constructor takes below parameters. :param dict config: Config parameters containing url for schema registry (``schema.registry.url``) and the standard Kafka client configuration (``bootstrap.servers`` et.al). :param str default_key_schema: Optional default avro schema for key :param str default_value_schema: Optional default avro schema for value """ def __init__(self, config, default_key_schema=None, default_value_schema=None, schema_registry=None): sr_conf = {key.replace("schema.registry.", ""): value for key, value in config.items() if key.startswith("schema.registry")} if sr_conf.get("basic.auth.credentials.source") == 'SASL_INHERIT': sr_conf['sasl.mechanisms'] = config.get('sasl.mechanisms', '') sr_conf['sasl.username'] = config.get('sasl.username', '') sr_conf['sasl.password'] = config.get('sasl.password', '') ap_conf = {key: value for key, value in config.items() if not key.startswith("schema.registry")} if schema_registry is None: schema_registry = CachedSchemaRegistryClient(sr_conf) elif sr_conf.get("url", None) is not None: raise ValueError("Cannot pass schema_registry along with schema.registry.url config") super(AvroProducer, self).__init__(ap_conf) self._serializer = MessageSerializer(schema_registry) self._key_schema = default_key_schema self._value_schema = default_value_schema def produce(self, **kwargs): """ Asynchronously sends message to Kafka by encoding with specified or default avro schema. :param str topic: topic name :param object value: An object to serialize :param str value_schema: Avro schema for value :param object key: An object to serialize :param str key_schema: Avro schema for key Plus any other parameters accepted by confluent_kafka.Producer.produce :raises SerializerError: On serialization failure :raises BufferError: If producer queue is full. :raises KafkaException: For other produce failures. """ # get schemas from kwargs if defined key_schema = kwargs.pop('key_schema', self._key_schema) value_schema = kwargs.pop('value_schema', self._value_schema) topic = kwargs.pop('topic', None) if not topic: raise ClientError("Topic name not specified.") value = kwargs.pop('value', None) key = kwargs.pop('key', None) if value is not None: if value_schema: value = self._serializer.encode_record_with_schema(topic, value_schema, value) else: raise ValueSerializerError("Avro schema required for values") if key is not None: if key_schema: key = self._serializer.encode_record_with_schema(topic, key_schema, key, True) else: raise KeySerializerError("Avro schema required for key") super(AvroProducer, self).produce(topic, value, key, **kwargs)
class AvroProducer(Producer): """ Kafka Producer client which does avro schema encoding to messages. Handles schema registration, Message serialization. Constructor takes below parameters. :param dict config: Config parameters containing url for schema registry (``schema.registry.url``) and the standard Kafka client configuration (``bootstrap.servers`` et.al). :param str default_key_schema: Optional default avro schema for key :param str default_value_schema: Optional default avro schema for value """ def __init__(self, config, default_key_schema=None, default_value_schema=None, schema_registry=None, **kwargs): sr_conf = { key.replace("schema.registry.", ""): value for key, value in config.items() if key.startswith("schema.registry") } if sr_conf.get("basic.auth.credentials.source") == 'SASL_INHERIT': # Fallback to plural 'mechanisms' for backward compatibility sr_conf['sasl.mechanism'] = config.get( 'sasl.mechanism', config.get('sasl.mechanisms', '')) sr_conf['sasl.username'] = config.get('sasl.username', '') sr_conf['sasl.password'] = config.get('sasl.password', '') sr_conf['auto.register.schemas'] = config.get( 'auto.register.schemas', True) ap_conf = { key: value for key, value in config.items() if not key.startswith("schema.registry") } if schema_registry is None: schema_registry = CachedSchemaRegistryClient(sr_conf) elif sr_conf.get("url", None) is not None: raise ValueError( "Cannot pass schema_registry along with schema.registry.url config" ) super(AvroProducer, self).__init__(ap_conf, **kwargs) self._serializer = MessageSerializer(schema_registry) self._key_schema = default_key_schema self._value_schema = default_value_schema def produce(self, **kwargs): """ Asynchronously sends message to Kafka by encoding with specified or default avro schema. :param str topic: topic name :param object value: An object to serialize :param str value_schema: Avro schema for value :param object key: An object to serialize :param str key_schema: Avro schema for key Plus any other parameters accepted by confluent_kafka.Producer.produce :raises SerializerError: On serialization failure :raises BufferError: If producer queue is full. :raises KafkaException: For other produce failures. """ # get schemas from kwargs if defined key_schema = kwargs.pop('key_schema', self._key_schema) value_schema = kwargs.pop('value_schema', self._value_schema) topic = kwargs.pop('topic', None) if not topic: raise ClientError("Topic name not specified.") value = kwargs.pop('value', None) key = kwargs.pop('key', None) if value is not None: if value_schema: value = self._serializer.encode_record_with_schema( topic, value_schema, value) else: raise ValueSerializerError("Avro schema required for values") if key is not None: if key_schema: key = self._serializer.encode_record_with_schema( topic, key_schema, key, True) else: raise KeySerializerError("Avro schema required for key") super(AvroProducer, self).produce(topic, value, key, **kwargs)
class AvroProducer(object): ''' Kafka Producer client which does avro schema encoding to messages. Handles schema registration, Message serialization. Constructor takes below parameters @:param: producer: confluent_kafka.Producer object @:param: message_serializer: Message Serializer object ''' def __init__(self, producer, schema_registry_url, default_key_schema=None, default_value_schema=None ): # real signature unknown; restored from __doc__ self._producer = producer self._serializer = MessageSerializer( CachedSchemaRegistryClient(url=schema_registry_url)) self.key_schema = default_key_schema self.value_schema = default_value_schema def produce(self, **kwargs): ''' Sends message to kafka by encoding with specified avro schema @:param: topic: topic name @:param: value: A dictionary object @:param: value_schema : Avro schema for value @:param: key: A dictionary object @:param: key_schema : Avro schema for key @:exception: SerializerError ''' # get schemas from kwargs if defined key_schema = kwargs.pop('key_schema', None) value_schema = kwargs.pop('value_schema', None) topic = kwargs.pop('topic', None) if not topic: log.error("Topic name not specified.") raise ClientError("Topic name not specified.") value = kwargs.pop('value', None) key = kwargs.pop('key', None) # if key_schema is not initialized, fall back on default key_schema passed as construction param. if not key_schema: key_schema = self.key_schema # if value_schema is not initialized, fall back on default value_schema passed as construction param. if not value_schema: value_schema = self.value_schema if value: if value_schema: value = self._serializer.encode_record_with_schema( topic, value_schema, value) else: log.error("Schema required for value serialization") raise SerializerError("Avro schema required for value") if key: if key_schema: key = self._serializer.encode_record_with_schema( topic, key_schema, key, True) else: log.error("Schema required for key serialization") raise SerializerError("Avro schema required for key") self._producer.produce(topic, value, key, **kwargs) def poll(self, timeout): self._producer.poll(timeout) def flush(self, *args, **kwargs): self._producer.flush(*args, **kwargs)
def delivery_report(err, msg): if err is not None: logging.error("Delivery failed for User record {}: {}".format( msg.key(), err)) else: logging.info( "Record successfully delivered -> TOPIC: {} - PARTITION: [{}] - OFFSET {}" .format(msg.topic(), msg.partition(), msg.offset())) producer = Producer(config) # Avro data needs to be loaded as an encoded dict with schema and the Producer takes care of inserting it properly # Just load 5 records into the source topic for i in range(5): data.id = i data.name = "Value of the string that was inserted is ~~~~~~ ========= -----> " + str( i) data_dict = data.__dict__ avro_record = serializer.encode_record_with_schema(topic="avro-topic-1", schema=RECORD_SCHEMA, record=data_dict, is_key=False) producer.produce("avro-topic", key=str(1), value=avro_record, on_delivery=(delivery_report)) time.sleep(3) # EOS - Synchronous writes producer.flush()