def __init__( self, schema_registry_url: str, auto_register_schemas: bool = True, subject_name_strategy: SubjectNameStrategy = SubjectNameStrategy. RecordNameStrategy, **kwargs): super().__init__(**kwargs) schema_registry_url = schema_registry_url self.schema_registry = CachedSchemaRegistryClient(schema_registry_url) self.auto_register_schemas = auto_register_schemas self.subject_name_strategy = subject_name_strategy self._serializer_impl = AvroSerDeBase(self.schema_registry)
def __init__(self): config = self.load_config(self.CONFIG_FILE) sc = CachedSchemaRegistryClient(url=config['kafkaSchemaRegistryUrl']) self.topic = config['kafkaTopics'][0] key_schema = sc.get_latest_schema(self.topic + "-key")[1] val_schema = sc.get_latest_schema(self.topic + "-value")[1] self.producer = AvroProducer( { 'bootstrap.servers': config['kafkaBootstrapServers'], 'schema.registry.url': config['kafkaSchemaRegistryUrl'] }, default_key_schema=key_schema, default_value_schema=val_schema)
class EasyAvroProducer(BaseProducer, AvroProducer): def __init__(self, schema_registry_url: str, kafka_brokers: List[str], kafka_topic: str, value_schema: schema.Schema = None, key_schema: schema.Schema = None, debug: bool = False, kafka_conf: dict = None, py_conf: dict = None) -> None: self.kafka_topic = kafka_topic self._client = CachedSchemaRegistryClient( dict(url=schema_registry_url)) # Value Schema if value_schema is None: vs_name = '{}-value'.format(self.kafka_topic) _, value_schema, _ = self._client.get_latest_schema(vs_name) if value_schema is None: raise ValueError( 'Schema "{}" not found in registry'.format(vs_name)) # Key Schema if key_schema is None: ks_name = '{}-key'.format(self.kafka_topic) _, key_schema, _ = self._client.get_latest_schema(ks_name) if key_schema is None: raise ValueError( 'Schema "{}" not found in registry'.format(ks_name)) conf = { 'bootstrap.servers': ','.join(kafka_brokers), 'schema.registry.url': schema_registry_url, 'client.id': self.__class__.__name__, 'api.version.request': 'true', } if debug is True: conf['debug'] = 'msg' kafka_conf = kafka_conf or {} py_conf = py_conf or {} super().__init__({ **conf, **kafka_conf }, default_value_schema=value_schema, default_key_schema=key_schema, **py_conf)
class EasyAvroProducer(AvroProducer): def __init__(self, schema_registry_url: str, kafka_brokers: List[str], kafka_topic: str, value_schema: schema.Schema = None, key_schema: schema.Schema = None, debug: bool = False) -> None: self.kafka_topic = kafka_topic self._client = CachedSchemaRegistryClient(url=schema_registry_url) # Value Schema if value_schema is None: vs_name = '{}-value'.format(self.kafka_topic) _, value_schema, _ = self._client.get_latest_schema(vs_name) if value_schema is None: raise ValueError( 'Schema "{}" not found in registry'.format(vs_name)) # Key Schema if key_schema is None: ks_name = '{}-key'.format(self.kafka_topic) _, key_schema, _ = self._client.get_latest_schema(ks_name) if key_schema is None: raise ValueError( 'Schema "{}" not found in registry'.format(ks_name)) conf = { 'bootstrap.servers': ','.join(kafka_brokers), 'schema.registry.url': schema_registry_url, 'client.id': self.__class__.__name__, 'api.version.request': 'true' } if debug: conf['debug'] = 'msg' super().__init__(conf, default_value_schema=value_schema, default_key_schema=key_schema) def produce(self, records: List[Tuple]) -> None: for i, r in enumerate(records): super().produce(topic=self.kafka_topic, key=r[0], value=r[1]) L.info("{}/{} messages".format(i + 1, len(records))) L.debug("Flushing producer...") self.flush() L.info("Done producing")
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas # Configure the broker properties self.broker_properties = { "BROKER_URL": "PLAINTEXT://localhost:9092", "SCHEMA_REGISTRY_URL": "http://localhost:8081", "group.id": f"{self.topic_name}", } # If the topic does not already exist, try to create it if self.topic_name not in Producer.existing_topics: self.create_topic() Producer.existing_topics.add(self.topic_name) # Configure the AvroProducer self.producer = AvroProducer( {"bootstrap.servers": self.broker_properties["BROKER_URL"]}, schema_registry=CachedSchemaRegistryClient( {"url": self.broker_properties["SCHEMA_REGISTRY_URL"]}, ))
def __init__( self, topic_name_pattern, message_handler, is_avro=True, offset_earliest=False, sleep_secs=1.0, consume_timeout=0.1, ): """Creates a consumer object for asynchronous use""" self.topic_name_pattern = topic_name_pattern self.message_handler = message_handler self.sleep_secs = sleep_secs self.consume_timeout = consume_timeout self.offset_earliest = offset_earliest self.BROKER_URL = 'PLAINTEXT://localhost:9092' self.SCHEMA_REGISTRY_URL = 'http://localhost:8081' self.broker_properties = { "bootstrap.servers": self.BROKER_URL, "group.id": "0" } if is_avro is True: schema_registry = CachedSchemaRegistryClient( {"url": self.SCHEMA_REGISTRY_URL}) self.consumer = AvroConsumer(self.broker_properties, schema_registry=schema_registry) else: self.consumer = Consumer(self.broker_properties) pass self.consumer.subscribe([self.topic_name_pattern], on_assign=self.on_assign)
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas self.broker_properties = {"bootstrap.servers": KAFKA_BROKER_URL} if self.topic_name not in Producer.existing_topics: self.create_topic() Producer.existing_topics.add(self.topic_name) schema_registry = CachedSchemaRegistryClient("http://localhost:8081") self.producer = AvroProducer(self.broker_properties, default_key_schema=self.key_schema, default_value_schema=self.value_schema, schema_registry=schema_registry)
async def consume(topic_name): """Consumes data from the Kafka Topic""" # # Create a CachedSchemaRegistryClient # schema_registry = CachedSchemaRegistryClient({"url": SCHEMA_REGISTRY_URL}) # # Use the Avro Consumer # c = AvroConsumer( { "bootstrap.servers": BROKER_URL, "group.id": "0" }, schema_registry=schema_registry, ) c.subscribe([topic_name]) while True: message = c.poll(1.0) if message is None: print("no message received by consumer") elif message.error() is not None: print(f"error from consumer {message.error()}") else: try: #The print from console print(message.value()) except KeyError as e: print(f"Failed to unpack message {e}") await asyncio.sleep(1.0)
async def consume(topic_name): """Consumes data from the Kafka Topic""" # # TODO: Create a CachedSchemaRegistryClient # schema_registry = CachedSchemaRegistryClient(SCHEMA_REGISTRY_URL) # # TODO: Use the Avro Consumer See: https://docs.confluent.io/current/clients/confluent-kafka-python/index.html # ?highlight=loads#confluent_kafka.avro.AvroConsumer # #c = AvroConsumer({"bootstrap.servers": BROKER_URL, "group.id": "0","auto.offset.reset":'earliest'}, schema_registry=schema_registry) c = Consumer({ "bootstrap.servers": BROKER_URL, "group.id": "0", "auto.offset.reset": 'earliest' }) c.subscribe([topic_name]) while True: try: msg = c.poll(1.0) except SerializerError as e: print("Message deserialization failed for {}: {}".format(msg, e)) raise SerializerError if msg is None: print("message is not received yet") elif msg.error(): print("AvroConsumer error: {}".format(msg.error())) return else: key, value = msg.key(), msg.value() print(key, value) await asyncio.sleep(1.0)
def __init__(self, topic_name_pattern, message_handler, is_avro=True, offset_earliest=True): """Creates a consumer object for asynchronous use""" self._topic_name_pattern = topic_name_pattern self._message_handler = message_handler self._offset_earliest = offset_earliest conf = { "bootstrap.servers": config["KAFKA"]["BROKER_URL"], "group.id": "0", "auto.offset.reset": "earliest" } if is_avro is True: schema_registry = CachedSchemaRegistryClient( {"url": config["KAFKA"]["SCHEMA_REGISTRY_URL"]}) self._consumer = AvroConsumer(conf, schema_registry=schema_registry) else: self._consumer = Consumer(conf) self._consumer.subscribe([self._topic_name_pattern], on_assign=self._on_assign) self._timeout = float(config["PARAM"]["CONSUMER_POLL_TIMEOUT"])
def __init__( self, topic_name_pattern, message_handler, is_avro=True, offset_earliest=False, sleep_secs=1.0, consume_timeout=0.1, ): """Creates a consumer object for asynchronous use""" self.topic_name_pattern = topic_name_pattern self.message_handler = message_handler self.sleep_secs = sleep_secs self.consume_timeout = consume_timeout self.offset_earliest = offset_earliest self.broker_properties = { 'BROKER_URL': 'PLAINTEXT://localhost:9092', 'SCHEMA_REGISTRY': 'http://localhost:8081', } schema_registry = CachedSchemaRegistryClient({ 'url': self.broker_properties['SCHEMA_REGISTRY'], 'ssl.ca.location': None, 'ssl.certificate.location': None, 'ssl.key.location': None }) # Create the Consumer, using the appropriate type. if is_avro is True: self.broker_properties[ "schema.registry.url"] = "http://localhost:8081" self.consumer = AvroConsumer( { "bootstrap.servers": self.broker_properties['BROKER_URL'], "group.id": "0" }, schema_registry=schema_registry) else: self.consumer = Consumer({ "bootstrap.servers": self.broker_properties['BROKER_URL'], "group.id": "0", "auto.offset.reset": "earliest" }) # # # Configure the AvroConsumer and subscribe to the topics. Make sure to think about # how the `on_assign` callback should be invoked. # # self.consumer.subscribe([self.topic_name_pattern], on_assign=self.on_assign)
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas # # # Configure the broker properties below. Make sure to reference the project README # and use the Host URL for Kafka and Schema Registry! # # self.broker_properties = { 'bootstrap.servers': BROKER_URL, } # If the topic does not already exist, try to create it if self.topic_name not in Producer.existing_topics: self.create_topic() Producer.existing_topics.add(self.topic_name) self.producer = AvroProducer( self.broker_properties, schema_registry=CachedSchemaRegistryClient(SCHEMA_REGISTRY_URL))
async def converter(CONSUME_TOPIC, PRODUCE_TOPIC, BROKER_URL, SCHEMA_REGISTRY_URL): """Consumes data from the Kafka Topic """ schema_registry = CachedSchemaRegistryClient({"url": SCHEMA_REGISTRY_URL}) c = AvroConsumer( { "bootstrap.servers": BROKER_URL, "client.id": "project-insight", "group.id": "convertor-in-consumer", "auto.offset.reset": "earliest", }, schema_registry=schema_registry, ) c.subscribe([CONSUME_TOPIC]) p = Producer({"bootstrap.servers": BROKER_URL}) while True: message = c.poll(1.0) if message is None: logger.info("no message received by consumer") elif message.error() is not None: logger.error(f"error from consumer {message.error()}") else: try: print(message.value()) p.produce(topic=PRODUCE_TOPIC, key=str(uuid4()), value=json.dumps(message.value())) except KeyError as e: logger.error(f"Failed to unpack message {e}") await asyncio.sleep(0.01)
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas self.broker_properties = { "bootstrap.servers": BROKER_URL, } # If the topic does not already exist, try to create it if self.topic_name not in Producer.existing_topics: self.create_topic(self.topic_name) Producer.existing_topics.add(self.topic_name) schema_registry = CachedSchemaRegistryClient(SCHEMA_REGISTRY_URL) self.producer = AvroProducer( self.broker_properties, schema_registry=schema_registry )
async def produce(topic_name): """Produces data into the Kafka Topic""" # # TODO: Create a CachedSchemaRegistryClient. Use SCHEMA_REGISTRY_URL. # See: https://github.com/confluentinc/confluent-kafka-python/blob/master/confluent_kafka/avro/cached_schema_registry_client.py#L47 # schema_registry = CachedSchemaRegistryClient({"url": SCHEMA_REGISTRY_URL}) # # TODO: Replace with an AvroProducer. # See: https://docs.confluent.io/current/clients/confluent-kafka-python/index.html?highlight=loads#confluent_kafka.avro.AvroProducer # p = AvroProducer({"bootstrap.servers": BROKER_URL}, schema_registry=schema_registry) while True: # # TODO: Replace with an AvroProducer produce. Make sure to specify the schema! # Tip: Make sure to serialize the ClickEvent with `asdict(ClickEvent())` # See: https://docs.confluent.io/current/clients/confluent-kafka-python/index.html?highlight=loads#confluent_kafka.avro.AvroProducer # p.produce( topic=topic_name, value=asdict(ClickEvent()), # TODO: Supply schema value_schema=ClickEvent.schema) await asyncio.sleep(1.0)
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas # If the topic does not already exist, try to create it if self.topic_name not in Producer.existing_topics: self.create_topic() Producer.existing_topics.add(self.topic_name) else: logger.debug("Topic already exists: %s", self.topic_name) schema_registry = CachedSchemaRegistryClient( {"url": SCHEMA_REGISTRY_URL}) self.producer = AvroProducer({"bootstrap.servers": BROKER_URL}, schema_registry=schema_registry)
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas schema_registry = CachedSchemaRegistryClient(self.SCHEMA_REGISTRY_URL) self.broker_properties = { "bootstrap.servers": self.BROKER_URL, "linger.ms": 500, "acks": 1, "retries": 3, "message.max.bytes": 4096, "batch.num.messages": 10 } # If the topic does not already exist, try to create it if self.topic_name not in Producer.existing_topics: self.create_topic() Producer.existing_topics.add(self.topic_name) self.producer: AvroProducer = AvroProducer( self.broker_properties, schema_registry=schema_registry)
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas self.client = AdminClient({"bootstrap.servers": BROKER_URL}) acks = (self.num_replicas - 1) if self.num_replicas > 2 else self.num_replicas self.broker_properties = { "bootstrap.servers": BROKER_URL, "client.id": f"producer_client_id_{self.topic_name.strip().replace(' ', '')}", "acks": acks } # If the topic does not already exist, try to create it if self.topic_name not in Producer.existing_topics: self.create_topic() Producer.existing_topics.add(self.topic_name) schema_registry = CachedSchemaRegistryClient( {"url": SCHEMA_REGISTRY_URL}) self.producer = AvroProducer(self.broker_properties, schema_registry=schema_registry)
async def consume(topic_name): """Consumes data from the Kafka Topic""" # # TODO: Create a CachedSchemaRegistryClient # schema_registry = CachedSchemaRegistryClient({"url": SCHEMA_REGISTRY_URL}) # # TODO: Use the Avro Consumer # See: https://docs.confluent.io/current/clients/confluent-kafka-python/index.html?highlight=loads#confluent_kafka.avro.AvroConsumer # c = AvroConsumer( { "bootstrap.servers": BROKER_URL, "group.id": "0" }, schema_registry=schema_registry, ) c.subscribe([topic_name]) while True: message = c.poll(1.0) if message is None: print("no message received by consumer") elif message.error() is not None: print(f"error from consumer {message.error()}") else: try: print(message.value()) except KeyError as e: print(f"Failed to unpack message {e}") await asyncio.sleep(1.0)
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas # configure kafka registry self.schema_registry = CachedSchemaRegistryClient(KAFKA_REGISTRY_URL) self.broker_properties = {'bootstrap.servers': KAFKA_BROKER_URL} self.client = AdminClient(self.broker_properties) # If the topic does not already exist, try to create it if self.topic_name not in Producer.existing_topics: self.create_topic() Producer.existing_topics.add(self.topic_name) self.producer = AvroProducer(self.broker_properties, default_key_schema=self.key_schema, default_value_schema=self.value_schema, schema_registry=self.schema_registry)
def __init__(self, config, schema_registry=None, reader_key_schema=None, reader_value_schema=None): sr_conf = { key.replace("schema.registry.", ""): value for key, value in config.items() if key.startswith("schema.registry") } if sr_conf.get("basic.auth.credentials.source") == 'SASL_INHERIT': sr_conf['sasl.mechanisms'] = config.get('sasl.mechanisms', '') sr_conf['sasl.username'] = config.get('sasl.username', '') sr_conf['sasl.password'] = config.get('sasl.password', '') ap_conf = { key: value for key, value in config.items() if not key.startswith("schema.registry") } if schema_registry is None: schema_registry = CachedSchemaRegistryClient(sr_conf) elif sr_conf.get("url", None) is not None: raise ValueError( "Cannot pass schema_registry along with schema.registry.url config" ) super(StringAvroConsumer, self).__init__(ap_conf) self._serializer = MessageSerializer(schema_registry, reader_key_schema, reader_value_schema)
def __init__( self, topic_name, key_schema, value_schema=None, num_partitions=1, num_replicas=1, ): """Initializes a Producer object with basic settings""" self.topic_name = topic_name self.key_schema = key_schema self.value_schema = value_schema self.num_partitions = num_partitions self.num_replicas = num_replicas self.broker_properties = { "bootstrap.servers": "PLAINTEXT://localhost:9092" } # If the topic does not already exist, try to create it if self.topic_name not in Producer.existing_topics: self.create_topic() Producer.existing_topics.add(self.topic_name) self.producer = AvroProducer( self.broker_properties, schema_registry=CachedSchemaRegistryClient('http://localhost:8081'), default_key_schema=self.key_schema, default_value_schema=self.value_schema )
def __init__( self, topic_name_pattern, message_handler, is_avro=True, offset_earliest=False, sleep_secs=1.0, consume_timeout=0.1, ): """Creates a consumer object for asynchronous use""" self.topic_name_pattern = topic_name_pattern self.message_handler = message_handler self.sleep_secs = sleep_secs self.consume_timeout = consume_timeout self.offset_earliest = offset_earliest schema_registry = CachedSchemaRegistryClient(SCHEMA_REGISTRY_URL) self.broker_properties = { "bootstrap.server": BROKER_URL, "group.id": "0", "auto.offset.reset": "earliest" } if is_avro is True: self.broker_properties["schema.registry.url"] = "http://localhost:8081" self.consumer = AvroConsumer( self.broker_properties, schema_registry=schema_registry ) else: self.consumer = Consumer(self.broker_properties) self.consumer.subscribe([self.topic_name_pattern])
def update(topic, schema_config, force=False): """Given a topic, update (or create) a schema""" client = CachedSchemaRegistryClient(schema_config) if topic == 'all': schema_files = Path(__file__).parent.glob('**/*.avsc') else: schema_files = Path(__file__).parent.glob(f'**/{topic}-*.avsc') for schema_file in schema_files: with open(schema_file) as f: schema_str = f.read() schema_dict = json.loads(schema_str) avro_schema = schema.Parse(schema_str) subject = schema_dict['namespace'].replace('.', '-') + '-' + schema_dict['name'] if force: client.update_compatibility('NONE', subject=subject) else: client.update_compatibility('BACKWARD', subject=subject) try: schema_id = client.register(subject, avro_schema) log.info(f'Added/updated {schema_file}\t Schema ID {schema_id}') except avro_error.ClientError as error: log.error(f'Error adding/updating {schema_file}: {error.message}')
async def produce(broker_url, topic, schema_registry_url, *, num_messages): schema_registry = CachedSchemaRegistryClient({"url": schema_registry_url}) conf = {"bootstrap.servers": broker_url, "client.id": socket.gethostname()} p = AvroProducer(conf, schema_registry=schema_registry) for _ in range(num_messages): p.produce(topic=topic, value=asdict(ClickEvent()), value_schema=ClickEvent.schema) await asyncio.sleep(1.0)
async def produce(topic_name): schema_registry = CachedSchemaRegistryClient({"url": SCHEMA_REGISTRY_URL}) p = AvroProducer({"bootstrap.servers": BROKER_URL}, schema_registry=schema_registry) while True: p.produce( topic=topic_name, value=asdict(ClickEvent()), value_schema=ClickEvent.schema ) await asyncio.sleep(1.0)
def __init__( self, topic_name_pattern, message_handler, is_avro=True, offset_earliest=False, sleep_secs=1.0, consume_timeout=0.1, ): """Creates a consumer object for asynchronous use""" self.topic_name_pattern = topic_name_pattern self.message_handler = message_handler self.sleep_secs = sleep_secs self.consume_timeout = consume_timeout self.offset_earliest = offset_earliest # # # TODO: Configure the broker properties below. Make sure to reference the project README # and use the Host URL for Kafka and Schema Registry! # # self.broker_properties = { "KAFKA_BROKER_URL": "PLAINTEXT://localhost:9092", "group.id": 'groupid' } if self.offset_earliest: self.broker_properties['auto.offset.reset'] = 'earliest' # TODO: Create the Consumer, using the appropriate type. if is_avro is True: self.broker_properties["SCHEMA_REGISTRY_URL"] = "http://localhost:8081" self.schema_registry = CachedSchemaRegistryClient(self.broker_properties.get("SCHEMA_REGISTRY_URL")) self.consumer = AvroConsumer( config={ "bootstrap.servers": self.broker_properties.get("KAFKA_BROKER_URL"), "group.id": self.broker_properties.get("group.id") }, schema_registry=self.schema_registry) else: self.consumer = Consumer( { "bootstrap.servers": self.broker_properties.get("KAFKA_BROKER_URL"), "group.id": self.broker_properties.get("group.id") }) # # # TODO: Configure the AvroConsumer and subscribe to the topics. Make sure to think about # how the `on_assign` callback should be invoked. # # self.consumer.subscribe([self.topic_name_pattern], on_assign=self.on_assign)
class AvroSerializer(Serializer): def __init__( self, schema_registry_url: str, auto_register_schemas: bool = True, subject_name_strategy: SubjectNameStrategy = SubjectNameStrategy. RecordNameStrategy, **kwargs): super().__init__(**kwargs) schema_registry_url = schema_registry_url self.schema_registry = CachedSchemaRegistryClient(schema_registry_url) self.auto_register_schemas = auto_register_schemas self.subject_name_strategy = subject_name_strategy self._serializer_impl = AvroSerDeBase(self.schema_registry) def _get_subject(self, topic: str, schema, is_key=False): if self.subject_name_strategy == SubjectNameStrategy.TopicNameStrategy: subject = topic + ('-key' if is_key else '-value') elif self.subject_name_strategy == SubjectNameStrategy.RecordNameStrategy: subject = schema.fullname elif self.subject_name_strategy == SubjectNameStrategy.TopicRecordNameStrategy: subject = '{}-{}'.format(topic, schema.fullname) else: raise ValueError('Unknown SubjectNameStrategy') return subject def _ensure_schema(self, topic: str, schema, is_key=False): subject = self._get_subject(topic, schema, is_key) if self.auto_register_schemas: schema_id = self.schema_registry.register(subject, schema) schema = self.schema_registry.get_by_id(schema_id) else: schema_id, schema, _ = self.schema_registry.get_latest_schema( subject) return schema_id, schema def serialize(self, value: AvroRecord, topic: str, is_key=False, **kwargs): schema_id, _ = self._ensure_schema(topic, value.schema, is_key) return self._serializer_impl.encode_record_with_schema_id( schema_id, value, is_key)
def __init__(self, group_id=None): """Initializes a Producer object with basic settings""" schema_registry = CachedSchemaRegistryClient({"url": KAFKA_SCHEMA_REGISTRY_URL}) producer_config = {"bootstrap.servers": KAFKA_BOOTSTRAP_SERVERS} if group_id: producer_config["client.id"] = group_id self.producer = AvroProducer(producer_config, schema_registry=schema_registry) logger.debug(f"Created producer with id {group_id}")
async def produce(topic_name): """Produces data into the Kafka Topic""" schema_registry = CachedSchemaRegistryClient(url=SCHEMA_REGISTRY_URL) p = AvroProducer(config={"bootstrap.servers": BROKER_URL}, schema_registry=schema_registry) while True: p.produce(topic=topic_name, value=asdict(ClickEvent()), value_schema=ClickEvent().serialize_schema()) await asyncio.sleep(1.0)