def target_topic_avro_consumer(unittest_config: Config, target_topic: Tuple[str, int]) -> AvroConsumer: consumer = AvroConsumer( { "group.id": "asdf", "enable.auto.commit": False, "enable.partition.eof": False, **unittest_config.create_confluent_config(include_schema_registry=True), } ) consumer.assign([TopicPartition(topic=target_topic[0], partition=i, offset=0) for i in range(target_topic[1])]) yield consumer consumer.close()
def test_confluent_config(config: Config): config.context_switch("context_5") expected_config = { "bootstrap.servers": "kafka:9094,kafka1:9094,kafka2:9094,kafka3:9094", "security.protocol": "SASL_SSL", "schema.registry.url": "http://schema-registry.example.com", "sasl.mechanisms": "PLAIN", "sasl.username": "******", "sasl.password": "******", "ssl.ca.location": "/my/ca.crt", "ssl.certificate.location": "/my/certificate.crt", "ssl.key.location": "/my/certificate.key", "ssl.key.password": "******", } actual_config = config.create_confluent_config(include_schema_registry=True) assert expected_config == actual_config
class Cluster: def __init__(self): self._config = Config() self.confluent_client = AdminClient( self._config.create_confluent_config()) self.pykafka_client = pykafka.client.KafkaClient( **self._config.create_pykafka_config(), broker_version="1.0.0") self.confluent_client.poll(timeout=1) self.__topic_controller = None @property def topic_controller(self) -> TopicController: if self.__topic_controller is None: self.__topic_controller = TopicController(self, self._config) return self.__topic_controller @property def bootstrap_servers(self): return self._config.bootstrap_servers def get_metadata(self): return self.confluent_client.list_topics(timeout=1) @property def brokers(self): metadata = self.confluent_client.list_topics(timeout=1) return sorted( [{ "id": broker.id, "host": broker.host, "port": broker.port } for broker in metadata.brokers.values()], key=operator.itemgetter("id"), ) def retrieve_config(self, config_type: ConfigResource.Type, id): requested_resources = [ConfigResource(config_type, str(id))] futures = self.confluent_client.describe_configs(requested_resources) (old_resource, future), = futures.items() future = ensure_kafka_futures_done([future]) result = future.result() return unpack_confluent_config(result)
def randomly_generated_consumer_groups(filled_topic, unittest_config: Config, prefix="") -> str: randomly_generated_consumer_group = prefix + "".join( random.choices(ascii_letters, k=8)) _config = unittest_config.create_confluent_config() _config.update({ "group.id": randomly_generated_consumer_group, "enable.auto.commit": False, "default.topic.config": { "auto.offset.reset": "latest" }, }) _consumer = confluent_kafka.Consumer(_config) _consumer.assign( [TopicPartition(topic=filled_topic.name, partition=0, offset=0)]) for i in range(2): msg = _consumer.consume(timeout=10)[0] _consumer.commit(msg, asynchronous=False) return randomly_generated_consumer_group
def consumer(topic_object: Topic, consumer_group: str, unittest_config: Config): _config = unittest_config.create_confluent_config() _config.update({ "group.id": consumer_group, "error_cb": log_error, # We need to commit offsets manually once we"re sure it got saved # to the sink "enable.auto.commit": False, "enable.partition.eof": False, # We need this to start at the last committed offset instead of the # latest when subscribing for the first time "default.topic.config": { "auto.offset.reset": "latest" }, }) _consumer = confluent_kafka.Consumer(_config) _consumer.assign( [TopicPartition(topic=topic_object.name, partition=0, offset=0)]) yield _consumer
def avro_producer(test_config: Config): producer_config = test_config.create_confluent_config() producer_config.update({"schema.registry.url": Config().schema_registry}) yield AvroProducer(producer_config)
def producer(test_config: Config): producer_config = test_config.create_confluent_config() yield Producer(producer_config)
def confluent_admin_client(test_config: Config) -> AdminClient: admin = AdminClient(test_config.create_confluent_config()) admin.poll(timeout=5) yield admin