def prepare_kafka_config(config: typing.Dict[str, typing.Any]) -> dict: connection_string = str(config.get("connection_string")) if not connection_string: raise Exception("No connection_string configured") parts = parse_event_hub_connection_string(connection_string) or {} if not parts: raise Exception(f"No parts found in connection_string") # fmt: off return { "bootstrap.servers": f"{parts.get('endpoint')}:9093", "security.protocol": config.get("security.protocol"), "ssl.ca.location": config.get("ssl.ca.location"), "sasl.mechanism": config.get("sasl.mechanism"), "sasl.username": "******", "sasl.password": config.get("connection_string"), "client.id": config.get("client.id"), "group.id": config.get("group.id"), "request.timeout.ms": config.get("request.timeout.ms"), "session.timeout.ms": config.get("session.timeout.ms"), "enable.auto.commit": config.get("enable.auto.commit"), "auto.commit.interval.ms": config.get("auto.commit.interval.ms"), "default.topic.config": config.get("default.topic.config"), "debug": config.get("debug"), "logger": logger_confluent, }
def prepare_kafka_config( config: typing.Dict[str, typing.Any] ) -> typing.Tuple[typing.Optional[dict], str, typing.Optional[str]]: connection_string = str(config.get("connection_string")) if not connection_string: return None, "", "No connection_string in kafka configuration" parts = parse_event_hub_connection_string(connection_string) or {} if not parts: return ( None, "", f"No parts found in kafka connection_string '{connection_string}'", ) topic = parts["entity_path"] or "" # fmt: off return { "bootstrap.servers": f"{parts.get('endpoint')}:9093", "security.protocol": config.get("security.protocol"), "ssl.ca.location": config.get("ssl.ca.location"), "sasl.mechanism": config.get("sasl.mechanism"), "sasl.username": "******", "sasl.password": config.get("connection_string"), "client.id": config.get("client.id"), "group.id": config.get("group.id"), "request.timeout.ms": config.get("request.timeout.ms"), "session.timeout.ms": config.get("session.timeout.ms"), "enable.auto.commit": config.get("enable.auto.commit"), "auto.commit.interval.ms": config.get("auto.commit.interval.ms"), "max.poll.interval.ms": config.get("max.poll.interval.ms"), "default.topic.config": config.get("default.topic.config"), "debug": config.get("debug"), "logger": logger_confluent, }, topic, None
def test_parse_event_hub_connection_string(): input = "Endpoint=sb://some.test.domain.com/;SharedAccessKeyName=some-key-name;SharedAccessKey=SomeKindOfKey2345=;EntityPath=some-topic" output = parse_event_hub_connection_string(input) # logger.info("RESULT:") # logger.info(pprint.pformat(output)) output_expected = { "endpoint": "some.test.domain.com", "entity_path": "some-topic", "shared_access_key": "SomeKindOfKey2345=", "shared_access_key_name": "some-key-name" } assert output == output_expected
def __init__(self, config: dict): # Consumer configuration # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md self.config = prepare_kafka_config(config) # Find our topic connection_string = str(config.get("connection_string")) if not connection_string: raise Exception("No connection_string configured") # self.config = prepare_kafka_config(config) parts = parse_event_hub_connection_string(connection_string) or {} if not parts: raise Exception(f"No parts found in connection_string") self.topic = parts.get("entity_path") if not self.topic: raise Exception("No topic configured") # Create Consumer instance self.consumer = Consumer(self.config) # Subscribe to topics self.consumer.subscribe([self.topic], on_assign=print_assignment)
def __init__(self, config: dict): # Producer configuration # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md # See https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka#prerequisites for SSL issues connection_string = str(config.get("connection_string")) if not connection_string: raise Exception("No connection_string configured") self.config = prepare_kafka_config(config) # Find our topic parts = parse_event_hub_connection_string(connection_string) or {} if not parts: raise Exception(f"No parts found in connection_string") self.topic = parts.get("entity_path") if not self.topic: raise Exception("No topic configured") # self._create_topics() # Create Producer instance self.producer = Producer(self.config) # Wait until all messages have been delivered logger.info(f"Waiting for {len(self.producer)} deliveries") self.producer.flush()