def prepare_kafka_config(config: typing.Dict[str, typing.Any]) -> dict:
connection_string = str(config.get("connection_string"))
if not connection_string:
raise Exception("No connection_string configured")
parts = parse_event_hub_connection_string(connection_string) or {}
if not parts:
raise Exception(f"No parts found in connection_string")
# fmt: off
return {
"bootstrap.servers": f"{parts.get('endpoint')}:9093",
"security.protocol": config.get("security.protocol"),
"ssl.ca.location": config.get("ssl.ca.location"),
"sasl.mechanism": config.get("sasl.mechanism"),
"sasl.username": "******",
"sasl.password": config.get("connection_string"),
"client.id": config.get("client.id"),
"group.id": config.get("group.id"),
"request.timeout.ms": config.get("request.timeout.ms"),
"session.timeout.ms": config.get("session.timeout.ms"),
"enable.auto.commit": config.get("enable.auto.commit"),
"auto.commit.interval.ms": config.get("auto.commit.interval.ms"),
"default.topic.config": config.get("default.topic.config"),
"debug": config.get("debug"),
"logger": logger_confluent,
}
def prepare_kafka_config(
config: typing.Dict[str, typing.Any]
) -> typing.Tuple[typing.Optional[dict], str, typing.Optional[str]]:
connection_string = str(config.get("connection_string"))
if not connection_string:
return None, "", "No connection_string in kafka configuration"
parts = parse_event_hub_connection_string(connection_string) or {}
if not parts:
return (
None,
"",
f"No parts found in kafka connection_string '{connection_string}'",
)
topic = parts["entity_path"] or ""
# fmt: off
return {
"bootstrap.servers": f"{parts.get('endpoint')}:9093",
"security.protocol": config.get("security.protocol"),
"ssl.ca.location": config.get("ssl.ca.location"),
"sasl.mechanism": config.get("sasl.mechanism"),
"sasl.username": "******",
"sasl.password": config.get("connection_string"),
"client.id": config.get("client.id"),
"group.id": config.get("group.id"),
"request.timeout.ms": config.get("request.timeout.ms"),
"session.timeout.ms": config.get("session.timeout.ms"),
"enable.auto.commit": config.get("enable.auto.commit"),
"auto.commit.interval.ms": config.get("auto.commit.interval.ms"),
"max.poll.interval.ms": config.get("max.poll.interval.ms"),
"default.topic.config": config.get("default.topic.config"),
"debug": config.get("debug"),
"logger": logger_confluent,
}, topic, None
def test_parse_event_hub_connection_string():
input = "Endpoint=sb://some.test.domain.com/;SharedAccessKeyName=some-key-name;SharedAccessKey=SomeKindOfKey2345=;EntityPath=some-topic"
output = parse_event_hub_connection_string(input)
# logger.info("RESULT:")
# logger.info(pprint.pformat(output))
output_expected = {
"endpoint": "some.test.domain.com",
"entity_path": "some-topic",
"shared_access_key": "SomeKindOfKey2345=",
"shared_access_key_name": "some-key-name"
}
assert output == output_expected
def __init__(self, config: dict):
# Consumer configuration
# See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
self.config = prepare_kafka_config(config)
# Find our topic
connection_string = str(config.get("connection_string"))
if not connection_string:
raise Exception("No connection_string configured")
# self.config = prepare_kafka_config(config)
parts = parse_event_hub_connection_string(connection_string) or {}
if not parts:
raise Exception(f"No parts found in connection_string")
self.topic = parts.get("entity_path")
if not self.topic:
raise Exception("No topic configured")
# Create Consumer instance
self.consumer = Consumer(self.config)
# Subscribe to topics
self.consumer.subscribe([self.topic], on_assign=print_assignment)
def __init__(self, config: dict):
# Producer configuration
# See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
# See https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka#prerequisites for SSL issues
connection_string = str(config.get("connection_string"))
if not connection_string:
raise Exception("No connection_string configured")
self.config = prepare_kafka_config(config)
# Find our topic
parts = parse_event_hub_connection_string(connection_string) or {}
if not parts:
raise Exception(f"No parts found in connection_string")
self.topic = parts.get("entity_path")
if not self.topic:
raise Exception("No topic configured")
# self._create_topics()
# Create Producer instance
self.producer = Producer(self.config)
# Wait until all messages have been delivered
logger.info(f"Waiting for {len(self.producer)} deliveries")
self.producer.flush()