def open(
url: str,
mode: str = 'r',
auth: Optional[auth.SASLAuth] = None,
start_at: consumer.ConsumerStartPosition = consumer.ConsumerStartPosition.
EARLIEST, # noqa: E501
read_forever: bool = True,
) -> Union[producer.Producer, Iterable[confluent_kafka.Message]]:
group_id, broker_addresses, topics = kafka.parse_kafka_url(url)
logger.debug("connecting to addresses=%s group_id=%s topics=%s",
broker_addresses, group_id, topics)
if mode == "r":
if group_id is None:
raise ValueError("group ID must be set when in reader mode")
return _open_consumer(group_id, broker_addresses, topics, auth,
start_at, read_forever)
elif mode == "w":
if len(topics) != 1:
raise ValueError("must specify exactly one topic in write mode")
if group_id is not None:
warnings.warn(
"group ID has no effect when opening a stream in write mode")
if start_at is not consumer.ConsumerStartPosition.EARLIEST:
warnings.warn(
"start_at has no effect when opening a stream in write mode")
if read_forever is not True:
warnings.warn(
"read_forever has no effect when opening a stream in write mode"
)
return _open_producer(broker_addresses, topics[0], auth)
else:
raise ValueError("mode must be either 'w' or 'r'")
def list_topics(url: str, auth: Union[bool, Auth] = True):
"""List the accessible topics on the Kafka broker referred to by url.
Args:
url: The Kafka broker URL. Only one broker may be specified. Topics
may be specified, in which case only topics in the intersection
of the set specified by the URL and actually present on the broker
will be returned. If a userinfo component is present in the URL and
auth is True, it will be treated as a hint to the automatic auth
lookup.
auth: A `bool` or :class:`Auth <hop.auth.Auth>` instance. Defaults to
loading from :meth:`auth.load_auth <hop.auth.load_auth>` if set to
True. To disable authentication, set to False. If a username is
specified as part of url but auth is a :class:`Auth <hop.auth.Auth>`
instance the url information will be ignored.
Returns:
A dictionary mapping topic names to
:class:`confluent_kafka.admin.TopicMetadata` instances.
Raises:
ValueError: If more than one broker is specified.
"""
username, broker_addresses, query_topics = kafka.parse_kafka_url(url)
if len(broker_addresses) > 1:
raise ValueError("Multiple broker addresses are not supported")
user_auth = None
if auth is True:
credentials = load_auth()
user_auth = select_matching_auth(credentials, broker_addresses[0],
username)
elif auth is not False:
user_auth = auth
group_id = _generate_group_id(username, 10)
config = {
"bootstrap.servers": ",".join(broker_addresses),
"error_cb": errors.log_client_errors,
"group.id": group_id,
}
if user_auth is not None:
config.update(user_auth())
consumer = confluent_kafka.Consumer(config)
valid_topics = {}
if query_topics is not None:
for topic in query_topics:
topic_data = consumer.list_topics(topic=topic).topics
for topic in topic_data.keys():
if topic_data[topic].error is None:
valid_topics[topic] = topic_data[topic]
else:
topic_data = consumer.list_topics().topics
valid_topics = {t: d for t, d in topic_data.items() if d.error is None}
return valid_topics
def open(self, url, mode="r"):
"""Opens a connection to an event stream.
Args:
url: Sets the broker URL to connect to.
mode: Read ('r') or write ('w') from the stream.
Returns:
An open connection to the client, either a :class:`Producer` instance
in write mode or a :class:`Consumer` instance in read mode.
Raises:
ValueError: If the mode is not set to read/write or if more than
one topic is specified in write mode.
"""
group_id, broker_addresses, topics = kafka.parse_kafka_url(url)
logger.debug("connecting to addresses=%s group_id=%s topics=%s",
broker_addresses, group_id, topics)
if topics is None:
raise ValueError("no topic(s) specified in kafka URL")
if mode == "w":
if len(topics) != 1:
raise ValueError(
"must specify exactly one topic in write mode")
if group_id is not None:
warnings.warn(
"group ID has no effect when opening a stream in write mode"
)
return Producer(broker_addresses, topics[0], auth=self.auth)
elif mode == "r":
if group_id is None:
username = self.auth.username if hasattr(
self.auth, "username") else None
group_id = _generate_group_id(username, 10)
logger.info(
f"group ID not specified, generating a random group ID: {group_id}"
)
return Consumer(
group_id,
broker_addresses,
topics,
start_at=self.start_at,
auth=self.auth,
read_forever=self.persist,
)
else:
raise ValueError("mode must be either 'w' or 'r'")
def test_empty_scheme(self):
with self.assertRaises(ValueError):
kafka.parse_kafka_url("://group@broker")
def test_fully_populated(self):
group, brokers, topic = kafka.parse_kafka_url(
"kafka://group@broker/topic")
self.assertEqual(group, "group")
self.assertListEqual(brokers, ["broker"])
self.assertEqual(topic, ["topic"])
def test_no_topic_trailing_slash(self):
group, brokers, topic = kafka.parse_kafka_url("kafka://group@broker/")
self.assertEqual(group, "group")
self.assertListEqual(brokers, ["broker"])
self.assertIs(topic, None)
def test_no_group(self):
group, brokers, topic = kafka.parse_kafka_url("kafka://broker/topic")
self.assertIs(group, None)
self.assertListEqual(brokers, ["broker"])
self.assertEqual(topic, ["topic"])
def test_hostport_address(self):
group, brokers, topic = kafka.parse_kafka_url(
"kafka://[email protected]:9092/topic")
self.assertEqual(group, "group")
self.assertListEqual(brokers, ["127.0.0.1:9092"])
self.assertEqual(topic, ["topic"])
def test_multiple_topics(self):
group, brokers, topic = kafka.parse_kafka_url(
"kafka://group@broker1,broker2/topic1,topic2")
self.assertEqual(group, "group")
self.assertListEqual(brokers, ["broker1", "broker2"])
self.assertEqual(topic, ["topic1", "topic2"])
def test_multiple_broker(self):
group, brokers, topic = kafka.parse_kafka_url(
"kafka://broker1,broker2/topic")
self.assertIs(group, None)
self.assertListEqual(brokers, ["broker1", "broker2"])
self.assertEqual(topic, ["topic"])
def open(self, url, mode="r", group_id=None, ignoretest=True, **kwargs):
"""Opens a connection to an event stream.
Args:
url: Sets the broker URL to connect to.
mode: Read ('r') or write ('w') from the stream.
group_id: The consumer group ID from which to read.
Generated automatically if not specified.
ignoretest: When True, read mode will silently discard
test messages.
Returns:
An open connection to the client, either a :class:`Producer` instance
in write mode or a :class:`Consumer` instance in read mode.
Raises:
ValueError: If the mode is not set to read/write, if more than
one topic is specified in write mode, or if more than one broker is specified
"""
username, broker_addresses, topics = kafka.parse_kafka_url(url)
if len(broker_addresses) > 1:
raise ValueError("Multiple broker addresses are not supported")
logger.debug("connecting to addresses=%s username=%s topics=%s",
broker_addresses, group_id, topics)
if topics is None:
raise ValueError("no topic(s) specified in kafka URL")
if self.auth is not None:
credential = select_matching_auth(self.auth, broker_addresses[0],
username)
else:
credential = None
if mode == "w":
if len(topics) != 1:
raise ValueError(
"must specify exactly one topic in write mode")
if group_id is not None:
warnings.warn(
"group ID has no effect when opening a stream in write mode"
)
return Producer(broker_addresses,
topics[0],
auth=credential,
**kwargs)
elif mode == "r":
if group_id is None:
username = credential.username if credential is not None else None
group_id = _generate_group_id(username, 10)
logger.info(
f"group ID not specified, generating a random group ID: {group_id}"
)
return Consumer(
group_id,
broker_addresses,
topics,
start_at=self.start_at,
auth=credential,
read_forever=not self.until_eos,
ignoretest=ignoretest,
**kwargs,
)
else:
raise ValueError("mode must be either 'w' or 'r'")