def from_kafka(cls,
topic,
partition=0,
servers=None, configuration=None, **kwargs):
"""Creates an `IOTensor` from a Kafka stream.
Args:
topic: A `tf.string` tensor containing topic subscription.
partition: A `tf.int64` tensor containing the partition, by default 0.
servers: An optional list of bootstrap servers, by default
`localhost:9092`.
configuration: An optional `tf.string` tensor containing
configurations in [Key=Value] format. There are three
types of configurations:
Global configuration: please refer to 'Global configuration properties'
in librdkafka doc. Examples include
["enable.auto.commit=false", "heartbeat.interval.ms=2000"]
Topic configuration: please refer to 'Topic configuration properties'
in librdkafka doc. Note all topic configurations should be
prefixed with `configuration.topic.`. Examples include
["conf.topic.auto.offset.reset=earliest"]
name: A name prefix for the IOTensor (optional).
Returns:
A `IOTensor`.
"""
with tf.name_scope(kwargs.get("name", "IOFromKafka")):
return kafka_io_tensor_ops.KafkaIOTensor(
topic=topic, partition=partition,
servers=servers, configuration=configuration, internal=True)
def from_kafka(cls, subscription, **kwargs):
"""Creates an `IOTensor` from a Kafka stream.
Args:
subscription: A `tf.string` tensor containing subscription,
in the format of [topic:partition:offset:length],
by default length is -1 for unlimited.
servers: An optional list of bootstrap servers, by default
`localhost:9092`.
configuration: An optional `tf.string` tensor containing
configurations in [Key=Value] format. There are three
types of configurations:
Global configuration: please refer to 'Global configuration properties'
in librdkafka doc. Examples include
["enable.auto.commit=false", "heartbeat.interval.ms=2000"]
Topic configuration: please refer to 'Topic configuration properties'
in librdkafka doc. Note all topic configurations should be
prefixed with `configuration.topic.`. Examples include
["conf.topic.auto.offset.reset=earliest"]
Dataset configuration: there are two configurations available,
`conf.eof=0|1`: if True, the KafkaDaset will stop on EOF (default).
`conf.timeout=milliseconds`: timeout value for Kafka Consumer to wait.
name: A name prefix for the IOTensor (optional).
Returns:
A `IOTensor`.
"""
with tf.name_scope(kwargs.get("name", "IOFromKafka")):
return kafka_io_tensor_ops.KafkaIOTensor(
subscription,
servers=kwargs.get("servers", None),
configuration=kwargs.get("configuration", None),
internal=True)
