def from_kafka(cls, topic, partition=0, offset=0, **kwargs):
"""Creates an `IODataset` from kafka server with only a start offset.
Args:
topic: A `tf.string` tensor containing topic subscription.
partition: A `tf.int64` tensor containing the partition, by default 0.
offset: A `tf.int64` tensor containing the start offset, by default 0.
servers: An optional list of bootstrap servers, by default
`localhost:9092`.
configuration: An optional `tf.string` tensor containing
configurations in [Key=Value] format. There are three
types of configurations:
Global configuration: please refer to 'Global configuration properties'
in librdkafka doc. Examples include
["enable.auto.commit=false", "heartbeat.interval.ms=2000"]
Topic configuration: please refer to 'Topic configuration properties'
in librdkafka doc. Note all topic configurations should be
prefixed with `configuration.topic.`. Examples include
["conf.topic.auto.offset.reset=earliest"]
Dataset configuration: there is one configuration available,
`conf.timeout=milliseconds`: timeout value for Kafka Consumer to wait.
name: A name prefix for the IODataset (optional).
Returns:
A `StreamIODataset`.
"""
with tf.name_scope(kwargs.get("name", "IOFromKafka")):
return kafka_dataset_ops.KafkaStreamIODataset(
topic,
partition=partition,
offset=offset,
servers=kwargs.get("servers", None),
configuration=kwargs.get("configuration", None),
internal=True)
def from_kafka(cls,
topic,
partition=0,
offset=0,
servers=None,
configuration=None,
**kwargs):
"""Creates a `StreamIODataset` from kafka server with only a start offset.
Args:
topic: A `tf.string` tensor containing topic subscription.
partition: A `tf.int64` tensor containing the partition, by default 0.
offset: A `tf.int64` tensor containing the start offset, by default 0.
servers: An optional list of bootstrap servers, by default
`localhost:9092`.
configuration: An optional `tf.string` tensor containing
configurations in [Key=Value] format. There are three
types of configurations:
Global configuration: please refer to 'Global configuration properties'
in librdkafka doc. Examples include
["enable.auto.commit=false", "heartbeat.interval.ms=2000"]
Topic configuration: please refer to 'Topic configuration properties'
in librdkafka doc. Note all topic configurations should be
prefixed with `conf.topic.`. Examples include
["conf.topic.auto.offset.reset=earliest"]
Reference: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
name: A name prefix for the IODataset (optional).
Returns:
A `StreamIODataset`.
"""
with tf.name_scope(kwargs.get("name", "IOFromKafka")):
return kafka_dataset_ops.KafkaStreamIODataset(
topic,
partition=partition,
offset=offset,
servers=servers,
configuration=configuration,
internal=True,
)