def from_kafka(cls,
topic,
partition=0,
offset=0,
tail=-1,
servers=None,
configuration=None,
**kwargs):
"""Creates an `IODataset` from kafka server with an offset range.
Args:
topic: A `tf.string` tensor containing topic subscription.
partition: A `tf.int64` tensor containing the partition, by default 0.
offset: A `tf.int64` tensor containing the start offset, by default 0.
tail: A `tf.int64` tensor containing the end offset, by default -1.
servers: An optional list of bootstrap servers, by default
`localhost:9092`.
configuration: An optional `tf.string` tensor containing
configurations in [Key=Value] format. There are three
types of configurations:
Global configuration: please refer to 'Global configuration properties'
in librdkafka doc. Examples include
["enable.auto.commit=false", "heartbeat.interval.ms=2000"]
Topic configuration: please refer to 'Topic configuration properties'
in librdkafka doc. Note all topic configurations should be
prefixed with `configuration.topic.`. Examples include
["conf.topic.auto.offset.reset=earliest"]
Dataset configuration: there are two configurations available,
`conf.eof=0|1`: if True, the KafkaDaset will stop on EOF (default).
`conf.timeout=milliseconds`: timeout value for Kafka Consumer to wait.
name: A name prefix for the IODataset (optional).
Returns:
A `IODataset`.
"""
with tf.name_scope(kwargs.get("name", "IOFromKafka")):
return kafka_dataset_ops.KafkaIODataset(
topic,
partition=partition,
offset=offset,
tail=tail,
servers=servers,
configuration=configuration,
internal=True)
def from_kafka(cls,
topic,
partition=0,
start=0,
stop=-1,
servers=None,
configuration=None,
**kwargs):
"""Creates an `IODataset` from kafka server with an offset range.
Args:
topic: A `tf.string` tensor containing topic subscription.
partition: A `tf.int64` tensor containing the partition, by default 0.
start: A `tf.int64` tensor containing the start offset, by default 0.
stop: A `tf.int64` tensor containing the end offset, by default -1.
servers: An optional list of bootstrap servers, by default
`localhost:9092`.
configuration: An optional `tf.string` tensor containing
configurations in [Key=Value] format. There are three
types of configurations:
Global configuration: please refer to 'Global configuration properties'
in librdkafka doc. Examples include
["enable.auto.commit=false", "heartbeat.interval.ms=2000"]
Topic configuration: please refer to 'Topic configuration properties'
in librdkafka doc. Note all topic configurations should be
prefixed with `conf.topic.`. Examples include
["conf.topic.auto.offset.reset=earliest"]
Reference: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
name: A name prefix for the IODataset (optional).
Returns:
A `IODataset`.
"""
with tf.name_scope(kwargs.get("name", "IOFromKafka")):
return kafka_dataset_ops.KafkaIODataset(
topic,
partition=partition,
start=start,
stop=stop,
servers=servers,
configuration=configuration,
internal=True,
)