def __init__(self, encoder=None, encoder_config=None):
"""Constructs a Text FeatureConnector.
Args:
encoder: `tfds.features.text.TextEncoder`, an encoder that can convert
text to integers. If None, the text will be utf-8 byte-encoded.
encoder_config: `tfds.features.text.TextEncoderConfig`, needed if
restoring from a file with `load_metadata`.
"""
if encoder and encoder_config:
raise ValueError(
"If encoder is provided, encoder_config must be None.")
if encoder:
encoder_config = text_lib.TextEncoderConfig(
encoder_cls=type(encoder), vocab_size=encoder.vocab_size)
elif encoder_config:
encoder = encoder_config.encoder
self._encoder = encoder
self._encoder_config = encoder_config
has_encoder = bool(encoder or self._encoder_cls)
super(Text, self).__init__(
shape=(None, ) if has_encoder else (),
dtype=tf.int64 if has_encoder else tf.string,
)
def __init__(self, encoder=None, encoder_config=None):
"""Constructs a Text FeatureConnector.
Args:
encoder: `tfds.features.text.TextEncoder`, an encoder that can convert
text to integers. If None, the text will be utf-8 byte-encoded.
encoder_config: `tfds.features.text.TextEncoderConfig`, needed if
restoring from a file with `load_metadata`.
"""
if encoder and encoder_config:
raise ValueError(
"If encoder is provided, encoder_config must be None.")
if encoder:
encoder_config = text_lib.TextEncoderConfig(
encoder_cls=type(encoder), vocab_size=encoder.vocab_size)
elif encoder_config:
encoder = encoder_config.encoder
self._encoder = encoder
self._encoder_config = encoder_config
has_encoder = bool(encoder or self._encoder_cls)
if has_encoder:
logging.warning(
"TFDS datasets with text encoding are deprecated and will be removed "
"in a future version. Instead, you should use the plain text version "
"and tokenize the text using `tensorflow_text` (See: "
"https://www.tensorflow.org/tutorials/tensorflow_text/intro#tfdata_example)"
)
super(Text, self).__init__(
shape=(None, ) if has_encoder else (),
dtype=tf.int64 if has_encoder else tf.string,
)