def get_static_table(tmpdir,
vocab_list,
mask_token=None,
dtype=tf.string,
oov_tokens=None):
vocabulary_file = os.path.join(tmpdir, "tmp_vocab.txt")
if dtype == tf.string:
with open(vocabulary_file, "w") as f:
f.write("\n".join(vocab_list) + "\n")
else:
with open(vocabulary_file, "w") as f:
f.write("\n".join([str(v) for v in vocab_list]) + "\n")
offset = ((0 if mask_token is None else 1) +
(len(oov_tokens) if oov_tokens is not None else 0))
init = tf.lookup.TextFileInitializer(vocabulary_file,
dtype,
tf.lookup.TextFileIndex.WHOLE_LINE,
tf.int64,
tf.lookup.TextFileIndex.LINE_NUMBER,
value_index_offset=offset)
if tf.executing_eagerly():
table = tf.lookup.StaticHashTable(init, default_value=-7)
else:
table = tf.compat.v1.lookup.StaticHashTable(init, default_value=-7)
return table_utils.TableHandler(table,
oov_tokens,
mask_token=mask_token,
use_v1_apis=(not tf.executing_eagerly()))
def get_table(dtype=tf.string, oov_tokens=None):
table = lookup_ops.MutableHashTable(key_dtype=dtype,
value_dtype=tf.int64,
default_value=-7,
name="index_table")
return table_utils.TableHandler(table,
oov_tokens,
use_v1_apis=(not tf.executing_eagerly()))
def __init__(self,
max_tokens,
num_oov_indices,
mask_token,
oov_token,
vocabulary=None,
invert=False,
output_mode=INT,
sparse=False,
pad_to_max_tokens=False,
**kwargs):
# If max_tokens is set, the value must be greater than 1 - otherwise we
# are creating a 0-element vocab, which doesn't make sense.
if max_tokens is not None and max_tokens <= 1:
raise ValueError("If set, `max_tokens` must be greater than 1. "
"You passed {}".format(max_tokens))
if num_oov_indices < 0:
raise ValueError(
"`num_oov_indices` must be greater than or equal to 0. "
"You passed {}".format(num_oov_indices))
# 'output_mode' must be one of (INT, BINARY, COUNT, TFIDF)
layer_utils.validate_string_arg(output_mode,
allowable_strings=(INT, BINARY, COUNT,
TFIDF),
layer_name=self.__class__.__name__,
arg_name="output_mode")
self.invert = invert
self.max_tokens = max_tokens
self.num_oov_indices = num_oov_indices
self.oov_token = oov_token
self.mask_token = mask_token
self.output_mode = output_mode
self.sparse = sparse
self.pad_to_max_tokens = pad_to_max_tokens
self._called = False
self._num_special_tokens = self.num_oov_indices
if self.mask_token is not None:
self._num_special_tokens += 1
self._vocab_size = 0
# We need to keep track our current vocab size outside of our layer weights
# to support a static output shape when `output_mode != INT`. The bincount
# ops do not set shape on their outputs, which means we have to set it
# ourselves. We persist the current vocab size as a hidden part of the
# config when serializing our model.
if "vocab_size" in kwargs:
self._vocab_size = kwargs["vocab_size"]
del kwargs["vocab_size"]
# If there is only one OOV bucket, we can determine the OOV value (either 0
# or 1 depending on whether 0 is reserved) and set that as the default
# value of the index_lookup table. If we hav multiple OOV values, we need to
# do a further hashing step; to make this easier, we set the OOV value to
# -1. (This lets us do a vectorized add and cast to boolean to determine
# locations where we need to do extra hashing.)
if self.num_oov_indices == 1:
self._oov_value = 0 if mask_token is None else 1
else:
self._oov_value = -1
if max_tokens is not None:
available_vocab_size = max_tokens - self._num_special_tokens
else:
available_vocab_size = None
super(IndexLookup, self).__init__(combiner=_IndexLookupCombiner(
vocab_size=available_vocab_size,
mask_value=mask_token,
oov_value=oov_token,
compute_idf=(output_mode == TFIDF)),
**kwargs)
# We need to save the key dtype so that we know if we're expecting int64
# keys. If we are, we will cast int32 inputs to int64 as well.
if invert:
self._key_dtype = tf.int64
self._value_dtype = self.dtype
oov_value = self.oov_token
oov_indices = None
else:
self._key_dtype = self.dtype
self._value_dtype = tf.int64
oov_value = self._oov_value
if self.num_oov_indices <= 1:
oov_indices = None
else:
oov_start = 1 if mask_token is not None else 0
oov_end = oov_start + num_oov_indices
oov_indices = list(range(oov_start, oov_end))
if vocabulary is not None and isinstance(
vocabulary, tf.lookup.TextFileInitializer):
self._table = self._static_table_class()(vocabulary,
default_value=oov_value)
self._table_handler = table_utils.TableHandler(
table=self._table,
mask_token=mask_token,
oov_tokens=oov_indices,
use_v1_apis=self._use_v1_apis())
self.max_tokens = (self._table_handler.table_size() +
self.num_oov_indices +
(0 if mask_token is None else 1))
else:
self._table = lookup_ops.MutableHashTable(
key_dtype=self._key_dtype,
value_dtype=self._value_dtype,
default_value=oov_value,
name=(self._name + "_index_table"))
self._table_handler = table_utils.TableHandler(
table=self._table,
oov_tokens=oov_indices,
use_v1_apis=self._use_v1_apis())
if vocabulary is not None:
self.set_vocabulary(vocabulary)
if self.output_mode == TFIDF:
# The TF-IDF weight may have a (None,) tensorshape. This creates
# a 1D variable with arbitrary shape, which we can assign any weight to
# so long as it has 1 dimension. In order to properly initialize this
# weight in Keras, we need to provide a custom callable initializer which
# does not depend on the shape of the weight (as all other initializers
# do) since the weight is not known. Hence the lambda shape, dtype: [0].
if not self.pad_to_max_tokens or max_tokens is None:
initializer = lambda shape, dtype: [0]
else:
initializer = tf.compat.v1.zeros_initializer
# We are adding these here instead of in build() since they do not depend
# on the input shape at all.
idf_shape = (max_tokens, ) if self.pad_to_max_tokens else (None, )
self.tf_idf_weights = self._add_state_variable(
name="idf",
shape=tf.TensorShape(idf_shape),
dtype=K.floatx(),
initializer=initializer)
tracked_table = self._add_trackable(self._table, trainable=False)
# This is a workaround for summary() on this layer. Because the table is
# not mutable during training, the effective number of parameters (and so
# the weight shape) is 0; we add this as an attr so that the parameter
# counting code in the Model object doesn't throw an attribute error.
tracked_table.shape = tf.TensorShape((0, ))
def __init__(self,
max_tokens,
num_oov_indices,
mask_token,
oov_token,
vocabulary=None,
invert=False,
output_mode=INT,
sparse=False,
pad_to_max_tokens=False,
**kwargs):
# If max_tokens is set, the value must be greater than 1 - otherwise we
# are creating a 0-element vocab, which doesn't make sense.
if max_tokens is not None and max_tokens <= 1:
raise ValueError("If set, `max_tokens` must be greater than 1. "
"You passed {}".format(max_tokens))
if num_oov_indices < 0:
raise ValueError(
"`num_oov_indices` must be greater than or equal to 0. "
"You passed {}".format(num_oov_indices))
# 'output_mode' must be one of (INT, BINARY, COUNT, TFIDF)
layer_utils.validate_string_arg(output_mode,
allowable_strings=(INT, BINARY, COUNT,
TFIDF),
layer_name=self.__class__.__name__,
arg_name="output_mode")
if invert and output_mode != INT:
raise ValueError(
"`output_mode` must be {} when `invert` is true. You "
"passed {}".format(INT, output_mode))
self.invert = invert
self.max_tokens = max_tokens
self.num_oov_indices = num_oov_indices
self.oov_token = oov_token
self.output_mode = output_mode
self.sparse = sparse
self.pad_to_max_tokens = pad_to_max_tokens
self._called = False
# A note on vocab_size: we need to always keep a non-Tensor representation
# of vocab_size around to use in graph building. Because we might be
# in a tf.function, we can't rely on evaluating the actual tables to
# find the value either.
self._vocab_size = None
# We need to keep track our current vocab size outside of our layer weights
# to support a static output shape when `output_mode != INT`. The bincount
# ops do not set shape on their outputs, which means we have to set it
# ourselves. We persist the current vocab size as a hidden part of the
# config when serializing our model.
if "vocabulary_size" in kwargs:
self._vocab_size = kwargs["vocabulary_size"]
del kwargs["vocabulary_size"]
restore_from_static_table = kwargs.pop("has_static_table", False)
# Make sure the mask token is truly of the dtype we want. We can ignore
# strings here, because they have only one dtype.
if mask_token is not None:
dtype = kwargs["dtype"]
if dtype == tf.int32:
mask_token = np.int32(mask_token)
elif dtype == tf.int64:
mask_token = np.int64(mask_token)
self.mask_token = mask_token
if max_tokens is not None:
available_vocab_size = max_tokens - self._token_start_index()
else:
available_vocab_size = None
super(IndexLookup, self).__init__(combiner=_IndexLookupCombiner(
vocab_size=available_vocab_size,
mask_value=mask_token,
oov_value=oov_token,
compute_idf=(output_mode == TFIDF)),
**kwargs)
# We need to save the key dtype so that we know if we're expecting int64
# keys. If we are, we will cast int32 inputs to int64 as well.
if invert:
self._key_dtype = tf.int64
self._value_dtype = self.dtype
self._mask_key = 0
self._mask_value = mask_token
key_index = tf.lookup.TextFileIndex.LINE_NUMBER
value_index = tf.lookup.TextFileIndex.WHOLE_LINE
default_value = self.oov_token
oov_indices = None
else:
self._key_dtype = self.dtype
self._value_dtype = tf.int64
self._mask_key = mask_token
key_index = tf.lookup.TextFileIndex.WHOLE_LINE
value_index = tf.lookup.TextFileIndex.LINE_NUMBER
# Masks should map to 0 for int output and be dropped otherwise. Max ints
# will be dropped from the bincount op.
self._mask_value = 0 if self.output_mode == INT else tf.int64.max
oov_start = self._oov_start_index()
token_start = self._token_start_index()
if self.num_oov_indices == 0:
# If there are no OOV indices, we map OOV tokens to -1 for int output
# and drop them from bagged output. Max ints will be dropped from the
# bincount op.
default_value = -1 if self.output_mode == INT else tf.int64.max
oov_indices = None
elif self.num_oov_indices == 1:
# If there is only one OOV index, we can set that index as the default
# value of the index_lookup table.
default_value = oov_start
oov_indices = None
else:
# If we hav multiple OOV values, we need to do a further hashing step;
# to make this easier, we set the OOV value to -1. (This lets us do a
# vectorized add and cast to boolean to determine locations where we
# need to do extra hashing.)
default_value = -1
oov_indices = list(range(oov_start, token_start))
self._static_vocabulary_path = None
has_vocab_path = (vocabulary is not None
and isinstance(vocabulary, str))
if has_vocab_path or restore_from_static_table:
self._has_static_table = True
if vocabulary is None:
# If we're restoring a layer that was saved with a static table
# initializer, we create a fake initializer object to let the code
# progress. The savedmodel restoration code will handle restoring
# the actual data.
initializer = _NullInitializer(self._key_dtype,
self._value_dtype)
else:
if not os.path.exists(vocabulary):
raise ValueError("Vocabulary file %s does not exist." %
(vocabulary, ))
self._static_vocabulary_path = vocabulary
num_tokens = table_utils.num_tokens_in_file(vocabulary)
self._vocab_size = self._token_start_index() + num_tokens
initializer = tf.lookup.TextFileInitializer(
filename=vocabulary,
key_dtype=self._key_dtype,
key_index=key_index,
value_dtype=self._value_dtype,
value_index=value_index,
value_index_offset=self._token_start_index())
self._table = self._static_table_class()(
initializer, default_value=default_value)
self._table_handler = table_utils.TableHandler(
table=self._table,
mask_token=self._mask_key,
mask_value=self._mask_value,
oov_tokens=oov_indices,
use_v1_apis=self._use_v1_apis())
tracked_table = self._add_trackable(self._table, trainable=False)
else:
self._has_static_table = False
self._table = lookup_ops.MutableHashTable(
key_dtype=self._key_dtype,
value_dtype=self._value_dtype,
default_value=default_value,
name=(self._name + "_index_table"))
self._table_handler = table_utils.TableHandler(
table=self._table,
oov_tokens=oov_indices,
use_v1_apis=self._use_v1_apis())
if vocabulary is not None:
self.set_vocabulary(vocabulary)
tracked_table = self._add_trackable(self._table, trainable=False)
if self.output_mode == TFIDF:
# The TF-IDF weight may have a (None,) tensorshape. This creates
# a 1D variable with arbitrary shape, which we can assign any weight to
# so long as it has 1 dimension. In order to properly initialize this
# weight in Keras, we need to provide a custom callable initializer which
# does not depend on the shape of the weight (as all other initializers
# do) since the weight is not known. Hence the lambda shape, dtype: [0].
if not self.pad_to_max_tokens or max_tokens is None:
initializer = lambda shape, dtype: [0]
else:
initializer = tf.compat.v1.zeros_initializer
# We are adding these here instead of in build() since they do not depend
# on the input shape at all.
idf_shape = (max_tokens, ) if self.pad_to_max_tokens else (None, )
self.tf_idf_weights = self._add_state_variable(
name="idf",
shape=tf.TensorShape(idf_shape),
dtype=backend.floatx(),
initializer=initializer)
# This is a workaround for summary() on this layer. Because the table is
# not mutable during training, the effective number of parameters (and so
# the weight shape) is 0; we add this as an attr so that the parameter
# counting code in the Model object doesn't throw an attribute error.
tracked_table.shape = tf.TensorShape((0, ))
def __init__(self,
max_tokens,
num_oov_indices,
mask_token,
oov_token,
vocabulary=None,
invert=False,
output_mode=INT,
sparse=False,
**kwargs):
# If max_tokens is set, the value must be greater than 1 - otherwise we
# are creating a 0-element vocab, which doesn't make sense.
if max_tokens is not None and max_tokens <= 1:
raise ValueError("If set, max_tokens must be greater than 1. "
"You passed %s" % (max_tokens,))
if num_oov_indices < 0:
raise ValueError("`num_oov_indices` must be greater than 0. You passed "
"%s" % (num_oov_indices,))
if invert and num_oov_indices != 1:
raise ValueError("`num_oov_tokens` must be 1 when `invert` is True.")
# 'output_mode' must be one of (INT, BINARY, COUNT)
layer_utils.validate_string_arg(
output_mode,
allowable_strings=(INT, BINARY, COUNT),
layer_name=self.__class__.__name__,
arg_name="output_mode")
self.invert = invert
self.max_tokens = max_tokens
self.num_oov_indices = num_oov_indices
self.oov_token = oov_token
self.mask_token = mask_token
self.output_mode = output_mode
self.sparse = sparse
# If there is only one OOV bucket, we can determine the OOV value (either 0
# or 1 depending on whether 0 is reserved) and set that as the default
# value of the index_lookup table. If we hav multiple OOV values, we need to
# do a further hashing step; to make this easier, we set the OOV value to
# -1. (This lets us do a vectorized add and cast to boolean to determine
# locations where we need to do extra hashing.)
if self.num_oov_indices == 1:
self._oov_value = 0 if mask_token is None else 1
else:
self._oov_value = -1
if max_tokens is not None:
num_mask_tokens = (0 if mask_token is None else 1)
vocab_size = max_tokens - (num_oov_indices + num_mask_tokens)
else:
vocab_size = None
super(IndexLookup, self).__init__(
combiner=_IndexLookupCombiner(vocab_size, self.mask_token), **kwargs)
self._output_dtype = tf.int64
# We need to save the key dtype so that we know if we're expecting int64
# keys. If we are, we will cast int32 inputs to int64 as well.
if invert:
self._key_dtype = self._output_dtype
value_dtype = self.dtype
oov_value = self.oov_token
else:
self._key_dtype = self.dtype
value_dtype = self._output_dtype
oov_value = self._oov_value
self._table = lookup_ops.MutableHashTable(
key_dtype=self._key_dtype,
value_dtype=value_dtype,
default_value=oov_value,
name=(self._name + "_index_table"))
tracked_table = self._add_trackable(self._table, trainable=False)
# This is a workaround for summary() on this layer. Because the table is
# not mutable during training, the effective number of parameters (and so
# the weight shape) is 0; we add this as an attr so that the parameter
# counting code in the Model object doesn't throw an attribute error.
tracked_table.shape = tf.TensorShape((0,))
if self.num_oov_indices <= 1:
oov_indices = None
else:
oov_start = 1 if mask_token is not None else 0
oov_end = oov_start + num_oov_indices
oov_indices = list(range(oov_start, oov_end))
self._table_handler = table_utils.TableHandler(
table=self._table,
oov_tokens=oov_indices,
use_v1_apis=self._use_v1_apis())
if vocabulary is not None:
self.set_vocabulary(vocabulary)
def get_table(dtype=tf.string, oov_tokens=None):
table = lookup_ops.MutableHashTable(key_dtype=dtype,
value_dtype=tf.int64,
default_value=-7,
name="index_table")
return table_utils.TableHandler(table, oov_tokens)
