def test_filter_input_filter_vocab(): """Tests input filtering based on vocab frequency table and thresholds.""" input_tensor = tf.constant( [b"the", b"answer", b"to", b"life", b"and", b"universe"]) keys = tf.constant([b"and", b"life", b"the", b"to", b"universe"]) values = tf.constant([0, 1, 2, 3, 4], tf.dtypes.int64) vocab_freq_table = tf.lookup.StaticHashTable( tf.lookup.KeyValueTensorInitializer(keys, values), -1) # No vocab_freq_table specified - output should be the same as input no_table_output = skip_gram_ops._filter_input( input_tensor=input_tensor, vocab_freq_table=None, vocab_min_count=None, vocab_subsampling=None, corpus_size=None, seed=None, ) np.testing.assert_equal(input_tensor.numpy(), np.asanyarray(no_table_output)) # vocab_freq_table specified, but no vocab_min_count - output should # have filtered out tokens not in the table (b"answer"). table_output = skip_gram_ops._filter_input( input_tensor=input_tensor, vocab_freq_table=vocab_freq_table, vocab_min_count=None, vocab_subsampling=None, corpus_size=None, seed=None, ) np.testing.assert_equal( np.asanyarray([b"the", b"to", b"life", b"and", b"universe"]), table_output.numpy(), ) # vocab_freq_table and vocab_min_count specified - output should have # filtered out tokens whose frequencies are below the threshold # (b"and": 0, b"life": 1). threshold_output = skip_gram_ops._filter_input( input_tensor=input_tensor, vocab_freq_table=vocab_freq_table, vocab_min_count=2, vocab_subsampling=None, corpus_size=None, seed=None, ) np.testing.assert_equal(np.asanyarray([b"the", b"to", b"universe"]), threshold_output.numpy())
def test_filter_input_subsample_vocab(self): """Tests input filtering based on vocab subsampling.""" # The outputs are non-deterministic, so set random seed to help ensure # that the outputs remain constant for testing. tf.compat.v1.set_random_seed(42) input_tensor = tf.constant([ # keep_prob = (sqrt(30/(0.05*100)) + 1) * (0.05*100/30) = 0.57. b"the", b"answer", # Not in vocab. (Always discarded) b"to", # keep_prob = 0.75. b"life", # keep_prob > 1. (Always kept) b"and", # keep_prob = 0.48. b"universe" # Below vocab threshold of 3. (Always discarded) ]) keys = tf.constant([b"and", b"life", b"the", b"to", b"universe"]) values = tf.constant([40, 8, 30, 20, 2], tf.dtypes.int64) vocab_freq_table = tf.lookup.StaticHashTable( tf.lookup.KeyValueTensorInitializer(keys, values), -1) output = skip_gram_ops._filter_input( input_tensor=input_tensor, vocab_freq_table=vocab_freq_table, vocab_min_count=3, vocab_subsampling=0.05, corpus_size=tf.math.reduce_sum(values), seed=9) self.assertAllEqual([b"the", b"to", b"life", b"and"], output)