def test_namespaces(self): data_indexer = DataIndexer() initial_vocab_size = data_indexer.get_vocab_size() word_index = data_indexer.add_word_to_index("word", namespace='1') assert "word" in data_indexer.words_in_index(namespace='1') assert data_indexer.get_word_index("word", namespace='1') == word_index assert data_indexer.get_word_from_index(word_index, namespace='1') == "word" assert data_indexer.get_vocab_size( namespace='1') == initial_vocab_size + 1 # Now add it again, in a different namespace and a different word, and make sure it's like # new. word2_index = data_indexer.add_word_to_index("word2", namespace='2') word_index = data_indexer.add_word_to_index("word", namespace='2') assert "word" in data_indexer.words_in_index(namespace='2') assert "word2" in data_indexer.words_in_index(namespace='2') assert data_indexer.get_word_index("word", namespace='2') == word_index assert data_indexer.get_word_index("word2", namespace='2') == word2_index assert data_indexer.get_word_from_index(word_index, namespace='2') == "word" assert data_indexer.get_word_from_index(word2_index, namespace='2') == "word2" assert data_indexer.get_vocab_size( namespace='2') == initial_vocab_size + 2
def test_set_from_file(self): # pylint: disable=protected-access vocab_filename = self.TEST_DIR + 'vocab_file' with codecs.open(vocab_filename, 'w', 'utf-8') as vocab_file: vocab_file.write('<S>\n') vocab_file.write('</S>\n') vocab_file.write('<UNK>\n') vocab_file.write('a\n') vocab_file.write('word\n') vocab_file.write('another\n') data_indexer = DataIndexer() data_indexer.set_from_file(vocab_filename, oov_token="<UNK>") assert data_indexer._oov_token == "<UNK>" assert data_indexer.get_word_index("random string") == 3 assert data_indexer.get_word_index("<S>") == 1 assert data_indexer.get_word_index("</S>") == 2 assert data_indexer.get_word_index("<UNK>") == 3 assert data_indexer.get_word_index("a") == 4 assert data_indexer.get_word_index("word") == 5 assert data_indexer.get_word_index("another") == 6 assert data_indexer.get_word_from_index( 0) == data_indexer._padding_token assert data_indexer.get_word_from_index(1) == "<S>" assert data_indexer.get_word_from_index(2) == "</S>" assert data_indexer.get_word_from_index(3) == "<UNK>" assert data_indexer.get_word_from_index(4) == "a" assert data_indexer.get_word_from_index(5) == "word" assert data_indexer.get_word_from_index(6) == "another"
def test_add_word_to_index_gives_consistent_results(self): data_indexer = DataIndexer() initial_vocab_size = data_indexer.get_vocab_size() word_index = data_indexer.add_word_to_index("word") assert "word" in data_indexer.words_in_index() assert data_indexer.get_word_index("word") == word_index assert data_indexer.get_word_from_index(word_index) == "word" assert data_indexer.get_vocab_size() == initial_vocab_size + 1 # Now add it again, and make sure nothing changes. data_indexer.add_word_to_index("word") assert "word" in data_indexer.words_in_index() assert data_indexer.get_word_index("word") == word_index assert data_indexer.get_word_from_index(word_index) == "word" assert data_indexer.get_vocab_size() == initial_vocab_size + 1