def test_contains(self): vocab = Vocabulary(unknown=None) vocab.update(text) self.assertTrue(text[-1] in vocab) self.assertFalse("~!@#" in vocab) self.assertEqual(text[-1] in vocab, vocab.has_word(text[-1])) self.assertEqual("~!@#" in vocab, vocab.has_word("~!@#"))
def test_index(self): vocab = Vocabulary() vocab.update(text) res = [vocab[w] for w in set(text)] self.assertEqual(len(res), len(set(res))) res = [vocab.to_index(w) for w in set(text)] self.assertEqual(len(res), len(set(res)))
def test_iteration(self): vocab = Vocabulary() text = [ "FastNLP", "works", "well", "in", "most", "cases", "and", "scales", "well", "in", "works", "well", "in", "most", "cases", "scales", "well" ] vocab.update(text) text = set(text) for word in vocab: self.assertTrue(word in text)
def test_iteration(self): vocab = Vocabulary(padding=None, unknown=None) text = [ "FastNLP", "works", "well", "in", "most", "cases", "and", "scales", "well", "in", "works", "well", "in", "most", "cases", "scales", "well" ] vocab.update(text) text = set(text) for word, idx in vocab: self.assertTrue(word in text) self.assertTrue(idx < len(vocab))
def test_warning(self): vocab = Vocabulary(max_size=len(set(text))) vocab.update(text) self.assertEqual(vocab.rebuild, True) print(len(vocab)) self.assertEqual(vocab.rebuild, False) vocab.update([ "hahahha", "hhh", "vvvv", "ass", "asss", "jfweiong", "eqgfeg", "feqfw" ]) # this will print a warning self.assertEqual(vocab.rebuild, True)
def test_additional_update(self): vocab = Vocabulary() vocab.update(text) _ = vocab["well"] self.assertEqual(vocab.rebuild, False) vocab.add("hahaha") self.assertEqual(vocab.rebuild, True) _ = vocab["hahaha"] self.assertEqual(vocab.rebuild, False) self.assertTrue("hahaha" in vocab)
def test_rebuild(self): # 测试build之后新加入词,原来的词顺序不变 vocab = Vocabulary() text = [str(idx) for idx in range(10)] vocab.update(text) for i in text: self.assertEqual(int(i) + 2, vocab.to_index(i)) indexes = [] for word, index in vocab: indexes.append((word, index)) vocab.add_word_lst([str(idx) for idx in range(10, 13)]) for idx, pair in enumerate(indexes): self.assertEqual(pair[1], vocab.to_index(pair[0])) for i in range(13): self.assertEqual(int(i) + 2, vocab.to_index(str(i)))
def test_update(self): vocab = Vocabulary() vocab.update(text) self.assertEqual(vocab.word_count, counter)
def test_to_word(self): vocab = Vocabulary() vocab.update(text) self.assertEqual( text, [vocab.to_word(idx) for idx in [vocab[w] for w in text]])
def test_len(self): vocab = Vocabulary(unknown=None, padding=None) vocab.update(text) self.assertEqual(len(vocab), len(counter))