def test_garbage_collection(self):
tv = TokenVector.make_token_vector(
["A", "", "B", "C", "D", "", "E", ""],
sep_size=1,
sentence_idx=0)
clean_tv = TokenVector([])
clean_tv.append(Token('A', sindex=0, eindex=0, sentence_idx=0))
clean_tv.append(Token('B', sindex=3, eindex=3, sentence_idx=0))
clean_tv.append(Token('C', sindex=5, eindex=5, sentence_idx=0))
clean_tv.append(Token('D', sindex=7, eindex=7, sentence_idx=0))
clean_tv.append(Token('E', sindex=10, eindex=10, sentence_idx=0))
tv.collect_garbage()
self.assertSequenceEqual(tv, clean_tv)
def test_make_token_vector(self):
made_tv = TokenVector.make_token_vector(
["A", "", "B", "C", "D", "", "E", ""],
sep_size=1,
sentence_idx=0)
real_tv = TokenVector([])
real_tv.append(Token('A', sindex=0, eindex=0, sentence_idx=0))
real_tv.append(Token('', sindex=2, eindex=2, sentence_idx=0))
real_tv.append(Token('B', sindex=3, eindex=3, sentence_idx=0))
real_tv.append(Token('C', sindex=5, eindex=5, sentence_idx=0))
real_tv.append(Token('D', sindex=7, eindex=7, sentence_idx=0))
real_tv.append(Token('', sindex=9, eindex=9, sentence_idx=0))
real_tv.append(Token('E', sindex=10, eindex=10, sentence_idx=0))
real_tv.append(Token('', sindex=12, eindex=12, sentence_idx=0))
self.assertSequenceEqual(made_tv, real_tv)
