def test_tokenize(self): document = "Matrix of TelePresence components continue" tokens = Tokenizer.tokenize(document) self.assertEqual(len(tokens),3)
def all_tokens(self): if len(self.tokens) > 0: return self.tokens self.tokens = Tokenizer.tokenize(self.content) return self.tokens