def test_join(self): tok = WhitespaceTokenizer() self.assertEquals("", tok.join([])) self.assertEquals("this is a test", tok.join(["this", "is", "a", "test"]))
def test_split(self): tok = WhitespaceTokenizer() self.assertEquals([], tok.split("")) self.assertEquals(["this", "is", "a", "test"], tok.split("this is a test"))