def test_join(self):
tok = WhitespaceTokenizer()
self.assertEquals("", tok.join([]))
self.assertEquals("this is a test",
tok.join(["this", "is", "a", "test"]))
def test_split(self):
tok = WhitespaceTokenizer()
self.assertEquals([], tok.split(""))
self.assertEquals(["this", "is", "a", "test"],
tok.split("this is a test"))