def test_tokenize_w(self):
tokz = SocialLM.tokenize('hello world', mode='w')
self.assertEqual(tokz,['hello','world'])
def test_tokenize_c(self):
tokz = SocialLM.tokenize('hello', mode='c')
self.assertEqual(tokz,['h','e','l','l','o'])
