def test_tokenize_w(self): tokz = SocialLM.tokenize('hello world', mode='w') self.assertEqual(tokz,['hello','world'])
def test_tokenize_c(self): tokz = SocialLM.tokenize('hello', mode='c') self.assertEqual(tokz,['h','e','l','l','o'])
def test_tokenize_w(self): tokz = SocialLM.tokenize('hello world', mode='w') self.assertEqual(tokz,['hello','world'])
def test_tokenize_c(self): tokz = SocialLM.tokenize('hello', mode='c') self.assertEqual(tokz,['h','e','l','l','o'])