def test_generate_alphabet_dict(self): s = ["testing", "123"] reserved_tokens = ["???"] alphabet = tokenizer._generate_alphabet_dict(s, reserved_tokens) self.assertIn("?", alphabet) self.assertIn("t", alphabet) self.assertIn("e", alphabet) self.assertIn("s", alphabet) self.assertIn("i", alphabet) self.assertIn("n", alphabet) self.assertIn("g", alphabet) self.assertIn("1", alphabet) self.assertIn("2", alphabet) self.assertIn("3", alphabet)
def test_generate_alphabet_dict(self): s = ["testing", "123"] reserved_tokens = ["???"] alphabet = tokenizer._generate_alphabet_dict(s, reserved_tokens) self.assertIn("?", alphabet) self.assertIn("t", alphabet) self.assertIn("e", alphabet) self.assertIn("s", alphabet) self.assertIn("i", alphabet) self.assertIn("n", alphabet) self.assertIn("g", alphabet) self.assertIn("1", alphabet) self.assertIn("2", alphabet) self.assertIn("3", alphabet)