Python Token示例

示例#1

 def test_punct_string(self):
     s = '- Can 2 mothers clean 100 windows???'
     self.assertEqual(
         self.t.tokenize_with_token_types(s)[0],
         Token("-", TokenType.PUNCTUATION, 0, 1))
     self.assertEqual(len(self.t.tokenize_with_token_types(s)), 14)

示例#2

 def test_space_string(self):
     s = ' 2 mothers clean 100 windows???'
     self.assertEqual(
         self.t.tokenize_with_token_types(s)[0],
         Token(" ", TokenType.SPACE, 0, 1))
     self.assertEqual(len(self.t.tokenize_with_token_types(s)), 11)

示例#3

 def test_alpha_string(self):
     s = 'mother cleans the window'
     self.assertEqual(
         self.t.tokenize_with_token_types(s)[0],
         Token("mother", TokenType.ALPHA, 0, 6))
     self.assertEqual(len(self.t.tokenize_with_token_types(s)), 7)

示例#4

 def test_digit_string(self):
     s = '2 mothers clean 100 windows!!!'
     self.assertEqual(
         self.t.tokenize_with_token_types(s)[0],
         Token("2", TokenType.NUMBER, 0, 1))
     self.assertEqual(len(self.t.tokenize_with_token_types(s)), 10)

示例#5

 def test_alpha_string(self):
     s = 'мама мыла раму'
     self.assertEqual(
         list(self.t.tokenize_gen_alpha_digit(s))[0],
         Token("мама", TokenType.ALPHA, 0, 4))
     self.assertEqual(len(list(self.t.tokenize_gen_alpha_digit(s))), 3)

示例#6

 def test_digit_string(self):
     s = '2 mothers clean 100 windows!!!'
     self.assertEqual(
         list(self.t.tokenize_gen_alpha_digit(s))[0],
         Token("2", TokenType.NUMBER, 0, 1))
     self.assertEqual(len(list(self.t.tokenize_gen_alpha_digit(s))), 5)

示例#7

 def test_alpha_string(self):
     s = 'mother cleans the window'
     self.assertEqual(
         list(self.t.tokenize_gen_alpha_digit(s))[0],
         Token("mother", TokenType.ALPHA, 0, 6))
     self.assertEqual(len(list(self.t.tokenize_gen_alpha_digit(s))), 7)

示例#8

 def test_alpha_string(self):
     s = 'мама мыла раму'
     self.assertEqual(
         self.t.tokenize_with_token_types(s)[0],
         Token("мама", TokenType.ALPHA, 0, 4))
     self.assertEqual(len(self.t.tokenize_with_token_types(s)), 5)

示例#9

 def test_last_character_string(self):
     s = '2 mothers clean 100 windows???'
     self.assertEqual(
         self.t.tokenize_with_token_types(s)[9],
         Token("???", TokenType.PUNCTUATION, 27, 30))
     self.assertEqual(len(self.t.tokenize_with_token_types(s)), 10)