Python tokenize示例

示例#1

文件： calc_test.py 项目： pliba/kaminpy

def test_evaluate(source, want):
    result = evaluate(tokenize(source))
    assert want == result

示例#2

文件： calctest.py 项目： mirandaio/calc

 def test_valid_expr(self):
     for expr, tokens in self.valid_expr:
         result = calc.tokenize(expr)
         self.assertEqual(tokens, result)

示例#3

文件： calc_tests.py 项目： JeffAMcGee/calc

 def test_tokenize(self):
     res = calc.tokenize('3+4')
     self.assertEqual(res,[3,operator.add,4])

示例#4

文件： calc_test.py 项目： pliba/kaminpy

def test_tokenize(source, want):
    tokens = tokenize(source)
    assert want == tokens

示例#5

 def test_empty(self):
     self.assertEqual([], calc.tokenize(''))

示例#6

 def test_bug_sl(self):
     self.assertEqual(['-', '1', '-', '1'], calc.tokenize('-1-1'))

示例#7

 def test_expressions(self):
     self.assertEqual(['1'], calc.tokenize('1'))
     self.assertEqual(['1', '+'], calc.tokenize('1+'))
     self.assertEqual(['1', '+', '2', '*', '3'], calc.tokenize('1+2*3'))
     self.assertEqual(['-', '1', '+', '2', '*', '3'],
                      calc.tokenize('-1+2*3'))
     self.assertEqual(['-', '1.5', '+', '2', '*', '3'],
                      calc.tokenize('-1.5+2*3'))
     self.assertEqual(['-', '1.5', '+', '2', '*', '3'],
                      calc.tokenize('- 1.5+2*3'))
     self.assertEqual(['1', '+', '2', '*', '3', '/', '4'],
                      calc.tokenize('1+2*3/4'))
     self.assertEqual(['1', '+', '-', '2', '*', '-', '3', '/', '4'],
                      calc.tokenize('1+-2*-3/4'))
     self.assertEqual(['1', '+', '2', '*', '3', '/', '4', '-', '5'],
                      calc.tokenize('1+2*3/4-5'))
     self.assertEqual(['10', '+', '20', '*', '30', '/', '40', '-', '50'],
                      calc.tokenize('10+20*30/40-50'))
     self.assertEqual(
         ['11.0', '+', '22.0', '*', '33.0', '/', '44.0', '-', '55.0'],
         calc.tokenize('11.0+22.0*33.0/44.0-55.0'))
     self.assertEqual(['10', '+', '20', '*', '30', '/', '40', '-', '50'],
                      calc.tokenize(' 10 + 20 * 30 / 40 - 50 '))

示例#8

 def test_float_without_ops(self):
     self.assertEqual(['1.2'], calc.tokenize('1.2'))
     self.assertEqual(['1.2'], calc.tokenize('  1.2 '))
     self.assertEqual(['1.2'], calc.tokenize(' \t 1.2'))
     self.assertEqual(['1.2', '34'], calc.tokenize('1.2 34'))
     self.assertEqual(['1.2', '3.4'], calc.tokenize(' 1.2\n3.4  '))

示例#9

 def test_int_without_ops(self):
     self.assertEqual(['12'], calc.tokenize('12'))
     self.assertEqual(['12'], calc.tokenize('  12 '))
     self.assertEqual(['12'], calc.tokenize(' \t 12'))
     self.assertEqual(['12', '34'], calc.tokenize('12 34'))
     self.assertEqual(['12', '34'], calc.tokenize(' 12\n34  '))