Python tokenize Examples

Example #1

 def test_and_or(self):
     t = 'Ar40>50 and age>10 or age<0'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('Ar40>50', 'and'), (
         'age>10',
         'or',
     ), ('age<0', None)])

Example #2

 def test_not(self):
     t = 'not age<10'
     tokens = tokenize(t)
     self.assertEqual(tokens, [('not age<10', None)])

Example #3

 def test_parse_deflection(self):
     t = 'CDD.deflection==2000'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('CDD.deflection==2000', None)])

Example #4

 def test_single_and(self):
     t = 'Ar40>50 and age>10'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('Ar40>50', 'and'), ('age>10', None)])

Example #5

 def test_single_or(self):
     t = 'Ar40>50 or age>10'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('Ar40>50', 'or'), ('age>10', None)])

Example #6

 def test_pressure(self):
     t = 'bone.ig.pressure<1e-7'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('bone.ig.pressure<1e-7', None)])

Example #7

 def test_single(self):
     t = 'Ar40>50'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('Ar40>50', None)])

Example #8

File: conditionals.py Project: UManPychron/pychron

 def test_not(self):
     t = 'not age<10'
     tokens = tokenize(t)
     self.assertEqual(tokens, [('not age<10', None)])

Example #9

 def test_device(self):
     t = 'device.pneumatics<80'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('device.pneumatics<80', None)])

Example #10

File: conditionals.py Project: UManPychron/pychron

 def test_and_or(self):
     t = 'Ar40>50 and age>10 or age<0'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('Ar40>50', 'and'), ('age>10', 'or',), ('age<0', None)])

Example #11

File: conditionals.py Project: UManPychron/pychron

 def test_parse_deflection(self):
     t = 'CDD.deflection==2000'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('CDD.deflection==2000', None)])

Example #12

File: conditionals.py Project: UManPychron/pychron

 def test_single_or(self):
     t = 'Ar40>50 or age>10'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('Ar40>50', 'or'), ('age>10', None)])

Example #13

File: conditionals.py Project: UManPychron/pychron

 def test_single_and(self):
     t = 'Ar40>50 and age>10'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('Ar40>50', 'and'), ('age>10', None)])

Example #14

File: conditionals.py Project: UManPychron/pychron

 def test_single(self):
     t = 'Ar40>50'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('Ar40>50', None)])

Example #15

 def test_pressure(self):
     t = 'bone.ig.pressure<1e-7'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('bone.ig.pressure<1e-7', None)])

Example #16

 def test_device(self):
     t = 'device.pneumatics<80'
     tokens = tokenize(t)
     self.assertListEqual(tokens, [('device.pneumatics<80', None)])