Python Untokenizer.compat Examples

Example #1

 def test_iter_compat(self):
     u = Untokenizer()
     token = (NAME, 'Hello')
     u.compat(token, iter([]))
     self.assertEqual(u.tokens, ["Hello "])
     u = Untokenizer()
     self.assertEqual(u.untokenize(iter([token])), 'Hello ')

Example #2

File: test_tokenize.py Project: billtsay/win-demo-opcua

 def test_iter_compat(self):
     u = Untokenizer()
     token = (NAME, 'Hello')
     u.compat(token, iter([]))
     self.assertEqual(u.tokens, ["Hello "])
     u = Untokenizer()
     self.assertEqual(u.untokenize(iter([token])), 'Hello ')

Example #3

File: test_tokenize.py Project: 5outh/Databases-Fall2014

 def test_iter_compat(self):
     u = Untokenizer()
     token = (NAME, 'Hello')
     tokens = [(ENCODING, 'utf-8'), token]
     u.compat(token, iter([]))
     self.assertEqual(u.tokens, ["Hello "])
     u = Untokenizer()
     self.assertEqual(u.untokenize(iter([token])), 'Hello ')
     u = Untokenizer()
     self.assertEqual(u.untokenize(iter(tokens)), 'Hello ')
     self.assertEqual(u.encoding, 'utf-8')
     self.assertEqual(untokenize(iter(tokens)), b'Hello ')

Example #4

 def test_iter_compat(self):
     u = Untokenizer()
     token = (NAME, 'Hello')
     tokens = [(ENCODING, 'utf-8'), token]
     u.compat(token, iter([]))
     self.assertEqual(u.tokens, ["Hello "])
     u = Untokenizer()
     self.assertEqual(u.untokenize(iter([token])), 'Hello ')
     u = Untokenizer()
     self.assertEqual(u.untokenize(iter(tokens)), 'Hello ')
     self.assertEqual(u.encoding, 'utf-8')
     self.assertEqual(untokenize(iter(tokens)), b'Hello ')