def test_long(self):
text = 'foo bar baz foo2 {x?_}!943 322 !# ^\t^& while (a<b): 0x00\n int a[0:]'
self.assertEqual([
'foo', ' ', 'bar', ' ', 'baz', ' ', 'foo2', ' ', ' ', '{', 'x',
'?', '_', '}!', '943', ' ', '322', ' ', '!#', ' ', '^', '\t', '^&',
' ', 'while', ' ', '(', 'a', '<', 'b', '):', ' ', '0x00', '\n',
' ', ' ', 'int', ' ', 'a', '[', '0', ':]'
], list(tokenize_by_lexems(text)))
def test_identifier_3(self):
text = 'FooBar'
self.assertEqual(['FooBar'], list(tokenize_by_lexems(text)))
def test_identifier_4(self):
text = 'fooBar_FOOBar_foobar012'
self.assertEqual(['fooBar_FOOBar_foobar012'],
list(tokenize_by_lexems(text)))
def test_chars_3(self):
text = '#include <math.h>'
self.assertEqual(['#', 'include', ' ', '<', 'math', '.', 'h', '>'],
list(tokenize_by_lexems(text)))
def test_identifier_1(self):
text = 'foo_bar'
self.assertEqual(['foo_bar'], list(tokenize_by_lexems(text)))
def test_chars_2(self):
text = '(n):'
self.assertEqual(['(', 'n', ')', ':'], tokenize_by_char(text))
self.assertEqual(['(', 'n', '):'], list(tokenize_by_lexems(text)))
def test_chars_1(self):
text = 'a->b'
self.assertEqual(['a', '-', '>', 'b'], tokenize_by_char(text))
self.assertEqual(['a', '->', 'b'], list(tokenize_by_lexems(text)))
def test_spaces_2(self):
text = ' foo'
self.assertEqual([' ', ' ', 'f', 'o', 'o'], tokenize_by_char(text))
self.assertEqual([' ', ' ', 'foo'], list(tokenize_by_lexems(text)))
def test_two_terms(self):
text = 'foo bar'
self.assertEqual(['f', 'o', 'o', ' ', 'b', 'a', 'r'],
tokenize_by_char(text))
self.assertEqual(['foo', ' ', 'bar'], list(tokenize_by_lexems(text)))
def test_one_term(self):
text = 'foo'
self.assertEqual(['f', 'o', 'o'], tokenize_by_char(text))
self.assertEqual(['foo'], list(tokenize_by_lexems(text)))