def test_multiline_not_closed(self):
'''Test that lexer fails when a multiline string isn't closed'''
data = '\n"""one\n\n\n\ntwo'
lexer.input(data)
with self.assertRaisesRegexp(lex.LexError, 'Multiline string not closed at line 2 col 1'):
list(lexer)
def test_bad_indentation(self):
'Test lexer failing on bad dedentation'
data = 'one\n two\n three'
lexer.input(data)
with self.assertRaisesRegexp(lex.LexError, 'Invalid indentation at line 3 col 2'):
list(lexer)
def test_multiline_not_closed(self):
'''Test that lexer fails when a multiline string isn't closed'''
data = '\n"""one\n\n\n\ntwo'
lexer.input(data)
with self.assertRaisesRegexp(
lex.LexError, 'Multiline string not closed at line 2 col 1'):
list(lexer)
def test_bad_indentation(self):
'Test lexer failing on bad dedentation'
data = 'one\n two\n three'
lexer.input(data)
with self.assertRaisesRegexp(lex.LexError,
'Invalid indentation at line 3 col 2'):
list(lexer)
def test_basic(self):
'Basic lexing test'
data = 'print("It\'s Never Lupus!")'
lexer.input(data)
tokens = list(lexer)
self.assertEqual([tok.type for tok in tokens],
['ID', '(', 'STRING', ')'])
def test_whitespace(self):
'Test that lexer ignores in/trailing whitespace, but not leading whitespace'
data = 'if a>0: \n print(a + 10) \n return a'
lexer.input(data)
tokens = list(lexer)
self.assertEqual(
[tok.type for tok in tokens],
['IF', 'ID', 'LCOMP', 'NUMBER', ':', 'INDENT', 'ID', '(', 'ID', '+', 'NUMBER', ')', 'RETURN', 'ID', 'DEDENT'])
def test_indentation(self):
'Test lexing indentation handling'
data = 'one\n \n two\n three\n four\n five\n\n six'
lexer.input(data)
tokens = list(lexer)
self.assertEqual(
[tok.type for tok in tokens],
['ID', 'INDENT', 'ID', 'ID', 'INDENT', 'ID', 'INDENT', 'ID', 'DEDENT', 'DEDENT', 'ID', 'DEDENT'])
def test_basic(self):
'Basic lexing test'
data = 'print("It\'s Never Lupus!")'
lexer.input(data)
tokens = list(lexer)
self.assertEqual(
[tok.type for tok in tokens],
['ID', '(', 'STRING', ')'])
def test_whitespace(self):
'Test that lexer ignores in/trailing whitespace, but not leading whitespace'
data = 'if a>0: \n print(a + 10) \n return a'
lexer.input(data)
tokens = list(lexer)
self.assertEqual([tok.type for tok in tokens], [
'IF', 'ID', 'LCOMP', 'NUMBER', ':', 'INDENT', 'ID', '(', 'ID', '+',
'NUMBER', ')', 'RETURN', 'ID', 'DEDENT'
])
def test_types(self):
'Basic type lexing test'
data = '1\n1.5\nTrue\nFalse\nNone'
lexer.input(data)
tokens = list(lexer)
self.assertEqual([tok.type for tok in tokens],
['NUMBER', 'NUMBER', 'TRUE', 'FALSE', 'NONE'])
self.assertEqual([tok.value for tok in tokens],
[1, 1.5, True, False, None])
def test_numbers(self):
'Basic number test'
data = '1\n1.0\n1.1\n'
lexer.input(data)
tokens = list(lexer)
self.assertEqual([tok.type for tok in tokens],
['NUMBER', 'NUMBER', 'NUMBER'])
self.assertEqual([repr(tok.value) for tok in tokens],
['1', '1.0', '1.1'])
def test_indentation(self):
'Test lexing indentation handling'
data = 'one\n \n two\n three\n four\n five\n\n six'
lexer.input(data)
tokens = list(lexer)
self.assertEqual([tok.type for tok in tokens], [
'ID', 'INDENT', 'ID', 'ID', 'INDENT', 'ID', 'INDENT', 'ID',
'DEDENT', 'DEDENT', 'ID', 'DEDENT'
])
def test_types(self):
'Basic type lexing test'
data = '1\n1.5\nTrue\nFalse\nNone'
lexer.input(data)
tokens = list(lexer)
self.assertEqual(
[tok.type for tok in tokens],
['NUMBER', 'NUMBER', 'TRUE', 'FALSE', 'NONE'])
self.assertEqual(
[tok.value for tok in tokens],
[1, 1.5, True, False, None])
def test_numbers(self):
'Basic number test'
data = '1\n1.0\n1.1\n'
lexer.input(data)
tokens = list(lexer)
self.assertEqual(
[tok.type for tok in tokens],
['NUMBER', 'NUMBER', 'NUMBER'])
self.assertEqual(
[repr(tok.value) for tok in tokens],
['1', '1.0', '1.1'])
def test_strings(self):
'Test lexing string/multiline-string handling'
data = '"str1" id1\n \'str2\'\n id2\n """str3\nstr4"""\nid3\n\'\'\'\n\nstr5\n\'\'\'\nid4'
lexer.input(data)
tokens = list(lexer)
self.assertEqual(
[tok.type for tok in tokens],
['STRING', 'ID', 'INDENT', 'STRING', 'ID', 'STRING', 'DEDENT', 'ID', 'STRING', 'ID'])
self.assertEqual(
[tok.value for tok in tokens],
['str1', 'id1', '', 'str2', 'id2', 'str3\nstr4', '', 'id3', '\n\nstr5\n', 'id4'])
self.assertEqual(
[tok.lineno for tok in tokens],
[1, 1, 2, 2, 3, 4, 6, 6, 7, 11])
def test_strings(self):
'Test lexing string/multiline-string handling'
data = '"str1" id1\n \'str2\'\n id2\n """str3\nstr4"""\nid3\n\'\'\'\n\nstr5\n\'\'\'\nid4'
lexer.input(data)
tokens = list(lexer)
self.assertEqual([tok.type for tok in tokens], [
'STRING', 'ID', 'INDENT', 'STRING', 'ID', 'STRING', 'DEDENT', 'ID',
'STRING', 'ID'
])
self.assertEqual([tok.value for tok in tokens], [
'str1', 'id1', '', 'str2', 'id2', 'str3\nstr4', '', 'id3',
'\n\nstr5\n', 'id4'
])
self.assertEqual([tok.lineno for tok in tokens],
[1, 1, 2, 2, 3, 4, 6, 6, 7, 11])
