def test_long():
for s, numtoken in [
("if x:\n print x", 10),
("if x:#foo\n x abc = 17", 12),
("1 a \\\n 2", 5)]:
tokens = lex(s)
assert len(tokens) == numtoken
print tokens
def test_name():
for s in ["abc",
"_",
"a_0",
"_0",
]:
tokens = lex(s)
token, newline, eof = tokens
assert token.name == 'Name'
assert newline.name == 'Newline'
assert eof.name == 'EOF'
s = '$' + s
tokens = lex(s)
token, newline, eof = tokens
assert token.name == 'PrimitiveName'
assert newline.name == 'Newline'
assert eof.name == 'EOF'
def test_single_quoted_string():
for s in ["""'abc'""",
"""'ab"c'""",
"""'ab\\'c'""",
]:
tokens = lex(s)
token, newline, eof = tokens
assert token.name == 'String'
assert newline.name == 'Newline'
assert eof.name == 'EOF'
def test_number():
for s in ["0",
"+1",
"-1",
"-1123123",
]:
tokens = lex(s)
token, newline, eof = tokens
assert token.name == 'Number'
assert newline.name == 'Newline'
assert eof.name == 'EOF'
def test_indentation():
s = """a
b
c
d
#some comment
e
f
"""
tokens = lex(s)
assert [t.name for t in tokens] == ["Name", "Newline", "Name", "Newline",
"Indent", "Name", "Newline", "Indent",
"Name", "Newline", "Dedent", "Name",
"Newline", "Indent", "Name", "Newline",
"Dedent", "Dedent", "EOF"]
def __init__(self, source):
self.source = source
self.tokens = lex(source)
self.i = 0
self.last_i = 0
def test_comment():
s = "(a = # foo this is a comment \n b)"
tokens = lex(s)
assert [t.name for t in tokens] == ["OpenBracket", "Name", "Special", "Name",
"CloseBracket", "Newline", "EOF"]
def test_parenthesis():
s = "(a = \n b)"
tokens = lex(s)
assert [t.name for t in tokens] == ["OpenBracket", "Name", "Special", "Name",
"CloseBracket", "Newline", "EOF"]
def test_linecont():
s = "a a \\\n b"
tokens = lex(s)
assert [t.name for t in tokens] == ["Name", "Name", "Name", "Newline",
"EOF"]
