def test0():
letter = Range("AZaz") | Any("_")
digit = Range("09")
space = Any(" \t\n")
ident = letter + Rep(letter | digit)
resword = Str("program", "unit", "uses", "const", "type", "var", "if",
"then", "else", "while", "do", "repeat", "until", "for",
"to", "downto", "and", "or", "not", "array", "of", "record",
"object")
number = Rep1(digit)
string = Str("'") + (Rep(AnyBut("'")) | Str("''")) + Str("'")
diphthong = Str(":=", "<=", ">=", "<>", "..")
punct = Any("^&*()-+=[]|;:<>,./")
spaces = Rep1(space)
comment_begin = Str("{")
comment_char = AnyBut("}")
comment_end = Str("}")
lex = Lexicon([(resword, TEXT), (ident, 'ident'), (number, 'num'),
(string, 'str'), (punct | diphthong, TEXT),
(spaces, IGNORE), (comment_begin, Begin('comment')),
State('comment', [(comment_char, IGNORE),
(comment_end, Begin(''))])],
debug=None,
timings=None)
run_scanner(lex, "test6")
def test0():
letters = "abc"
spaces = " \t\n"
all = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz*/{} \t\n"
letter = Any(letters)
space = Any(" \t\n")
ident = Rep1(letter)
spaces = Rep1(space)
begin_comment = Str("{")
end_comment = Str("}")
lex = Lexicon(
[
(ident, 'ident'),
(spaces, IGNORE),
(begin_comment, Begin('comment')),
State('comment', [
(end_comment, Begin('')),
(AnyBut("}"), IGNORE),
])
],
debug = None,
timings = None
)
run_scanner(lex, "test5")
def test12():
lex = Lexicon(
[(Str("'") + Rep(AnyBut("'")) + Str("'"), TEXT), (AnyChar, IGNORE)],
debug=None,
timings=None,
)
run_scanner(lex, "test12", trace=False)
def test0():
lex = Lexicon(
[(Seq(Any("ab"), Rep(Any("ab01"))), 'ident'),
(Seq(Any("01"), Rep(Any("01"))), 'num'), (Any(' \n'), IGNORE),
(Str("abba"), 'abba'),
(Any('([{!"#') + Rep(AnyBut('!"#}])')) + Any('!"#}])'), IGNORE)],
debug=None,
timings=None)
run_scanner(lex, "test2")
def test0():
lex = Lexicon(
[
(Any("ab") + Rep(Any("ab01")), 'ident'),
(Any(" \n"), IGNORE)
],
debug = None,
timings = None
)
run_scanner(lex, "test1")
def test0():
spaces = Rep1(Any(" \t\n"))
lex = Lexicon([(Bol + Rep1(Str("a")), 'begin'), (Rep1(Str("b")), 'middle'),
(Rep1(Str("c")) + Eol, 'end'),
(Bol + Rep1(Str("d")) + Eol, 'everything'),
(spaces, IGNORE)],
debug=None,
timings=None)
run_scanner(lex, "test7")
def test0():
lex = Lexicon(
[
(Str("a"), 'thing'),
(Any("\n"), IGNORE)
],
debug = None,
timings = None
)
run_scanner(lex, "test0")
def test12():
lex = Lexicon(
[
(Str("'") + Rep(AnyBut("'")) + Str("'"), TEXT),
(AnyChar, IGNORE)
],
debug = None,
timings = None,
)
run_scanner(lex, "test12", trace = False)
def test0():
lex = Lexicon(
[
(Seq(Any("ab"), Rep(Any("ab01"))), "ident"),
(Seq(Any("01"), Rep(Any("01"))), "num"),
(Any(" \n"), IGNORE),
(Str("abba"), "abba"),
(Any('([{!"#') + Rep(AnyBut('!"#}])')) + Any('!"#}])'), IGNORE),
],
debug=None,
timings=None,
)
run_scanner(lex, "test2")
def test11():
lex = Lexicon(
[
(Str("Python"), 'upper-python'),
(Str("python"), 'lower-python'),
(NoCase(Str("COBOL", "perl", "Serbo-Croatian")), 'other-language'),
(NoCase(Str("real") + Case(Str("basic"))), 'real-1'),
(NoCase(Str("real") + Case(Str("Basic"))), 'real-2'),
(Any(" \t\n"), IGNORE)
],
debug = None,
timings = None
)
run_scanner(lex, "test11")
def test0():
spaces = Rep1(Any(" \t\n"))
lex = Lexicon(
[
(Str("ftangftang"), 'two_ftangs'),
(Str("ftang"), 'one_ftang'),
(Str("fta"), 'one_fta'),
(spaces, IGNORE)
],
debug = None,
timings = None
)
run_scanner(lex, "test8")
def test0():
spaces = Rep1(Any(" \t\n"))
lex = Lexicon(
[
(Bol + Rep1(Str("a")), 'begin'),
( Rep1(Str("b")), 'middle'),
( Rep1(Str("c")) + Eol, 'end'),
(Bol + Rep1(Str("d")) + Eol, 'everything'),
(spaces, IGNORE)
],
debug = None,
timings = None
)
run_scanner(lex, "test7")
def test0():
letter = Range("AZaz") | Any("_")
digit = Range("09")
space = Any(" \t\n")
ident = Seq(letter, Rep(Alt(letter, digit)))
number = Seq(digit, Rep(digit))
punct = Any("*()-+=[]{};:<>,./")
spaces = Seq(space, Rep(space))
resword = Str("program", "begin", "end", "repeat", "until")
lex = Lexicon([(resword, TEXT), (ident, 'ident'), (number, 'num'),
(punct, TEXT), (spaces, IGNORE)],
debug=None,
timings=None)
run_scanner(lex, "test3")
def test0():
letters = "abc"
spaces = " \t\n"
all = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz*/{} \t\n"
letter = Any(letters)
space = Any(" \t\n")
ident = Rep1(letter)
spaces = Rep1(space)
begin_comment = Str("{")
end_comment = Str("}")
lex = Lexicon([(ident, 'ident'), (spaces, IGNORE),
(begin_comment, Begin('comment')),
State('comment', [
(end_comment, Begin('')),
(AnyBut("}"), IGNORE),
])],
debug=None,
timings=None)
run_scanner(lex, "test5")
def test0():
letter = Range("AZaz") | Any("_")
digit = Range("09")
space = Any(" \t\n")
ident = Seq(letter, Rep(Alt(letter, digit)))
number = Seq(digit, Rep(digit))
punct = Any("*()-+=[]{};:<>,./")
spaces = Seq(space, Rep(space))
resword = Str("program", "begin", "end", "repeat", "until")
lex = Lexicon(
[
(resword, TEXT),
(ident, 'ident'),
(number, 'num'),
(punct, TEXT),
(spaces, IGNORE)
],
debug = None,
timings = None
)
run_scanner(lex, "test3")
def test0():
letter = Range("AZaz") | Any("_")
digit = Range("09")
space = Any(" \t\n")
ident = letter + Rep(letter | digit)
resword = Str("program", "unit", "uses", "const", "type", "var",
"if", "then", "else", "while", "do", "repeat", "until",
"for", "to", "downto", "and", "or", "not",
"array", "of", "record", "object")
number = Rep1(digit)
string = Str("'") + (Rep(AnyBut("'")) | Str("''")) + Str("'")
diphthong = Str(":=", "<=", ">=", "<>", "..")
punct = Any("^&*()-+=[]|;:<>,./")
spaces = Rep1(space)
comment_begin = Str("{")
comment_char = AnyBut("}")
comment_end = Str("}")
lex = Lexicon(
[
(resword, TEXT),
(ident, 'ident'),
(number, 'num'),
(string, 'str'),
(punct | diphthong, TEXT),
(spaces, IGNORE),
(comment_begin, Begin('comment')),
State('comment', [
(comment_char, IGNORE),
(comment_end, Begin(''))
])
],
debug = None,
timings = None
)
run_scanner(lex, "test6")
def test0():
letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"
wax = Any("(")
wane = Any(")")
letter = Any(letters)
space = Any(" \t\n")
def open_paren(s, t):
s.counter = s.counter + 1
def close_paren(s, t):
s.counter = s.counter - 1
def got_a_letter(s, t):
if s.counter == 0:
return 'letter'
else:
return None
lex = Lexicon(
[
(wax, open_paren),
(wane, close_paren),
(letter, got_a_letter),
(space, IGNORE)
],
debug = None,
timings = None
)
class MyScanner(Scanner):
counter = 0
trace = 0
run_scanner(lex, "test4", scanner_class = MyScanner)
def test9():
run_scanner(None, "test9", scanner_class = MyScanner)