Пример #1
0
def test0():
    letter = Range("AZaz") | Any("_")
    digit = Range("09")
    space = Any(" \t\n")

    ident = letter + Rep(letter | digit)
    resword = Str("program", "unit", "uses", "const", "type", "var", "if",
                  "then", "else", "while", "do", "repeat", "until", "for",
                  "to", "downto", "and", "or", "not", "array", "of", "record",
                  "object")
    number = Rep1(digit)
    string = Str("'") + (Rep(AnyBut("'")) | Str("''")) + Str("'")
    diphthong = Str(":=", "<=", ">=", "<>", "..")
    punct = Any("^&*()-+=[]|;:<>,./")
    spaces = Rep1(space)
    comment_begin = Str("{")
    comment_char = AnyBut("}")
    comment_end = Str("}")

    lex = Lexicon([(resword, TEXT), (ident, 'ident'), (number, 'num'),
                   (string, 'str'), (punct | diphthong, TEXT),
                   (spaces, IGNORE), (comment_begin, Begin('comment')),
                   State('comment', [(comment_char, IGNORE),
                                     (comment_end, Begin(''))])],
                  debug=None,
                  timings=None)
    run_scanner(lex, "test6")
Пример #2
0
def test0():
    letters = "abc"
    spaces = " \t\n"
    all = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz*/{} \t\n"

    letter = Any(letters)
    space = Any(" \t\n")

    ident = Rep1(letter)
    spaces = Rep1(space)
    begin_comment = Str("{")
    end_comment = Str("}")

    lex = Lexicon(
        [
            (ident, 'ident'),
            (spaces, IGNORE),
            (begin_comment, Begin('comment')),
            State('comment', [
                (end_comment, Begin('')),
                (AnyBut("}"), IGNORE),
            ])
        ],
        debug = None,
        timings = None
    )
    run_scanner(lex, "test5")
Пример #3
0
def test12():
    lex = Lexicon(
        [(Str("'") + Rep(AnyBut("'")) + Str("'"), TEXT), (AnyChar, IGNORE)],
        debug=None,
        timings=None,
    )

    run_scanner(lex, "test12", trace=False)
Пример #4
0
def test0():
    lex = Lexicon(
        [(Seq(Any("ab"), Rep(Any("ab01"))), 'ident'),
         (Seq(Any("01"), Rep(Any("01"))), 'num'), (Any(' \n'), IGNORE),
         (Str("abba"), 'abba'),
         (Any('([{!"#') + Rep(AnyBut('!"#}])')) + Any('!"#}])'), IGNORE)],
        debug=None,
        timings=None)
    run_scanner(lex, "test2")
Пример #5
0
def test0():
    lex = Lexicon(
        [
            (Any("ab") + Rep(Any("ab01")), 'ident'),
            (Any(" \n"), IGNORE)
        ],
        debug = None,
        timings = None
    )
    run_scanner(lex, "test1")
Пример #6
0
def test0():
    spaces = Rep1(Any(" \t\n"))

    lex = Lexicon([(Bol + Rep1(Str("a")), 'begin'), (Rep1(Str("b")), 'middle'),
                   (Rep1(Str("c")) + Eol, 'end'),
                   (Bol + Rep1(Str("d")) + Eol, 'everything'),
                   (spaces, IGNORE)],
                  debug=None,
                  timings=None)
    run_scanner(lex, "test7")
Пример #7
0
def test0():
    lex = Lexicon(
        [
            (Str("a"), 'thing'),
            (Any("\n"), IGNORE)
        ],
        debug = None,
        timings = None
    )
    run_scanner(lex, "test0")
Пример #8
0
def test12():
    lex = Lexicon(
        [
            (Str("'") + Rep(AnyBut("'")) + Str("'"), TEXT),
            (AnyChar, IGNORE)
        ],
        debug = None,
        timings = None,
    )

    run_scanner(lex, "test12", trace = False)
Пример #9
0
def test0():
    lex = Lexicon(
        [
            (Seq(Any("ab"), Rep(Any("ab01"))), "ident"),
            (Seq(Any("01"), Rep(Any("01"))), "num"),
            (Any(" \n"), IGNORE),
            (Str("abba"), "abba"),
            (Any('([{!"#') + Rep(AnyBut('!"#}])')) + Any('!"#}])'), IGNORE),
        ],
        debug=None,
        timings=None,
    )
    run_scanner(lex, "test2")
Пример #10
0
def test11():
    lex = Lexicon(
        [
            (Str("Python"), 'upper-python'),
            (Str("python"), 'lower-python'),
            (NoCase(Str("COBOL", "perl", "Serbo-Croatian")), 'other-language'),
            (NoCase(Str("real") + Case(Str("basic"))), 'real-1'),
            (NoCase(Str("real") + Case(Str("Basic"))), 'real-2'),
            (Any(" \t\n"), IGNORE)
        ],
        debug = None,
        timings = None
    )
    run_scanner(lex, "test11")
Пример #11
0
def test0():
    spaces = Rep1(Any(" \t\n"))

    lex = Lexicon(
        [
            (Str("ftangftang"), 'two_ftangs'),
            (Str("ftang"),      'one_ftang'),
            (Str("fta"),        'one_fta'),
            (spaces, IGNORE)
        ],
        debug = None,
        timings = None
    )
    run_scanner(lex, "test8")
Пример #12
0
def test0():
    spaces = Rep1(Any(" \t\n"))

    lex = Lexicon(
        [
            (Bol + Rep1(Str("a")),       'begin'),
            (      Rep1(Str("b")),       'middle'),
            (      Rep1(Str("c")) + Eol, 'end'),
            (Bol + Rep1(Str("d")) + Eol, 'everything'),
            (spaces, IGNORE)
        ],
        debug = None,
        timings = None
    )
    run_scanner(lex, "test7")
Пример #13
0
def test0():
    letter = Range("AZaz") | Any("_")
    digit = Range("09")
    space = Any(" \t\n")

    ident = Seq(letter, Rep(Alt(letter, digit)))
    number = Seq(digit, Rep(digit))
    punct = Any("*()-+=[]{};:<>,./")
    spaces = Seq(space, Rep(space))
    resword = Str("program", "begin", "end", "repeat", "until")

    lex = Lexicon([(resword, TEXT), (ident, 'ident'), (number, 'num'),
                   (punct, TEXT), (spaces, IGNORE)],
                  debug=None,
                  timings=None)
    run_scanner(lex, "test3")
Пример #14
0
def test0():
    letters = "abc"
    spaces = " \t\n"
    all = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz*/{} \t\n"

    letter = Any(letters)
    space = Any(" \t\n")

    ident = Rep1(letter)
    spaces = Rep1(space)
    begin_comment = Str("{")
    end_comment = Str("}")

    lex = Lexicon([(ident, 'ident'), (spaces, IGNORE),
                   (begin_comment, Begin('comment')),
                   State('comment', [
                       (end_comment, Begin('')),
                       (AnyBut("}"), IGNORE),
                   ])],
                  debug=None,
                  timings=None)
    run_scanner(lex, "test5")
Пример #15
0
def test0():
    letter = Range("AZaz") | Any("_")
    digit = Range("09")
    space = Any(" \t\n")

    ident = Seq(letter, Rep(Alt(letter, digit)))
    number = Seq(digit, Rep(digit))
    punct = Any("*()-+=[]{};:<>,./")
    spaces = Seq(space, Rep(space))
    resword = Str("program", "begin", "end", "repeat", "until")

    lex = Lexicon(
        [
            (resword, TEXT),
            (ident, 'ident'),
            (number, 'num'),
            (punct, TEXT),
            (spaces, IGNORE)
        ],
        debug = None,
        timings = None
    )
    run_scanner(lex, "test3")
Пример #16
0
def test0():
    letter = Range("AZaz") | Any("_")
    digit = Range("09")
    space = Any(" \t\n")

    ident = letter + Rep(letter | digit)
    resword = Str("program", "unit", "uses", "const", "type", "var",
                  "if", "then", "else", "while", "do", "repeat", "until",
                  "for", "to", "downto", "and", "or", "not",
                  "array", "of", "record", "object")
    number = Rep1(digit)
    string = Str("'") + (Rep(AnyBut("'")) | Str("''")) + Str("'")
    diphthong = Str(":=", "<=", ">=", "<>", "..")
    punct = Any("^&*()-+=[]|;:<>,./")
    spaces = Rep1(space)
    comment_begin = Str("{")
    comment_char = AnyBut("}")
    comment_end = Str("}")

    lex = Lexicon(
        [
            (resword, TEXT),
            (ident, 'ident'),
            (number, 'num'),
            (string, 'str'),
            (punct | diphthong, TEXT),
            (spaces, IGNORE),
            (comment_begin, Begin('comment')),
            State('comment', [
                (comment_char, IGNORE),
                (comment_end, Begin(''))
            ])
        ],
        debug = None,
        timings = None
    )
    run_scanner(lex, "test6")
Пример #17
0
def test0():
    letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"

    wax = Any("(")
    wane = Any(")")
    letter = Any(letters)
    space = Any(" \t\n")

    def open_paren(s, t):
        s.counter = s.counter + 1

    def close_paren(s, t):
        s.counter = s.counter - 1

    def got_a_letter(s, t):
        if s.counter == 0:
            return 'letter'
        else:
            return None

    lex = Lexicon(
        [
            (wax, open_paren),
            (wane, close_paren),
            (letter, got_a_letter),
            (space, IGNORE)
        ],
        debug = None,
        timings = None
    )

    class MyScanner(Scanner):
        counter = 0
        trace = 0

    run_scanner(lex, "test4", scanner_class = MyScanner)
Пример #18
0
def test9():
    run_scanner(None, "test9", scanner_class = MyScanner)