/
ebnf_with_lexer.py
executable file
·91 lines (66 loc) · 3.67 KB
/
ebnf_with_lexer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#!/usr/bin/env python
# coding UTF-8
## Copyright 2013 Constantin Kulikov
##
## Author: Constantin Kulikov (Bad_ptr) <zxnotdead@gmail.com>
## Date: 2013/07/23 11:37:31
## License: GPL either version 2 or any later version
from common_classes import thing_pprint
from lexer_test import (tt_identifier, tt_number, tt_operator, tt_terminator
, tt_space, tt_comment, tt_literal, tt_list, tt_skip_list
, tok_openbr, tok_closebr, tok_opencurlybr, tok_closecurlbr
, tok_opensqbr, tok_closesqbr, tok_squote, tok_dquote
, tok_comma, tok_period, tok_colon, tok_semicolon
, tok_accent
, token_iterator, token_reader)
from parser import (Grammar, Node, TRef)
from lexer import (TokenIterator, TokenReader)
from iters_readers import TextReader
from readable_things import (Literal, NotNeed, Seq, Num
, ZeroOrOne, ZeroOrMore, OneOrMore
, Look, Rx, Concat, Or, Not)
from utils import SetRecursionLimit
cur_grammar = Grammar()
pn_identifier = Node("pn_identifier", OneOrMore(tt_identifier), grammar=cur_grammar)
pn_terminal = Node("terminal", OneOrMore(tt_literal), grammar=cur_grammar)
pn_ident_or_term = Or(pn_identifier, pn_terminal)
pn_lhs = Node("pn_lhs", pn_identifier, grammar=cur_grammar)
pn_option = Node("pn_option"
, Seq(NotNeed(tok_opensqbr), TRef("pn_rhs", grammar=cur_grammar, flat=True), NotNeed(tok_closesqbr))
, grammar=cur_grammar)
pn_repetition = Node("pn_repetition"
, Seq(NotNeed(tok_opencurlybr), TRef("pn_rhs", grammar=cur_grammar, flat=True), NotNeed(tok_closecurlbr))
, grammar=cur_grammar)
pn_grouping = Node("group"
, Seq(NotNeed(tok_openbr), TRef("pn_rhs", grammar=cur_grammar, flat=True), NotNeed(tok_closebr))
, grammar=cur_grammar)
pn_negation = Node("pn_negation"
, Seq(NotNeed("^"), TRef("pn_rhs", grammar=cur_grammar, flat=True), NotNeed("^"))
, grammar=cur_grammar)
pn_regexp = Node("pn_regexp"
, Seq(NotNeed("rx"), tt_literal)
, grammar=cur_grammar)
pn_groups = Or(pn_option, pn_repetition, pn_grouping, pn_negation, pn_regexp)
pn_ident_or_term_or_pn_groups = Or(pn_ident_or_term, pn_groups)
pn_concatenation = Node("pn_concatenation"
, Seq(pn_ident_or_term_or_pn_groups, NotNeed(","), TRef("pn_rhs", grammar=cur_grammar, flat=True))
, flat_eq_name=True, grammar=cur_grammar)
pn_alteration = Node("pn_alteration"
, Seq(pn_ident_or_term_or_pn_groups, NotNeed("|"), TRef("pn_rhs", grammar=cur_grammar, flat=True))
, flat_eq_name=True, grammar=cur_grammar)
pn_rhs = Node("pn_rhs", Or(pn_concatenation
, pn_alteration
, pn_groups
, pn_ident_or_term)
, grammar=cur_grammar)
pn_rule = Node("pn_rule", Seq(pn_lhs, NotNeed("="), pn_rhs, NotNeed(";")), grammar=cur_grammar)
pn_grammar = Node("ebnf_grammar", ZeroOrMore(pn_rule), grammar=cur_grammar)
#token_iterator = TokenIterator(TextReader(open("test1.txt"))
# , tt_list, tt_skip_list)
#token_reader = TokenReader(token_iterator)
#token_reader = TokenReader(TokenIterator(TextReader(open("test1.txt")), tt_list, tt_skip_list))
if __name__ == '__main__':
# SetRecursionLimit(5000)
# lexing with tokens reduces recursion level
rslt = pn_grammar.read_from(token_reader)
thing_pprint(rslt.readedlist)