def test_02_text_line_pair (self) : def body(): return OneOrMore ( [ text_line ], rule_name='body' ) def document(): return Sequence( ( body, EOF ), rule_name='document' ) # print('\n: document') ; pp(document()) parser = ParserPython( document, skipws=False ) text = self.line1 + self.line2 parsed = parser.parse(text) # print('\n: parsed') ; pp(parsed) # # print('\n: flatten') ; pp(flatten(parsed)) p_newline = Terminal(newline(), 0, '\n') p_l1_words = Terminal(words(), 0, self.line1[:-1]) p_l1_text_line = NonTerminal(text_line(), [ p_l1_words, p_newline ]) p_l2_words = Terminal(words(), 0, self.line2[:-1]) p_l2_text_line = NonTerminal(text_line(), [ p_l2_words, p_newline ]) p_body = NonTerminal(body(), [ p_l1_text_line, p_l2_text_line ]) p_eof = Terminal(EOF(), 0, '') expect = NonTerminal(document(), [p_body, p_eof] ) # print('\n: expect') ; pp(expect) assert parsed == expect, ( f"text = '{text}' :\n" f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}" )
def visit(self, node, depth=0): i = ' ' * 3 * depth # print(f"{i} [ p2 : node = {node}") if not hasattr(node, 'rule_name'): # print(f"{i} - not a known container : {str(type(node))}") # print(f"{i} => itself") # print(f"{i} ]") if isinstance(node, list) and len(node) == 1: return node[0] return node #---------------------------------------------------------------------- children = [] if isinstance(node, (NonTerminal, SemanticActionResults)): # print(f"{i} - visiting children of '{node.name}' : len = {len(node)}") # each of these object types is a list for child in node: response = self.visit(child, 1 + depth) if response: children.append(response) # print(f"{i} - visited children = {children}") #---------------------------------------------------------------------- # rule name specific visitor ? rule_name = str(node.rule_name) method = f"visit_{rule_name}" # print(f"{i} - {method} ?") if hasattr(self, method): # print(f"{i} - method found, applying to {node.name}") out = getattr(self, method)(node, children, 1 + depth) # print(f"{i} => {_res(out,i)}") # print(f"{i} ]") return out # else : # print(f"{i} - no such method") #---------------------------------------------------------------------- if len(children) <= 0: out = Terminal(node.rule, 0, node.value) # print(f"{i} => {_res(out,i)}") # print(f"{i} ]") return out if len(children) == 1: children = children[0] try: out = NonTerminal(node.rule, children) except: out = NonTerminal(node.rule, [Unwrap(children)]) # automatically unwrap out[0] = out[0].value # print(f"{i} => {_res(out,i)}") # print(f"{i} ]") return out
def test_05_paragraph_multiple (self) : def body(): return OneOrMore ( OrderedChoice( [ paragraph, newline ] ), rule_name='body' ) def document(): return Sequence( ( body, EOF ), rule_name='document' ) # print('\n: document') ; pp(document()) parser = ParserPython( document, skipws=False ) text = self.line1 + self.line2 + '\n' text = text * 3 parsed = parser.parse(text) # print('\n: parsed') ; pp(parsed) p_newline = Terminal(newline(), 0, '\n') p_l1_words = Terminal(words(), 0, self.line1[:-1]) p_l1_text_line = NonTerminal(text_line(), [ p_l1_words, p_newline ]) p_l2_words = Terminal(words(), 0, self.line2[:-1]) p_l2_text_line = NonTerminal(text_line(), [ p_l2_words, p_newline ]) p_paragraph = NonTerminal(paragraph(), [ p_l1_text_line, p_l2_text_line ]) p_body = NonTerminal(body(), [ p_paragraph, p_newline, p_paragraph, p_newline, p_paragraph, p_newline, ]) p_eof = Terminal(EOF(), 0, '') expect = NonTerminal(document(), [ p_body, p_eof ] ) # print('\n: expect') ; pp(expect) assert parsed == expect, ( f"text = '{text}' :\n" f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}" )
def test_004__n1_eq_t1__with_eq(self): if False: from prettyprinter import cpprint as pp import p print(f": test_004__n1_eq_t1__with_eq") print(f": n1 :") pp(self.n1) print(f": t1 :") pp(self.t1) NonTerminal_enable_structural_eq() with self.assertRaises(AssertionError) as context: assert self.n1 == self.t1 # now it fails self.assertTrue('Internal error, AssertionError not raised !!!') assert self.n1 != self.t1 assert self.n1 == deepcopy(self.n1) t2 = Terminal(self.dot, 0, 'one') n2 = NonTerminal(self.dot, [t2]) assert self.n1 == n2 t2 = Terminal(self.dot, 0, 'one') n2 = NonTerminal(self.dot, [t2]) assert self.n1 == n2 bang = StrMatch('!', rule_name='bang') t3 = Terminal(bang, 0, 'one') n3 = NonTerminal(bang, [t3]) assert self.n1 != n3
def setUp(self): self.dot = StrMatch('.', rule_name='self.dot') self.s1 = 's1 : string' self.s2 = 's2 : string' self.s3 = 's3 : string' # rule, position, value self.t1 = Terminal(self.dot, 0, 'one') self.t2 = Terminal(self.dot, 0, 'two') self.t3 = Terminal(self.dot, 0, 'three') assert not isinstance(self.t1, list) assert not isinstance(self.t2, list) assert not isinstance(self.t3, list) # rule, value : a list where the first element is a node # self.n1 = NonTerminal(self.dot, self.t1) # TypeError: 'Terminal' object is not subscriptable self.n2 = NonTerminal(self.dot, [self.t1]) self.n3 = NonTerminal(self.dot, self.n2) self.n4 = NonTerminal(self.dot, [self.n2]) assert isinstance(self.n2, list) assert isinstance(self.n3, list) assert isinstance(self.n4, list) self.v0 = self.n2 self.v1 = [self.s1, self.s2] self.v2 = self.t1 self.v3s = self.s3 self.v3t = (self.s1, self.s2)
def test_with_some_words_2(self): parser = ParserPython(document, skipws=False) text = f"{self.words1}\n\n" parsed = parser.parse(text) # print('\n: parsed') ; pp(parsed) expect = NonTerminal(document(), [ NonTerminal(body(), [ NonTerminal(element(), [ Terminal(words(), 0, self.words1), ]), NonTerminal(element(), [ Terminal(newline(), 0, '\n'), ]), NonTerminal(element(), [ Terminal(blank_line(), 0, '\n'), ]), ]), Terminal(EOF(), 0, ''), ]) # print('\n: expect') ; pp(expect) assert parsed == expect, ( f"text = '{text}' :\n" f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}")
def test_short_adj_arg(self): value = '-fFILE' option_ = Terminal(long_no_arg(), 0, '-f') operand_ = Terminal(operand_all_caps(), 0, 'FILE') operand_ = NonTerminal(operand(), [operand_]) expect = NonTerminal(short_adj_arg(), [option_, operand_]) expect = NonTerminal(option(), [expect]) expect = NonTerminal(body(), [expect, t_eof]) super().single(body, value, expect)
def test_short_gap_arg(self): value = '-f FILE' option_ = Terminal(short_no_arg(), 0, '-f') option_ = NonTerminal(option(), [option_]) p_ws1 = p_ws(' ') operand_ = Terminal(operand_all_caps(), 0, 'FILE') operand_ = NonTerminal(operand(), [operand_]) expect = NonTerminal(body(), [option_, p_ws1.expect, operand_, t_eof]) super().single(body, value, expect)
def test_long_gap_arg(self): value = '--file <a-file>' option_ = Terminal(long_no_arg(), 0, '--file') option_ = NonTerminal(option(), [option_]) p_ws1 = p_ws(' ') operand_ = Terminal(operand_angled(), 0, '<a-file>') operand_ = NonTerminal(operand(), [operand_]) expect = NonTerminal(body(), [option_, p_ws1.expect, operand_, t_eof]) super().single(body, value, expect)
def test_long_eq_arg(self): value = '--file=<a-file>' option_ = Terminal(long_no_arg(), 0, '--file') operand_ = Terminal(operand_angled(), 0, '<a-file>') operand_ = NonTerminal(operand(), [operand_]) expect = NonTerminal(long_eq_arg(), [option_, t_equals, operand_]) expect = NonTerminal(option(), [expect]) expect = NonTerminal(body(), [expect, t_eof]) super().single(body, value, expect)
def expect_document(sections): if len(sections) <= 0: raise ValueError("No sections provided. Please provide at least one.") expect = NonTerminal(document(), [ NonTerminal(body(), [NonTerminal(element(), sections)]), Terminal(EOF(), 0, ''), ]) return expect
def test_single_short_with_one_arg (self): text = '-fNORM' parsed = self.parser.parse(text) # tprint("[parsed]") ; pp(parsed) expect = create_expect ( NonTerminal( option(), [ NonTerminal( short_adj_arg(), [ Terminal( short_adj_arg__option(), 0, '-f' ) , NonTerminal( operand(), [ Terminal( operand_all_caps(), 0, 'NORM' ) , ]) , ]) , ]) , )
def test_single_short_w_arg(self): text = '-fNORM' expect = create_expect( NonTerminal(option(), [ NonTerminal(short_adj_arg(), [ Terminal(short_adj_arg__option(), 0, '-f'), NonTerminal(operand(), [ Terminal(operand_all_caps(), 0, 'NORM'), ]), ]), ]), eof=(text[-1] != '\n'), ) self.parse_and_verify(text, expect)
def test_mixed(self): text = ' <a> <b> CC <d> EE FILE NORM ' text = text.strip() texts = text.split() p_ws1 = p_ws(' ') elements = [] for value in texts: rule = operand_angled if value[0] == '<' else operand_all_caps elements.append( NonTerminal(operand(), [Terminal(rule(), 0, value)])) elements.append(p_ws1.expect) if len(elements) > 0: del elements[-1] expect = NonTerminal(body(), [*elements, t_eof]) super().single(body, text, expect)
def option_line_expect(*terminals, eof=False, sep=None, indent=None, gap=None, help_=None, extra=0): if len(terminals) <= 0: raise ValueError( "No terminals provided. Please provide at least one.") separator = expect_separator(sep) sep_space = expect_separator(' ') # required for operands # print("[sep-space]") # pp(sep_space) members = [ NonTerminal( option_list(), [ NonTerminal(ol_first_option(), [ terminals[0], ]), *[ NonTerminal( ol_term_with_separator(), [ # separator, (sep_space if term.rule_name == 'operand' else separator), NonTerminal(ol_term(), [term]), ]) for term in terminals[1:] ], ]), Terminal(newline(), 0, '\n'), ] if indent and len(indent) > 0: members.insert(0, Terminal(StrMatch(' ', rule_name='wx'), 0, indent)) if help_ and len(help_) > 0: if extra < 0: extra = 0 # print(f": extra = {extra}") gap += ' ' * extra members.insert( -1, Terminal(StrMatch(gap, rule_name='option_line_gap'), 0, gap)) members.insert( -1, Terminal(StrMatch('.', rule_name='option_line_help'), 0, help_)) expect = NonTerminal(option_line(), [*members]) if eof: expect.append(Terminal(EOF(), 0, '')) return expect
def test_single_short_no_arg(self): text = '-f' expect = create_expect( NonTerminal(option(), [Terminal(short_no_arg(), 0, '-f')]), eof=(text[-1] != '\n'), ) self.parse_and_verify(text, expect)
def section_optdesc(line_specs, sep=', ', intro=None, indent=' ', offset=16): text = '' if intro: text = intro + ('' if intro[-1] == '\n' else '\n') opt_desc = NonTerminal(option_description_section(), [Terminal(StrMatch('.'), 0, 'place-holder')]) del opt_desc[0] for spec in line_specs: ( text_, expect_ ) = option_line_generate \ ( spec, sep=sep, indent=indent, offset=offset ) text += text_ opt_desc.append(expect_) return (text, opt_desc)
def test_usage__003_three_lines (self): # # FIXME: exactly where does the help text get inserted ? # optlst = olst( opt('-f', ), opt('-q', ), opt('--file', '=', '<file>') ) expr = _expr ( _wrap ( _optlst_expr( optlst ), (_repeatable, _term) ) ) choice_ = _choice ( ( expr, ) ) program_ = usage_prepare_program("naval-fate") pattern = usage_prepare_pattern( program_, choice_ ) line_1 = usage_prepare_line( pattern, t_newline ) optlst = olst( opt('-q', ' ', '<query>'), opt('--query', '=', '<query>') ) expr = _expr ( _wrap ( _optlst_expr( optlst ), (_repeatable, _term) ) ) choice_ = _choice ( ( expr, ) ) program_ = usage_prepare_program("naval-fate") pattern = usage_prepare_pattern( program_, choice_ ) line_2 = usage_prepare_line( pattern, t_newline ) optlst = olst( opt('-e', '', '<query>'), opt('--extract', '=', '<query>') ) expr = _expr ( _wrap ( _optlst_expr( optlst ), (_repeatable, _term) ) ) choice_ = _choice ( ( expr, ) ) program_ = usage_prepare_program("naval-fate") pattern = usage_prepare_pattern( program_, choice_ ) line_3 = usage_prepare_line( pattern, t_newline ) intro = usage_prepare_intro("Usage : ") text = intro[0] + line_1[0] + line_2[0] + line_3[0] expect = NonTerminal( usage_section(), [ intro[1], line_1[1], line_2[1], line_3[1] ] ) self.single ( usage_section, text, expect )
def create_expect ( *terminals, sep = None ) : separator = expect_separator(sep) sep_space = expect_separator(' ') # FIXME: create global for 'SPACE' if len(terminals) <= 0 : raise ValueError("No terminals provided. Please provide at least one.") expect = NonTerminal( document(), [ NonTerminal( body(), [ NonTerminal( element(), [ NonTerminal( option_list(), [ NonTerminal( ol_first_option(), [ terminals[0], ]) , * [ NonTerminal( ol_term_with_separator(), [ # separator, (sep_space if term.rule_name == 'operand' else separator) , NonTerminal( ol_term(), [ term ]) , ]) for term in terminals[1:] ], ]) , ]) , ]) , Terminal(EOF(), 0, '') , ]) return expect
def test_mixed(self): text = ' -a -b --file --form -l --why ' # text = text.strip() # texts = text.split() p_ws1 = p_ws(' ') elements = [] for value in texts: rule = long_no_arg if value[1] == '-' else short_no_arg option_ = Terminal(rule(), 0, value) option_ = NonTerminal(option(), [option_]) elements.append(option_) elements.append(p_ws1.expect) if len(elements) > 0: del elements[-1] expect = NonTerminal(body(), [*elements, t_eof]) super().single(body, text, expect)
def test_newline_elements_only(self): parser = ParserPython(document, skipws=False) text = '\n\n\n' parsed = parser.parse(text) # print('\n: parsed') ; pp(parsed) p_newline = Terminal(newline(), 0, '\n') p_element = NonTerminal(element(), [p_newline]) p_body = NonTerminal(body(), [p_element, p_element, p_element]) p_eof = Terminal(EOF(), 0, '') expect = NonTerminal(document(), [p_body, p_eof]) # print('\n: expect') ; pp(expect) assert parsed == expect, ( f"text = '{text}' :\n" f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}")
def usage_prepare_pattern(program, choice_=None): assert len(program) == 2 text = program[0] expect = [program[1]] if isinstance(choice_, tuple): assert len(choice_) == 2 text += ' ' + choice_[0] expect += [choice_[1]] return (text, NonTerminal(usage_pattern(), expect))
def usage_prepare_repeatable(text: str, child: ParseTreeNode, repeating=False, gap=''): """BRITTLE code since text does not also incorporate an enclosure.""" elements = [child] if repeating: text += gap + REPEATING elements.append(t_repeating) return (text, NonTerminal(repeatable(), elements))
def test_01_text_line_single (self) : def document(): return Sequence( ( text_line, EOF ), rule_name='document' ) # print('\n: document') ; pp(document()) parser = ParserPython( document, skipws=False ) text = self.line1 parsed = parser.parse(text) # print('\n: parsed') ; pp(parsed) p_newline = Terminal(newline(), 0, '\n') p_l1_words = Terminal(words(), 0, self.line1[:-1]) p_l1_text_line = NonTerminal(text_line(), [ p_l1_words, p_newline ]) p_eof = Terminal(EOF(), 0, '') expect = NonTerminal(document(), [p_l1_text_line, p_eof] ) # print('\n: expect') ; pp(expect) assert parsed == expect, ( f"text = '{text}' :\n" f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}" )
def test_with_some_paragraphs(self): parser = ParserPython(document, skipws=False) paragraph = f"{self.words1}\n{self.words2}\n{self.words3}\n" text = paragraph + '\n' + paragraph parsed = parser.parse(text) # print('\n: parsed') ; pp(parsed) x_paragraph = [ NonTerminal(element(), [ Terminal(words(), 0, self.words1), ]), NonTerminal(element(), [ Terminal(newline(), 0, '\n'), ]), NonTerminal(element(), [ Terminal(words(), 0, self.words2), ]), NonTerminal(element(), [ Terminal(newline(), 0, '\n'), ]), NonTerminal(element(), [ Terminal(words(), 0, self.words3), ]), NonTerminal(element(), [ Terminal(newline(), 0, '\n'), ]), ] expect = NonTerminal(document(), [ NonTerminal(body(), [ *x_paragraph, NonTerminal(element(), [ Terminal(blank_line(), 0, '\n'), ]), *x_paragraph, ]), Terminal(EOF(), 0, ''), ]) # print('\n: expect') ; pp(expect) assert parsed == expect, ( f"text = '{text}' :\n" f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}")
def term__operand(op): if re_operand_angled.fullmatch(op) : operand_type = operand_angled elif re_operand_all_caps.fullmatch(op) : operand_type = operand_all_caps else : raise ValueError( f"Invalid optdef operand '{op}'. Expected either an " f"angle operand, '<foo>', or all caps, 'FOO'. Please address.") return NonTerminal( operand(), [ Terminal( operand_type(), 0, op ) ] )
def test_single_short_no_arg (self): text = '-f' parsed = self.parser.parse(text) # tprint("[parsed]") ; pp(parsed) expect = create_expect ( NonTerminal( option(), [Terminal(short_no_arg(), 0, '-f')] ) ) assert nodes_equal(parsed, expect, verbose=True), \ ( f"[expect]\n{pp_str(expect)}\n" f"[parsed]\n{pp_str(parsed)}" f"text = '{text}' :\n" )
def test_create_expect(self): text = '-f -x -l' expect = create_expect( NonTerminal(option(), [Terminal(short_no_arg(), 0, '-f')]), NonTerminal(option(), [Terminal(short_no_arg(), 0, '-x')]), NonTerminal(option(), [Terminal(short_no_arg(), 0, '-l')]), eof=(text[-1] != '\n'), ) # print("[ expect ]") # pp(expect[0][0][0][0][1]) parsed = self.parse(text, expect) # tprint("[parsed]") ; pp(parsed) # print("[ parsed ]") # pp(parsed[0][0][0][0][1]) self.verify(text, expect, parsed)
def visit_repeatable(self, node, children, depth=0): if False and str(children) in [ "[command 'turn' [0], command 'rise' [0]]", "[command 'move' [0]]" ]: print(f": repeatable : {node.name} : {children}") pp(NonTerminal(node.rule, children)) print('') n_children = len(children) assert n_children in [1, 2] term = children[0] if n_children == 1: return term assert children[1] == self.REPEATING return NonTerminal(StrMatch(':repeating:', 'repeating'), [term])
def usage_prepare_line(pattern: tuple, end: ParseTreeNode): if not isinstance(end, Terminal) and end.rule_name in ['newline', 'EOF']: raise ValueError( f"<end> must be t_newline, t_ws_newline or t_eof from grammar.python.common. " f"Not {str(type(end))}. Please address.") (text, expect) = pattern if end is t_newline: text += '\n' return (text, NonTerminal(usage_line(), [expect, end]))