def test_02_text_line_pair (self) :
def body():
return OneOrMore ( [ text_line ], rule_name='body' )
def document():
return Sequence( ( body, EOF ), rule_name='document' )
# print('\n: document') ; pp(document())
parser = ParserPython( document, skipws=False )
text = self.line1 + self.line2
parsed = parser.parse(text)
# print('\n: parsed') ; pp(parsed)
#
# print('\n: flatten') ; pp(flatten(parsed))
p_newline = Terminal(newline(), 0, '\n')
p_l1_words = Terminal(words(), 0, self.line1[:-1])
p_l1_text_line = NonTerminal(text_line(), [ p_l1_words, p_newline ])
p_l2_words = Terminal(words(), 0, self.line2[:-1])
p_l2_text_line = NonTerminal(text_line(), [ p_l2_words, p_newline ])
p_body = NonTerminal(body(), [ p_l1_text_line, p_l2_text_line ])
p_eof = Terminal(EOF(), 0, '')
expect = NonTerminal(document(), [p_body, p_eof] )
# print('\n: expect') ; pp(expect)
assert parsed == expect, ( f"text = '{text}' :\n"
f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}" )
def visit(self, node, depth=0):
i = ' ' * 3 * depth
# print(f"{i} [ p2 : node = {node}")
if not hasattr(node, 'rule_name'):
# print(f"{i} - not a known container : {str(type(node))}")
# print(f"{i} => itself")
# print(f"{i} ]")
if isinstance(node, list) and len(node) == 1:
return node[0]
return node
#----------------------------------------------------------------------
children = []
if isinstance(node, (NonTerminal, SemanticActionResults)):
# print(f"{i} - visiting children of '{node.name}' : len = {len(node)}")
# each of these object types is a list
for child in node:
response = self.visit(child, 1 + depth)
if response:
children.append(response)
# print(f"{i} - visited children = {children}")
#----------------------------------------------------------------------
# rule name specific visitor ?
rule_name = str(node.rule_name)
method = f"visit_{rule_name}"
# print(f"{i} - {method} ?")
if hasattr(self, method):
# print(f"{i} - method found, applying to {node.name}")
out = getattr(self, method)(node, children, 1 + depth)
# print(f"{i} => {_res(out,i)}")
# print(f"{i} ]")
return out
# else :
# print(f"{i} - no such method")
#----------------------------------------------------------------------
if len(children) <= 0:
out = Terminal(node.rule, 0, node.value)
# print(f"{i} => {_res(out,i)}")
# print(f"{i} ]")
return out
if len(children) == 1:
children = children[0]
try:
out = NonTerminal(node.rule, children)
except:
out = NonTerminal(node.rule, [Unwrap(children)])
# automatically unwrap
out[0] = out[0].value
# print(f"{i} => {_res(out,i)}")
# print(f"{i} ]")
return out
def test_05_paragraph_multiple (self) :
def body():
return OneOrMore ( OrderedChoice( [ paragraph, newline ] ), rule_name='body' )
def document():
return Sequence( ( body, EOF ), rule_name='document' )
# print('\n: document') ; pp(document())
parser = ParserPython( document, skipws=False )
text = self.line1 + self.line2 + '\n'
text = text * 3
parsed = parser.parse(text)
# print('\n: parsed') ; pp(parsed)
p_newline = Terminal(newline(), 0, '\n')
p_l1_words = Terminal(words(), 0, self.line1[:-1])
p_l1_text_line = NonTerminal(text_line(), [ p_l1_words, p_newline ])
p_l2_words = Terminal(words(), 0, self.line2[:-1])
p_l2_text_line = NonTerminal(text_line(), [ p_l2_words, p_newline ])
p_paragraph = NonTerminal(paragraph(), [ p_l1_text_line, p_l2_text_line ])
p_body = NonTerminal(body(), [
p_paragraph, p_newline,
p_paragraph, p_newline,
p_paragraph, p_newline,
])
p_eof = Terminal(EOF(), 0, '')
expect = NonTerminal(document(), [ p_body, p_eof ] )
# print('\n: expect') ; pp(expect)
assert parsed == expect, ( f"text = '{text}' :\n"
f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}" )
def test_004__n1_eq_t1__with_eq(self):
if False:
from prettyprinter import cpprint as pp
import p
print(f": test_004__n1_eq_t1__with_eq")
print(f": n1 :")
pp(self.n1)
print(f": t1 :")
pp(self.t1)
NonTerminal_enable_structural_eq()
with self.assertRaises(AssertionError) as context:
assert self.n1 == self.t1 # now it fails
self.assertTrue('Internal error, AssertionError not raised !!!')
assert self.n1 != self.t1
assert self.n1 == deepcopy(self.n1)
t2 = Terminal(self.dot, 0, 'one')
n2 = NonTerminal(self.dot, [t2])
assert self.n1 == n2
t2 = Terminal(self.dot, 0, 'one')
n2 = NonTerminal(self.dot, [t2])
assert self.n1 == n2
bang = StrMatch('!', rule_name='bang')
t3 = Terminal(bang, 0, 'one')
n3 = NonTerminal(bang, [t3])
assert self.n1 != n3
def setUp(self):
self.dot = StrMatch('.', rule_name='self.dot')
self.s1 = 's1 : string'
self.s2 = 's2 : string'
self.s3 = 's3 : string'
# rule, position, value
self.t1 = Terminal(self.dot, 0, 'one')
self.t2 = Terminal(self.dot, 0, 'two')
self.t3 = Terminal(self.dot, 0, 'three')
assert not isinstance(self.t1, list)
assert not isinstance(self.t2, list)
assert not isinstance(self.t3, list)
# rule, value : a list where the first element is a node
# self.n1 = NonTerminal(self.dot, self.t1) # TypeError: 'Terminal' object is not subscriptable
self.n2 = NonTerminal(self.dot, [self.t1])
self.n3 = NonTerminal(self.dot, self.n2)
self.n4 = NonTerminal(self.dot, [self.n2])
assert isinstance(self.n2, list)
assert isinstance(self.n3, list)
assert isinstance(self.n4, list)
self.v0 = self.n2
self.v1 = [self.s1, self.s2]
self.v2 = self.t1
self.v3s = self.s3
self.v3t = (self.s1, self.s2)
def test_with_some_words_2(self):
parser = ParserPython(document, skipws=False)
text = f"{self.words1}\n\n"
parsed = parser.parse(text)
# print('\n: parsed') ; pp(parsed)
expect = NonTerminal(document(), [
NonTerminal(body(), [
NonTerminal(element(), [
Terminal(words(), 0, self.words1),
]),
NonTerminal(element(), [
Terminal(newline(), 0, '\n'),
]),
NonTerminal(element(), [
Terminal(blank_line(), 0, '\n'),
]),
]),
Terminal(EOF(), 0, ''),
])
# print('\n: expect') ; pp(expect)
assert parsed == expect, (
f"text = '{text}' :\n"
f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}")
def test_short_adj_arg(self):
value = '-fFILE'
option_ = Terminal(long_no_arg(), 0, '-f')
operand_ = Terminal(operand_all_caps(), 0, 'FILE')
operand_ = NonTerminal(operand(), [operand_])
expect = NonTerminal(short_adj_arg(), [option_, operand_])
expect = NonTerminal(option(), [expect])
expect = NonTerminal(body(), [expect, t_eof])
super().single(body, value, expect)
def test_short_gap_arg(self):
value = '-f FILE'
option_ = Terminal(short_no_arg(), 0, '-f')
option_ = NonTerminal(option(), [option_])
p_ws1 = p_ws(' ')
operand_ = Terminal(operand_all_caps(), 0, 'FILE')
operand_ = NonTerminal(operand(), [operand_])
expect = NonTerminal(body(), [option_, p_ws1.expect, operand_, t_eof])
super().single(body, value, expect)
def test_long_gap_arg(self):
value = '--file <a-file>'
option_ = Terminal(long_no_arg(), 0, '--file')
option_ = NonTerminal(option(), [option_])
p_ws1 = p_ws(' ')
operand_ = Terminal(operand_angled(), 0, '<a-file>')
operand_ = NonTerminal(operand(), [operand_])
expect = NonTerminal(body(), [option_, p_ws1.expect, operand_, t_eof])
super().single(body, value, expect)
def test_long_eq_arg(self):
value = '--file=<a-file>'
option_ = Terminal(long_no_arg(), 0, '--file')
operand_ = Terminal(operand_angled(), 0, '<a-file>')
operand_ = NonTerminal(operand(), [operand_])
expect = NonTerminal(long_eq_arg(), [option_, t_equals, operand_])
expect = NonTerminal(option(), [expect])
expect = NonTerminal(body(), [expect, t_eof])
super().single(body, value, expect)
def expect_document(sections):
if len(sections) <= 0:
raise ValueError("No sections provided. Please provide at least one.")
expect = NonTerminal(document(), [
NonTerminal(body(), [NonTerminal(element(), sections)]),
Terminal(EOF(), 0, ''),
])
return expect
def test_single_short_with_one_arg (self):
text = '-fNORM'
parsed = self.parser.parse(text)
# tprint("[parsed]") ; pp(parsed)
expect = create_expect (
NonTerminal( option(), [
NonTerminal( short_adj_arg(), [
Terminal( short_adj_arg__option(), 0, '-f' ) ,
NonTerminal( operand(), [
Terminal( operand_all_caps(), 0, 'NORM' ) ,
]) ,
]) ,
]) ,
)
def test_single_short_w_arg(self):
text = '-fNORM'
expect = create_expect(
NonTerminal(option(), [
NonTerminal(short_adj_arg(), [
Terminal(short_adj_arg__option(), 0, '-f'),
NonTerminal(operand(), [
Terminal(operand_all_caps(), 0, 'NORM'),
]),
]),
]),
eof=(text[-1] != '\n'),
)
self.parse_and_verify(text, expect)
def test_mixed(self):
text = ' <a> <b> CC <d> EE FILE NORM '
text = text.strip()
texts = text.split()
p_ws1 = p_ws(' ')
elements = []
for value in texts:
rule = operand_angled if value[0] == '<' else operand_all_caps
elements.append(
NonTerminal(operand(), [Terminal(rule(), 0, value)]))
elements.append(p_ws1.expect)
if len(elements) > 0:
del elements[-1]
expect = NonTerminal(body(), [*elements, t_eof])
super().single(body, text, expect)
def option_line_expect(*terminals,
eof=False,
sep=None,
indent=None,
gap=None,
help_=None,
extra=0):
if len(terminals) <= 0:
raise ValueError(
"No terminals provided. Please provide at least one.")
separator = expect_separator(sep)
sep_space = expect_separator(' ') # required for operands
# print("[sep-space]")
# pp(sep_space)
members = [
NonTerminal(
option_list(),
[
NonTerminal(ol_first_option(), [
terminals[0],
]),
*[
NonTerminal(
ol_term_with_separator(),
[
# separator,
(sep_space
if term.rule_name == 'operand' else separator),
NonTerminal(ol_term(), [term]),
]) for term in terminals[1:]
],
]),
Terminal(newline(), 0, '\n'),
]
if indent and len(indent) > 0:
members.insert(0, Terminal(StrMatch(' ', rule_name='wx'), 0, indent))
if help_ and len(help_) > 0:
if extra < 0:
extra = 0
# print(f": extra = {extra}")
gap += ' ' * extra
members.insert(
-1, Terminal(StrMatch(gap, rule_name='option_line_gap'), 0, gap))
members.insert(
-1, Terminal(StrMatch('.', rule_name='option_line_help'), 0,
help_))
expect = NonTerminal(option_line(), [*members])
if eof:
expect.append(Terminal(EOF(), 0, ''))
return expect
def test_single_short_no_arg(self):
text = '-f'
expect = create_expect(
NonTerminal(option(), [Terminal(short_no_arg(), 0, '-f')]),
eof=(text[-1] != '\n'),
)
self.parse_and_verify(text, expect)
def section_optdesc(line_specs, sep=', ', intro=None, indent=' ', offset=16):
text = ''
if intro:
text = intro + ('' if intro[-1] == '\n' else '\n')
opt_desc = NonTerminal(option_description_section(),
[Terminal(StrMatch('.'), 0, 'place-holder')])
del opt_desc[0]
for spec in line_specs:
( text_, expect_ ) = option_line_generate \
( spec, sep=sep, indent=indent, offset=offset )
text += text_
opt_desc.append(expect_)
return (text, opt_desc)
def test_usage__003_three_lines (self):
#
# FIXME: exactly where does the help text get inserted ?
#
optlst = olst( opt('-f', ), opt('-q', ), opt('--file', '=', '<file>') )
expr = _expr ( _wrap ( _optlst_expr( optlst ), (_repeatable, _term) ) )
choice_ = _choice ( ( expr, ) )
program_ = usage_prepare_program("naval-fate")
pattern = usage_prepare_pattern( program_, choice_ )
line_1 = usage_prepare_line( pattern, t_newline )
optlst = olst( opt('-q', ' ', '<query>'), opt('--query', '=', '<query>') )
expr = _expr ( _wrap ( _optlst_expr( optlst ), (_repeatable, _term) ) )
choice_ = _choice ( ( expr, ) )
program_ = usage_prepare_program("naval-fate")
pattern = usage_prepare_pattern( program_, choice_ )
line_2 = usage_prepare_line( pattern, t_newline )
optlst = olst( opt('-e', '', '<query>'), opt('--extract', '=', '<query>') )
expr = _expr ( _wrap ( _optlst_expr( optlst ), (_repeatable, _term) ) )
choice_ = _choice ( ( expr, ) )
program_ = usage_prepare_program("naval-fate")
pattern = usage_prepare_pattern( program_, choice_ )
line_3 = usage_prepare_line( pattern, t_newline )
intro = usage_prepare_intro("Usage : ")
text = intro[0] + line_1[0] + line_2[0] + line_3[0]
expect = NonTerminal( usage_section(), [ intro[1], line_1[1], line_2[1], line_3[1] ] )
self.single ( usage_section, text, expect )
def create_expect ( *terminals, sep = None ) :
separator = expect_separator(sep)
sep_space = expect_separator(' ')
# FIXME: create global for 'SPACE'
if len(terminals) <= 0 :
raise ValueError("No terminals provided. Please provide at least one.")
expect = NonTerminal( document(), [
NonTerminal( body(), [
NonTerminal( element(), [
NonTerminal( option_list(), [
NonTerminal( ol_first_option(), [ terminals[0], ]) ,
* [
NonTerminal( ol_term_with_separator(), [
# separator,
(sep_space if term.rule_name == 'operand'
else separator) ,
NonTerminal( ol_term(), [ term ]) ,
])
for term in terminals[1:]
],
]) ,
]) ,
]) ,
Terminal(EOF(), 0, '') ,
])
return expect
def test_mixed(self):
text = ' -a -b --file --form -l --why '
#
text = text.strip()
#
texts = text.split()
p_ws1 = p_ws(' ')
elements = []
for value in texts:
rule = long_no_arg if value[1] == '-' else short_no_arg
option_ = Terminal(rule(), 0, value)
option_ = NonTerminal(option(), [option_])
elements.append(option_)
elements.append(p_ws1.expect)
if len(elements) > 0:
del elements[-1]
expect = NonTerminal(body(), [*elements, t_eof])
super().single(body, text, expect)
def test_newline_elements_only(self):
parser = ParserPython(document, skipws=False)
text = '\n\n\n'
parsed = parser.parse(text)
# print('\n: parsed') ; pp(parsed)
p_newline = Terminal(newline(), 0, '\n')
p_element = NonTerminal(element(), [p_newline])
p_body = NonTerminal(body(), [p_element, p_element, p_element])
p_eof = Terminal(EOF(), 0, '')
expect = NonTerminal(document(), [p_body, p_eof])
# print('\n: expect') ; pp(expect)
assert parsed == expect, (
f"text = '{text}' :\n"
f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}")
def usage_prepare_pattern(program, choice_=None):
assert len(program) == 2
text = program[0]
expect = [program[1]]
if isinstance(choice_, tuple):
assert len(choice_) == 2
text += ' ' + choice_[0]
expect += [choice_[1]]
return (text, NonTerminal(usage_pattern(), expect))
def usage_prepare_repeatable(text: str,
child: ParseTreeNode,
repeating=False,
gap=''):
"""BRITTLE code since text does not also incorporate an enclosure."""
elements = [child]
if repeating:
text += gap + REPEATING
elements.append(t_repeating)
return (text, NonTerminal(repeatable(), elements))
def test_01_text_line_single (self) :
def document():
return Sequence( ( text_line, EOF ), rule_name='document' )
# print('\n: document') ; pp(document())
parser = ParserPython( document, skipws=False )
text = self.line1
parsed = parser.parse(text)
# print('\n: parsed') ; pp(parsed)
p_newline = Terminal(newline(), 0, '\n')
p_l1_words = Terminal(words(), 0, self.line1[:-1])
p_l1_text_line = NonTerminal(text_line(), [ p_l1_words, p_newline ])
p_eof = Terminal(EOF(), 0, '')
expect = NonTerminal(document(), [p_l1_text_line, p_eof] )
# print('\n: expect') ; pp(expect)
assert parsed == expect, ( f"text = '{text}' :\n"
f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}" )
def test_with_some_paragraphs(self):
parser = ParserPython(document, skipws=False)
paragraph = f"{self.words1}\n{self.words2}\n{self.words3}\n"
text = paragraph + '\n' + paragraph
parsed = parser.parse(text)
# print('\n: parsed') ; pp(parsed)
x_paragraph = [
NonTerminal(element(), [
Terminal(words(), 0, self.words1),
]),
NonTerminal(element(), [
Terminal(newline(), 0, '\n'),
]),
NonTerminal(element(), [
Terminal(words(), 0, self.words2),
]),
NonTerminal(element(), [
Terminal(newline(), 0, '\n'),
]),
NonTerminal(element(), [
Terminal(words(), 0, self.words3),
]),
NonTerminal(element(), [
Terminal(newline(), 0, '\n'),
]),
]
expect = NonTerminal(document(), [
NonTerminal(body(), [
*x_paragraph,
NonTerminal(element(), [
Terminal(blank_line(), 0, '\n'),
]),
*x_paragraph,
]),
Terminal(EOF(), 0, ''),
])
# print('\n: expect') ; pp(expect)
assert parsed == expect, (
f"text = '{text}' :\n"
f"[expect]\n{pp_str(expect)}\n[parsed]\n{pp_str(parsed)}")
def term__operand(op):
if re_operand_angled.fullmatch(op) :
operand_type = operand_angled
elif re_operand_all_caps.fullmatch(op) :
operand_type = operand_all_caps
else :
raise ValueError(
f"Invalid optdef operand '{op}'. Expected either an "
f"angle operand, '<foo>', or all caps, 'FOO'. Please address.")
return NonTerminal( operand(), [ Terminal( operand_type(), 0, op ) ] )
def test_single_short_no_arg (self):
text = '-f'
parsed = self.parser.parse(text)
# tprint("[parsed]") ; pp(parsed)
expect = create_expect (
NonTerminal( option(), [Terminal(short_no_arg(), 0, '-f')] )
)
assert nodes_equal(parsed, expect, verbose=True), \
( f"[expect]\n{pp_str(expect)}\n"
f"[parsed]\n{pp_str(parsed)}"
f"text = '{text}' :\n" )
def test_create_expect(self):
text = '-f -x -l'
expect = create_expect(
NonTerminal(option(), [Terminal(short_no_arg(), 0, '-f')]),
NonTerminal(option(), [Terminal(short_no_arg(), 0, '-x')]),
NonTerminal(option(), [Terminal(short_no_arg(), 0, '-l')]),
eof=(text[-1] != '\n'),
)
# print("[ expect ]")
# pp(expect[0][0][0][0][1])
parsed = self.parse(text, expect)
# tprint("[parsed]") ; pp(parsed)
# print("[ parsed ]")
# pp(parsed[0][0][0][0][1])
self.verify(text, expect, parsed)
def visit_repeatable(self, node, children, depth=0):
if False and str(children) in [
"[command 'turn' [0], command 'rise' [0]]",
"[command 'move' [0]]"
]:
print(f": repeatable : {node.name} : {children}")
pp(NonTerminal(node.rule, children))
print('')
n_children = len(children)
assert n_children in [1, 2]
term = children[0]
if n_children == 1:
return term
assert children[1] == self.REPEATING
return NonTerminal(StrMatch(':repeating:', 'repeating'), [term])
def usage_prepare_line(pattern: tuple, end: ParseTreeNode):
if not isinstance(end, Terminal) and end.rule_name in ['newline', 'EOF']:
raise ValueError(
f"<end> must be t_newline, t_ws_newline or t_eof from grammar.python.common. "
f"Not {str(type(end))}. Please address.")
(text, expect) = pattern
if end is t_newline:
text += '\n'
return (text, NonTerminal(usage_line(), [expect, end]))