def test_basic3(self):
g = Grammar("start: '\(' name_list (COMMA MUL NAME)? '\)'; @name_list: NAME | name_list COMMA NAME ; MUL: '\*'; COMMA: ','; NAME: '\w+'; ")
l = g.parse('(a,b,c,*x)')
g = Grammar("start: '\(' name_list (COMMA MUL NAME)? '\)'; @name_list: NAME | name_list COMMA NAME ; MUL: '\*'; COMMA: ','; NAME: '\w+'; ")
l2 = g.parse('(a,b,c,*x)')
assert l == l2, '%s != %s' % (l, l2)
def test_basic2(self):
# Multiple parsers and colliding tokens
g = Grammar("start: B A ; B: '12'; A: '1'; ", auto_filter_tokens=False)
g2 = Grammar("start: B A; B: '12'; A: '2'; ", auto_filter_tokens=False)
x = g.parse('121')
assert x.head == 'start' and x.tail == ['12', '1'], x
x = g2.parse('122')
assert x.head == 'start' and x.tail == ['12', '2'], x
def test_basic1(self):
g = Grammar("start: a+ b a+? 'b' a*; b: 'b'; a: 'a';")
r = g.parse('aaabaab')
self.assertEqual(''.join(x.head for x in r.tail), 'aaabaa')
r = g.parse('aaabaaba')
self.assertEqual(''.join(x.head for x in r.tail), 'aaabaaa')
self.assertRaises(ParseError, g.parse, 'aaabaa')
def test_basic1(self):
g = Grammar("start: a+ b a+? 'b' a*; b: 'b'; a: 'a';")
r = g.parse('aaabaab')
self.assertEqual( ''.join(x.head for x in r.tail), 'aaabaa' )
r = g.parse('aaabaaba')
self.assertEqual( ''.join(x.head for x in r.tail), 'aaabaaa' )
self.assertRaises(ParseError, g.parse, 'aaabaa')
def test_recurse_expansion(self):
"""Verify that stack depth doesn't get exceeded on recursive rules marked for expansion."""
g = Grammar(r"""@start: a | start a ; a : A ; A : 'a' ;""")
# Force PLY to write to the debug log, but prevent writing it to the terminal (uses repr() on the half-built
# STree data structures, which uses recursion).
g._grammar.debug = yacc.NullLogger()
g.parse("a" * (sys.getrecursionlimit() / 4))
def test_recurse_expansion(self):
"""Verify that stack depth doesn't get exceeded on recursive rules marked for expansion."""
g = Grammar(r"""@start: a | start a ; a : A ; A : 'a' ;""")
# Force PLY to write to the debug log, but prevent writing it to the terminal (uses repr() on the half-built
# STree data structures, which uses recursion).
g._grammar.debug = yacc.NullLogger()
g.parse("a" * (sys.getrecursionlimit() / 4))
def test_python_lex(code=FIB, expected=54):
g = Grammar(_read('python.g'))
l = list(g.lex(code))
for x in l:
y = x.value
if isinstance(y, TokValue):
logging.debug('%s %s %s', y.type, y.line, y.column)
else:
logging.debug('%s %s', x.type, x.value)
assert len(l) == expected, len(l)
def test_python_lex(code=FIB, expected=54):
g = Grammar(_read('python.g'))
l = list(g.lex(code))
for x in l:
y = x.value
if isinstance(y, TokValue):
logging.debug('%s %s %s', y.type, y.line, y.column)
else:
logging.debug('%s %s', x.type, x.value)
assert len(l) == expected, len(l)
def test_expand1_lists_with_one_item(self):
g = Grammar(r"""start: list ;
?list: item+ ;
item : A ;
A: 'a' ;
""")
r = g.parse("a")
# because 'list' is an expand-if-contains-one rule and we only provided one element it should have expanded to 'item'
self.assertSequenceEqual([subtree.head for subtree in r.tail], ('item',))
# regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
self.assertEqual(len(r.tail), 1)
def test_multiple_item_flatten_list(self):
g = Grammar(r"""start: list ;
#list: | item ',' list ;
item : A ;
A: 'a' ;
""")
r = g.parse("a,a,")
# Because 'list' is a flatten rule it's top-level element should *never* be expanded
self.assertSequenceEqual([subtree.head for subtree in r.tail], ('list',))
# Sanity check: verify that 'list' contains exactly the two 'item's we've given it
[list] = r.tail
self.assertSequenceEqual([item.head for item in list.tail], ('item', 'item'))
def test_dont_expand1_lists_with_multiple_items_2(self):
g = Grammar(r"""start: list ;
?list: item+ '!';
item : A ;
A: 'a' ;
""")
r = g.parse("aa!")
# because 'list' is an expand-if-contains-one rule and we've provided more than one element it should *not* have expanded
self.assertSequenceEqual([subtree.head for subtree in r.tail], ('list',))
# regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
self.assertEqual(len(r.tail), 1)
# Sanity check: verify that 'list' contains the two 'item's we've given it
[list] = r.tail
self.assertSequenceEqual([item.head for item in list.tail], ('item', 'item'))
def test_empty_expand1_list_2(self):
g = Grammar(r"""start: list ;
?list: item* '!'?;
item : A ;
A: 'a' ;
""")
r = g.parse("")
# because 'list' is an expand-if-contains-one rule and we've provided less than one element (i.e. none) it should *not* have expanded
self.assertSequenceEqual([subtree.head for subtree in r.tail], ('list',))
# regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
self.assertEqual(len(r.tail), 1)
# Sanity check: verify that 'list' contains no 'item's as we've given it none
[list] = r.tail
self.assertSequenceEqual([item.head for item in list.tail], ())
def test_basic3(self):
g = Grammar("start: '\(' name_list (COMMA MUL NAME)? '\)'; @name_list: NAME | name_list COMMA NAME ; MUL: '\*'; COMMA: ','; NAME: '\w+'; ")
l = g.parse('(a,b,c,*x)')
g = Grammar("start: '\(' name_list (COMMA MUL NAME)? '\)'; @name_list: NAME | name_list COMMA NAME ; MUL: '\*'; COMMA: ','; NAME: '\w+'; ")
l2 = g.parse('(a,b,c,*x)')
assert l == l2, '%s != %s' % (l, l2)
def test_basic2(self):
# Multiple parsers and colliding tokens
g = Grammar("start: B A ; B: '12'; A: '1'; ", auto_filter_tokens=False)
g2 = Grammar("start: B A; B: '12'; A: '2'; ", auto_filter_tokens=False)
x = g.parse('121')
assert x.head == 'start' and x.tail == ['12', '1'], x
x = g2.parse('122')
assert x.head == 'start' and x.tail == ['12', '2'], x
def test_unicode(self):
g = Grammar(r"""start: UNIA UNIB UNIA;
UNIA: '\xa3';
UNIB: '\u0101';
""")
g.parse(u'\xa3\u0101\u00a3')
def setUp(self):
tree_grammar = Grammar("start: branch; branch: name ('{' branch* '}')?; name: '[a-z]';")
self.tree1 = tree_grammar.parse("a{b{cde}}")
self.tree2 = tree_grammar.parse("a{abc{bbab}c}")
def test_unicode(self):
g = Grammar(r"""start: UNIA UNIB UNIA;
UNIA: '\xa3';
UNIB: '\u0101';
""")
g.parse(u'\xa3\u0101\u00a3')
def test_stringio_unicode(self):
"""Verify that a Grammar can be created from file-like objects other than Python's standard 'file' object"""
Grammar(uStringIO(u"start: a+ b a+? 'b' a*; b: 'b'; a: 'a';"))
def setUp(self):
with grammars.open('config.g') as g:
self.g = Grammar(g)
def setUp(self):
with grammars.open('python.g') as g:
self.g = Grammar(g)
bin_expr = _bin_operator
# arg = _assignment
assign = _assignment
if_expr = _if
def _read(n, *args):
kwargs = {'encoding': 'iso-8859-1'}
with open(os.path.join(CUR_PATH, n), *args, **kwargs) as f:
return f.read()
if os.name == 'nt':
if 'PyPy' in sys.version:
PYTHON_LIB = os.path.join(sys.prefix, 'lib-python', sys.winver)
else:
PYTHON_LIB = os.path.join(sys.prefix, 'Lib')
else:
PYTHON_LIB = [
x for x in sys.path if x.endswith('%s.%s' % sys.version_info[:2])
][0]
# print(os.getcwd())
with grammars.open(os.getcwd() + '/grammar.g') as g:
g = Grammar(g)
p = RParser()
tree = g.parse(_read('sample.r'))
tree.to_png_with_pydot('list_parser_tree.png')