def auto_memoize(self, conservative=None, full=True, d=0):
'''
This configuration attempts to detect which memoizer is most effective
for each matcher. As such it is a general "fix" for left-recursive
grammars and is suggested in the warning shown when the right-only
memoizer detects left recursion.
Lepl does not guarantee that all left-recursive grammars are handled
correctly. The corrections applied may be incomplete and can be
inefficient. It is always better to re-write a grammar to avoid
left-recursion. One way to improve efficiency, at the cost of less
accurate matching, is to specify a non-zero ``d`` parameter - this
is the maximum iteration depth that will be used (by default, when
``d`` is zero, it is the length of the remaining input, which can
be very large).
'''
from lepl.core.rewriters import AutoMemoize
from lepl.matchers.memo import LMemo, RMemo
self.no_memoize()
return self.add_rewriter(
AutoMemoize(conservative=conservative,
left=LMemo,
right=RMemo if full else None,
d=d))
def auto_memoize(self, conservative=False, full=False):
'''
LEPL can add memoization so that (1) complex matching is more
efficient and (2) left recursive grammars do not loop indefinitely.
However, in many common cases memoization decreases performance.
This matcher therefore, by default, only adds memoization if it
appears necessary for stability (case 2 above, when left recursion
is possible).
The ``conservative`` parameter controls how left-recursive loops are
detected. If False (default) then matchers are assumed to "consume"
input. This may be incorrect, in which case some left-recursive loops
may be missed. If True then all loops are considered left-recursive.
This is safer, but results in much more expensive (slower) memoisation.
The `full` parameter can be used (set to True) to add memoization
for case (1) above, in addition to case (2). In other words, when
True, all nodes are memozied; when False (the default) only
left-recursive nodes are memoized.
This is part of the default configuration.
See also `no_memoize()`.
'''
from lepl.core.rewriters import AutoMemoize
from lepl.matchers.memo import LMemo, RMemo
self.no_memoize()
return self.add_rewriter(
AutoMemoize(conservative, LMemo, RMemo if full else None))
def test_describe(self):
'''
Use a description of the graph to check against changes.
'''
#basicConfig(level=DEBUG)
name = Word(Letter()) > 'name'
expression = Delayed()
variable = Delayed()
function = (expression / '()') > 'function'
expression += (variable | function) > 'expression'
variable += (name | expression / '.' / name)
dotted_name = function & Eos()
base = dotted_name.tree()
# print(base)
desc0 = NodeStats(dotted_name)
print(desc0)
assert desc0.total == 18, desc0
self.assert_count(desc0, And, 5)
self.assert_count(desc0, Or, 2)
self.assert_count(desc0, Delayed, 2)
clone0 = clone_matcher(dotted_name)
# print(clone0.tree())
diff = Differ()
diff_text = '\n'.join(
diff.compare(base.split('\n'),
clone0.tree().split('\n')))
#print(diff_text)
descx = NodeStats(clone0)
print(descx)
assert descx == desc0
clone1 = Flatten()(dotted_name)
print(clone1.tree())
desc1 = NodeStats(clone1)
print(desc1)
# flattened And (Or no longer flattened as Delayed intervenes)
assert desc1.total == 17, desc1
self.assert_count(desc1, And, 4)
self.assert_count(desc1, Or, 2)
self.assert_count(desc1, Delayed, 2)
self.assert_count(desc1, Transform, 7)
self.assert_count(desc1, TransformationWrapper, 7)
clone2 = ComposeTransforms()(clone1)
desc2 = NodeStats(clone2)
#print(desc2)
# compressed a transform
assert desc2.total == 17, desc2
self.assert_count(desc2, And, 4)
self.assert_count(desc2, Or, 2)
self.assert_count(desc2, Delayed, 2)
self.assert_count(desc2, Transform, 6)
self.assert_count(desc2, TransformationWrapper, 6)
clone3 = RightMemoize()(clone2)
desc3 = NodeStats(clone3)
#print(desc3)
assert desc3.total == 17, desc3
self.assert_count(desc3, _RMemo, 17)
self.assert_count(desc3, Delayed, 2)
clone4 = LeftMemoize()(clone2)
desc4 = NodeStats(clone4)
#print(desc4)
assert desc4.total == 17, desc4
self.assert_count(desc4, _LMemo, 20)
# left memo duplicates delayed
self.assert_count(desc4, Delayed, 3)
clone5 = AutoMemoize(left=LMemo, right=RMemo)(clone2)
desc5 = NodeStats(clone5)
#print(desc5)
assert desc5.total == 17, desc5
self.assert_count(desc5, _RMemo, 5)
self.assert_count(desc5, _LMemo, 15)
# left memo duplicates delayed
self.assert_count(desc5, Delayed, 3)
try:
clone3.config.clear()
clone3.parse_string('1join()')
assert False, 'Expected error'
except MemoException as error:
assert 'Left recursion was detected' in str(error), str(error)
clone4.config.clear()
clone4.parse_string('1join()')
clone5.config.clear()
clone5.parse_string('1join()')
def test_describe(self):
'''
Use a description of the graph to check against changes.
'''
#basicConfig(level=DEBUG)
name = Word(Letter()) > 'name'
expression = Delayed()
variable = Delayed()
function = (expression / '()') > 'function'
expression += (variable | function) > 'expression'
variable += (name | expression / '.' / name)
dotted_name = function & Eos()
desc0 = NodeStats(dotted_name)
#print(desc0)
assert desc0.total == 18, desc0
self.assert_count(desc0, And, 5)
self.assert_count(desc0, Or, 2)
self.assert_count(desc0, Delayed, 2)
clone1 = Flatten()(dotted_name)
desc1 = NodeStats(clone1)
#print(desc1)
# flattened two matchers - one each of And and Or
assert desc1.total == 16, desc1
self.assert_count(desc1, And, 4)
self.assert_count(desc1, Or, 1)
self.assert_count(desc1, Delayed, 2)
self.assert_count(desc1, Transform, 7)
self.assert_count(desc1, TransformationWrapper, 7)
clone2 = ComposeTransforms()(clone1)
desc2 = NodeStats(clone2)
#print(desc2)
# compressed two transforms
assert desc2.total == 16, desc2
self.assert_count(desc2, And, 4)
self.assert_count(desc2, Or, 1)
self.assert_count(desc2, Delayed, 2)
self.assert_count(desc2, Transform, 2)
self.assert_count(desc2, TransformationWrapper, 2)
clone3 = Memoize(RMemo)(clone2)
desc3 = NodeStats(clone3)
#print(desc3)
assert desc3.total == 16, desc3
self.assert_count(desc3, _RMemo, 18)
self.assert_count(desc3, Delayed, 2)
clone4 = Memoize(LMemo)(clone2)
desc4 = NodeStats(clone4)
#print(desc4)
assert desc4.total == 16, desc4
self.assert_count(desc4, _LMemo, 18)
self.assert_count(desc4, Delayed, 2)
clone5 = AutoMemoize(left=LMemo, right=RMemo)(clone2)
desc5 = NodeStats(clone5)
#print(desc5)
assert desc5.total == 16, desc5
self.assert_count(desc5, _RMemo, 13)
self.assert_count(desc5, _LMemo, 5)
self.assert_count(desc5, Delayed, 2)
try:
clone3.config.clear()
clone3.parse_string('1join()')
assert False, 'Expected error'
except MemoException as error:
assert 'Left recursion with RMemo?' in str(error), str(error)
clone4.config.clear()
clone4.parse_string('1join()')
clone5.config.clear()
clone5.parse_string('1join()')
def test_describe(self):
'''
Use a description of the graph to check against changes.
'''
#basicConfig(level=DEBUG)
name = Word(Letter()) > 'name'
expression = Delayed()
variable = Delayed()
function = (expression / '()') > 'function'
expression += (variable | function) > 'expression'
variable += (name | expression / '.' / name)
dotted_name = function & Eos()
base = dotted_name.tree()
# print(base)
desc0 = NodeStats(dotted_name)
print(desc0)
assert desc0.total == 18, desc0
self.assert_count(desc0, And, 5)
self.assert_count(desc0, Or, 2)
self.assert_count(desc0, Delayed, 2)
clone0 = clone_matcher(dotted_name)
# print(clone0.tree())
diff = Differ()
diff_text = '\n'.join(diff.compare(base.split('\n'), clone0.tree().split('\n')))
#print(diff_text)
descx = NodeStats(clone0)
print(descx)
assert descx == desc0
clone1 = Flatten()(dotted_name)
print(clone1.tree())
desc1 = NodeStats(clone1)
print(desc1)
# flattened And (Or no longer flattened as Delayed intervenes)
assert desc1.total == 17, desc1
self.assert_count(desc1, And, 4)
self.assert_count(desc1, Or, 2)
self.assert_count(desc1, Delayed, 2)
self.assert_count(desc1, Transform, 7)
self.assert_count(desc1, TransformationWrapper, 7)
clone2 = ComposeTransforms()(clone1)
desc2 = NodeStats(clone2)
#print(desc2)
# compressed a transform
assert desc2.total == 17, desc2
self.assert_count(desc2, And, 4)
self.assert_count(desc2, Or, 2)
self.assert_count(desc2, Delayed, 2)
self.assert_count(desc2, Transform, 6)
self.assert_count(desc2, TransformationWrapper, 6)
clone3 = RightMemoize()(clone2)
desc3 = NodeStats(clone3)
#print(desc3)
assert desc3.total == 17, desc3
self.assert_count(desc3, _RMemo, 17)
self.assert_count(desc3, Delayed, 2)
clone4 = LeftMemoize()(clone2)
desc4 = NodeStats(clone4)
#print(desc4)
assert desc4.total == 17, desc4
self.assert_count(desc4, _LMemo, 20)
# left memo duplicates delayed
self.assert_count(desc4, Delayed, 3)
clone5 = AutoMemoize(left=LMemo, right=RMemo)(clone2)
desc5 = NodeStats(clone5)
#print(desc5)
assert desc5.total == 17, desc5
self.assert_count(desc5, _RMemo, 5)
self.assert_count(desc5, _LMemo, 15)
# left memo duplicates delayed
self.assert_count(desc5, Delayed, 3)
try:
clone3.config.clear()
clone3.parse_string('1join()')
assert False, 'Expected error'
except MemoException as error:
assert 'Left recursion was detected' in str(error), str(error)
clone4.config.clear()
clone4.parse_string('1join()')
clone5.config.clear()
clone5.parse_string('1join()')
def test_describe(self):
'''
Use a description of the graph to check against changes.
'''
#basicConfig(level=DEBUG)
name = Word(Letter()) > 'name'
expression = Delayed()
variable = Delayed()
function = (expression / '()') > 'function'
expression += (variable | function) > 'expression'
variable += (name | expression / '.' / name)
dotted_name = function & Eos()
desc0 = NodeStats(dotted_name)
#print(desc0)
assert desc0.total == 18, desc0
self.assert_count(desc0, And, 5)
self.assert_count(desc0, Or, 2)
self.assert_count(desc0, Delayed, 2)
clone1 = Flatten()(dotted_name)
desc1 = NodeStats(clone1)
#print(desc1)
# flattened two matchers - one each of And and Or
assert desc1.total == 16, desc1
self.assert_count(desc1, And, 4)
self.assert_count(desc1, Or, 1)
self.assert_count(desc1, Delayed, 2)
self.assert_count(desc1, Transform, 7)
self.assert_count(desc1, TransformationWrapper, 7)
clone2 = ComposeTransforms()(clone1)
desc2 = NodeStats(clone2)
#print(desc2)
# compressed two transforms
assert desc2.total == 16, desc2
self.assert_count(desc2, And, 4)
self.assert_count(desc2, Or, 1)
self.assert_count(desc2, Delayed, 2)
self.assert_count(desc2, Transform, 5)
self.assert_count(desc2, TransformationWrapper, 5)
clone3 = Memoize(RMemo)(clone2)
desc3 = NodeStats(clone3)
#print(desc3)
assert desc3.total == 16, desc3
self.assert_count(desc3, _RMemo, 14)
self.assert_count(desc3, Delayed, 2)
clone4 = Memoize(LMemo)(clone2)
desc4 = NodeStats(clone4)
#print(desc4)
assert desc4.total == 16, desc4
self.assert_count(desc4, _LMemo, 14)
self.assert_count(desc4, Delayed, 2)
clone5 = AutoMemoize(left=LMemo, right=RMemo)(clone2)
desc5 = NodeStats(clone5)
#print(desc5)
assert desc5.total == 16, desc5
self.assert_count(desc5, _RMemo, 9)
self.assert_count(desc5, _LMemo, 5)
self.assert_count(desc5, Delayed, 2)
try:
clone3.config.clear()
clone3.parse_string('1join()')
assert False, 'Expected error'
except MemoException as error:
assert 'Left recursion with RMemo?' in str(error), str(error)
clone4.config.clear()
clone4.parse_string('1join()')
clone5.config.clear()
clone5.parse_string('1join()')