def test_multi_processor_does_not_accept_post_and_pre_order_mix_in_one_slot():
ParseTreeMultiProcessor(ParseTreeProcessor(pre_order=False),
ParseTreeProcessor(pre_order=False))
ParseTreeMultiProcessor(ParseTreeProcessor(pre_order=True),
ParseTreeProcessor(pre_order=True))
with pytest.raises(RuntimeError):
ParseTreeMultiProcessor(ParseTreeProcessor(pre_order=True),
ParseTreeProcessor(pre_order=False))
with pytest.raises(RuntimeError):
ParseTreeMultiProcessor(ParseTreeProcessor(pre_order=False),
ParseTreeProcessor(pre_order=True))
def compile(input_str):
# TODO Add tests for this one
"""
This also demonstrates how to express basic processor using functions rather than classes.
The function is used as process_unrecognised(node).
It must include "_pre_" in name in order to be registered as pre_order processor.
"""
def contacts_pre_processor(node):
# Initialise state in the root
if node.x.is_root:
node.x.foreign_lookups = {}
node.mark_operands(foreign_lookups=node.x.foreign_lookups)
if node.is_leaf and 'contacts.' in node.a:
node.x.foreign_lookups[node.a] = node.b
return node
def contacts_post_processor(node):
if node.x.is_root:
# Do the lookups
for k, v in node.x.foreign_lookups.items():
node.x.foreign_lookups[k] = '<looked-up-value-of-{}>'.format(v)
# Dummy replacement
if node.is_leaf and node.a in node.x.foreign_lookups:
node.b = node.x.foreign_lookups[node.a]
return node
parse_tree = BetterFiltersG.simple_parse(input_str)
return (ParseTreeMultiProcessor().slot(contacts_pre_processor).slot(
ParseTreeInspector()).slot(contacts_post_processor).slot(
ParseTreeInspector())).process(parse_tree)
def test_multi_processor_application_order():
parse_tree = ('and', ('eq', 'a', 'b'), ('not', ('c')))
calls = []
class P1(ParseTreeProcessor):
def process_unrecognised(self, node):
calls.append(1)
return node
class P2(ParseTreeProcessor):
def process_unrecognised(self, node):
calls.append(2)
return node
class P3(ParseTreeProcessor):
def process_unrecognised(self, node):
calls.append(3)
return node
# Sequential processing with 2 slots
pt1 = copy.deepcopy(parse_tree)
ParseTreeMultiProcessor(P1()).slot(P2()).process(pt1)
assert calls == [1, 1, 1, 2, 2, 2]
calls[:] = []
# Parallel processing with 1 slot
pt2 = copy.deepcopy(parse_tree)
ParseTreeMultiProcessor(P1(), P2()).process(pt2)
assert calls == [1, 2, 1, 2, 1, 2]
calls[:] = []
# Parallel processing with 2 slots
pt2 = copy.deepcopy(parse_tree)
ParseTreeMultiProcessor(P1(), P2()).slot(P3()).process(pt2)
assert calls == [1, 2, 1, 2, 1, 2, 3, 3, 3]
def test_multi_processor_supports_pre_order(parse_tree):
class DepthMarker(ParseTreeProcessor):
pre_order = True
def process_unrecognised(self, node):
node.x.depth = node.x.depth or 0
node.mark_operands(depth=node.x.depth + 1)
return node
proc = ParseTreeMultiProcessor(DepthMarker())
pt = proc.process(('eq', 'a', 'b'))
assert pt.x.depth == 0
pt = proc.process(('and', ('eq', 'a', 'b'), ('ne', 'c', 'd')))
assert pt.x.depth == 0
assert pt.a.x.depth == 1
assert pt.b.x.depth == 1
pt = proc.process(parse_tree)
assert pt.a.b.a.to_tuple() == ('eq', 'type', 'special')
assert pt.a.b.a.x.depth == 3
def test_parse_tree_multiprocessor_calls_all_processors(parse_tree):
class PrimitivesProcessor(ParseTreeProcessor):
def process_primitive(self, primitive):
return '{}!'.format(primitive)
class OperatorsProcessor(ParseTreeProcessor):
def process_unrecognised(self, parse_tree):
parse_tree.op = '<{}>'.format(parse_tree.op)
return parse_tree
class PostProcessor(ParseTreeProcessor):
def process_primitive(self, primitive):
return '{}?'.format(primitive)
proc = ParseTreeMultiProcessor(PrimitivesProcessor(), OperatorsProcessor(),
PostProcessor())
pt = proc.process(parse_tree)
t = pt.to_tuple()
assert t[0] == '<and>'
assert t[1] == ('<or>', ('<in>', 'off!?', 'tags!?'),
('<not>', ('<eq>', 'type!?', 'special!?')))
assert t[2] == ('<gt>', 'weight!?', '100!?')
def test_strict_multi_processor_raises_exception_if_none_of_same_slot_processors_recognises_a_node(
):
class LogicalOperatorProcessor(ParseTreeProcessor):
@ParseTreeProcessor.delegate_of('process_and', 'process_or',
'process_not')
def process_logical_ops(self, node):
return node
class ComparisonOperatorProcessor(ParseTreeProcessor):
@ParseTreeProcessor.delegate_of('process_eq', 'process_ne')
def process_comparison_ops(self, node):
return node
strict_proc = ParseTreeMultiProcessor().strict_slot(
LogicalOperatorProcessor(), ComparisonOperatorProcessor())
strict_proc.process(('or', ('ne', 'a', 'b'), ('eq', 'c', 'd')))
with pytest.raises(PtNodeNotRecognised) as excinfo:
strict_proc.process(('or', ('ne', 'a', 'b'), ('=', 'c', 'd')))
exc = excinfo.value
assert exc.node.to_tuple() == ('=', 'c', 'd')
non_strict_proc = ParseTreeMultiProcessor().slot(
LogicalOperatorProcessor(), ComparisonOperatorProcessor())
non_strict_proc.process(('or', ('ne', 'a', 'b'), ('eq', 'c', 'd')))
non_strict_proc.process(('or', ('ne', 'a', 'b'), ('=', 'c', 'd')))
def test_multi_processor_marks_is_root(parse_tree):
pt = ParseTreeMultiProcessor(ParseTreeProcessor(),
ParseTreeProcessor()).process(parse_tree)
assert pt.x.is_root
assert not pt.a.x.is_root
assert not pt.b.x.is_root
def test_multi_processor_with_no_processors_does_nothing(parse_tree):
parse_tree_copy = copy.deepcopy(parse_tree)
pt = ParseTreeMultiProcessor().process(parse_tree)
assert pt.to_tuple() == parse_tree_copy
def __init__(self):
self._grammar = BetterFiltersG
self._proc = (ParseTreeMultiProcessor().slot(
EsDslFilterPreprocessor()).strict_slot(EsDslFilterGenerator()))