def _reduce_nodes(nodes: List[Node], rule: Rule) -> Node:
node = Node(rule.key)
for symbol, child in zip(rule.symbols, nodes):
if symbol != child.key:
raise ParserError(f'Unable to apply rule {rule!r}. Unexpected token: {child!r}')
node.add_child(child)
return node
def _squash(parent: Node, key: str) -> Node:
new_parent = Node(parent.key, token=parent.token)
new_children = [_squash(child, key) for child in parent.children]
if not new_children or parent.key != key:
new_parent.add_children(*new_children)
return new_parent
if (new_child := new_children[0]).key == key and len(new_children) == 1:
return new_child
def tree(tree_def: Tuple[Any, ...]) -> Node:
key, token_or_children = tree_def
if isinstance(token_or_children, Token):
return Node(key, token=token_or_children)
if isinstance(token_or_children, list):
node = Node(key)
node.add_children(*(tree(child) for child in token_or_children))
return node
raise TypeError(f'Unexpected type {type(token_or_children)!r}.')
def test_children_are_copy(self) -> None:
node = Node('node')
child = Node('child', token=Token(0, 0, 'value'))
node.add_child(child)
children = node.children
children.append(Node('no_child_of_node'))
self.assertEqual([child], node.children)
def _skip(parent: Node, key: str) -> Node:
new_parent = Node(parent.key, token=parent.token)
new_children = [_skip(child, key) for child in parent.children]
for new_child in new_children:
if new_child.key != key or not new_child.children:
new_parent.add_child(new_child)
else:
new_parent.add_children(*new_child.children)
return new_parent
def parse(tokens: List[Node], grammar: Grammar, table: ParseTable) -> Node:
from cmaj.parser.table import Action
assert table.num_rows > 0
stack: Stack = []
row = 0
tokens = tokens + [Node(Grammar.AUGMENTED_EOF)]
token_index = 0
while True:
token = tokens[token_index]
action = table.action(row, token.key)
if action is None:
raise ParserError(f'Unexpected token: {tokens[token_index]!r}')
elif action.key == Action.ACCEPT:
break
elif action.key == Action.SHIFT:
stack.append((row, tokens[token_index]))
row = action.index
token_index += 1
elif action.key == Action.GOTO:
row = action.index
elif action.key == Action.REDUCE:
rule_index = action.index
rule = grammar.rule_at(rule_index)
stack, row, nodes = _reduce_stack(stack, rule)
node = _reduce_nodes(nodes, rule)
stack.append((row, node))
action = table.action(row, node.key)
assert action.key == Action.GOTO
row = action.index
else:
raise ParserError(f'Unexpected parser action {action!r} for token: {tokens[token_index]!r}')
if len(stack) != 1:
raise ParserError(f'Found unprocessed tokens: {_to_symbols(stack)!r}')
return stack[0][1]
def prune(parent: Node, *keys: str) -> Node:
new_node = Node(parent.key, token=parent.token)
gen = (prune(child, *keys) for child in parent.children
if child.key not in keys)
new_node.add_children(*(child for child in gen if len(child) > 0))
return new_node
def test_given_different_tokens_then_not_equal(self) -> None:
node = Node('node', token=Token(0, 0, 'value'))
self.assertNotEqual(Node('node', token=Token(0, 0, 'other')), node)
self.assertNotEqual(Node('node', token=Token(0, 1, 'value')), node)
self.assertNotEqual(Node('node', token=Token(1, 0, 'value')), node)
def test_given_node_with_token_when_adding_child_then_error(self) -> None:
node = Node('node', token=Token(0, 0, 'value'))
child = Node('child', token=Token(1, 0, 'value'))
self.assertRaises(AssertionError, node.add_child, child)
def test_when_adding_node_then_node_is_child(self) -> None:
node = Node('node')
child = Node('child', token=Token(0, 0, 'value'))
node.add_child(child)
self.assertIn(child, node.children)
def test_given_token_then_length_of_token(self) -> None:
node = Node('node', Token(3, 2, 'value'))
self.assertEqual(5, len(node))
def test_new_node_is_leaf(self) -> None:
node = Node('node')
self.assertEqual(0, len(node.children))
def test_given_leaf_then_length_is_zero(self) -> None:
node = Node('node')
self.assertEqual(0, len(node))
def test_node_is_not_hashable(self) -> None:
node = Node('key')
self.assertRaises(TypeError, hash, node)
def tokens(keys: str) -> List[Node]:
return [
Node(key, token=Token(0, column, 'x'))
for column, key in enumerate(keys)
]
def test_given_different_keys_then_not_equal(self) -> None:
node = Node('node')
other = Node('other')
self.assertNotEqual(other, node)
def match(self, line_index: int, column_index: int,
sequence: str) -> Optional[Node]:
from cmaj.ast.node import Token
if (result := self._regex(sequence)) is not None:
return Node(self._key,
token=Token(line_index, column_index, result))