def assert_parse_tree(test, nodes, tups, tree=None, history=None):
"""
Check the output tree structure against that of expect_tree: a nested tuple
tree.
"""
if tree is None:
tree = nodes
if history is None:
history = []
for node, tup in overzip(nodes, tups):
if isinstance(node, lexer.Token):
continue
message = ("For node {} at\n {} within \n{}. "
"If this is infact correct, copy-paste this:\n\n{}".format(
node, tree_string([node]), tree_string(tree, history),
test_string(tree, ' ' * 6, ' ' * 2)))
test.assertIsNotNone(node, msg="Missing node " + message)
test.assertIsNotNone(tup, msg="Extra node " + message)
expect_type, expect_children = tup
test.assertEqual(node.node_type,
expect_type,
msg="Expected type={} ".format(expect_type) + message)
assert_parse_tree(test, node.children, expect_children, tree,
history + [node])
def consume(cls, ctx, tokens):
"""
Consume a complete statement, removing tokens from the input list and
returning a STATEMENT node.
"""
node = cls()
# Consume the function name
fnname = tokens[0].spelling.lower()
node.funnode = funnode = FunctionNameNode.parse(ctx, tokens)
node.children.append(funnode)
# Consume whitespace up to the parenthesis
while tokens and tokens[0].type in WHITESPACE_TOKENS:
node.children.append(tokens.pop(0))
# TODO(josh): should the parens belong to the statement node or the
# group node?
if tokens[0].type != lexer.TokenType.LEFT_PAREN:
raise ValueError(
"Unexpected {} token at {}, expecting l-paren, got {}".format(
tokens[0].type.name, tokens[0].get_location(),
repr(tokens[0].content)))
lparen = TreeNode(NodeType.LPAREN)
lparen.children.append(tokens.pop(0))
node.children.append(lparen)
while tokens and tokens[0].type in WHITESPACE_TOKENS:
node.children.append(tokens.pop(0))
continue
breakstack = [ParenBreaker()]
parse_fun = ctx.parse_db.get(fnname, None)
if parse_fun is None:
# If the parse_db provides a "_default" then use that. Otherwise use the
# standard parser with no kwargs or flags.
parse_fun = ctx.parse_db.get("_default", StandardParser())
node.argtree = subtree = parse_fun(ctx, tokens, breakstack)
node.children.append(subtree)
# NOTE(josh): technically we may have a statement specification with
# an exact number of arguments. At this point we have broken out of that
# statement but we might have some comments or whitespace to consume
while tokens and tokens[0].type != lexer.TokenType.RIGHT_PAREN:
if tokens[0].type in WHITESPACE_TOKENS:
node.children.append(tokens.pop(0))
continue
if tokens[0].type in COMMENT_TOKENS:
cnode = CommentNode.consume(ctx, tokens)
node.children.append(cnode)
continue
raise UserError(
"Unexpected {} token at {}, expecting r-paren, got {}".format(
tokens[0].type.name, tokens[0].get_location(),
repr(tokens[0].content)))
if not tokens:
raise UserError(
"Unexpected end of token stream while parsing statement:\n {}".
format(tree_string([node])))
if tokens[0].type != lexer.TokenType.RIGHT_PAREN:
raise UserError(
"Unexpected {} token at {}, expecting r-paren, got {}".format(
tokens[0].type.name, tokens[0].get_location(),
repr(tokens[0].content)))
rparen = TreeNode(NodeType.RPAREN)
rparen.children.append(tokens.pop(0))
node.children.append(rparen)
CommentNode.consume_trailing(ctx, tokens, node)
return node