def parse(cls, ctx, tokens, npargs, flags, breakstack, sortable=False): """ Parse a continuous sequence of `npargs` positional arguments. If npargs is an integer we will consume exactly that many arguments. If it is not an integer then it is a string meaning: * "?": zero or one * "*": zero or more * "+": one or more """ tree = cls(sortable=sortable) tree.spec = PositionalSpec(npargs, flags) nconsumed = 0 # Strip off any preceeding whitespace (note that in most cases this has # already been done but in some cases (such ask kwarg subparser) where # it hasn't while tokens and tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) # If the first non-whitespace token is a cmake-format tag annotating # sortability, then parse it out here and record the annotation if tokens and get_tag(tokens[0]) in ("sortable", "sort"): tree.sortable = True elif tokens and get_tag(tokens[0]) in ("unsortable", "unsort"): tree.sortable = False while tokens: # Break if we have consumed enough positional arguments if pargs_are_full(npargs, nconsumed): break # Break if the next token belongs to a parent parser, i.e. if it # matches a keyword argument of something higher in the stack, or if # it closes a parent group. if should_break(tokens[0], breakstack): # NOTE(josh): if npargs is an exact number of arguments, then we # shouldn't break on kwarg match from a parent parser. Instead, we # should consume the token. This is a hack to deal with # ```install(RUNTIME COMPONENT runtime)``. In this case the second # occurance of "runtime" should not match the ``RUNTIME`` keyword # and should not break the positional parser. # TODO(josh): this is kind of hacky because it will force the positional # parser to consume a right parenthesis and will lead to parse errors # in the event of a missing positional argument. Such errors will be # difficult to debug for the user. if not npargs_is_exact(npargs): break if tokens[0].type == lexer.TokenType.RIGHT_PAREN: break # If this is the start of a parenthetical group, then parse the group # NOTE(josh): syntatically this probably shouldn't be allowed here, but # cmake seems to accept it so we probably should too. if tokens[0].type == lexer.TokenType.LEFT_PAREN: subtree = ParenGroupNode.parse(ctx, tokens, breakstack) tree.children.append(subtree) continue # If it is a whitespace token then put it directly in the parse tree at # the current depth if tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # If it's a comment token not associated with an argument, then put it # directly into the parse tree at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): before = len(tokens) child = CommentNode.consume(ctx, tokens) assert len(tokens) < before, \ "consume_comment didn't consume any tokens" tree.children.append(child) continue # Otherwise is it is a positional argument, so add it to the tree as such if get_normalized_kwarg(tokens[0]) in flags: child = TreeNode(NodeType.FLAG) else: child = TreeNode(NodeType.ARGUMENT) child.children.append(tokens.pop(0)) CommentNode.consume_trailing(ctx, tokens, child) tree.children.append(child) nconsumed += 1 return tree
def parse2(cls, ctx, tokens, cmdspec, kwargs, breakstack): """ Standard parser for the commands in the form of:: command_name(parg1 parg2 parg3... KEYWORD1 kwarg1 kwarg2... KEYWORD2 kwarg3 kwarg4... FLAG1 FLAG2 FLAG3) The parser starts off as a positional parser. If a keyword or flag is encountered the positional parser is popped off the parse stack. If it was a keyword then the keyword parser is pushed on the parse stack. If it was a flag than a new flag parser is pushed onto the stack. """ # NOTE(josh): we will pop things off this list, so let's make a copy pargspecs = list(cmdspec.pargs) tree = cls() tree.cmdspec = cmdspec # If it is a whitespace token then put it directly in the parse tree at # the current depth while tokens and tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # NOTE(josh): if there is only one non-exact legacy specification then we # reuse that specification for any additional positional arguments that we # pick up. This is to maintain the current/legacy behavior of simple # positional argument specifications # TODO(josh): double check the reasoning for this. I think it might be # mistaken and unnecessary default_spec = DEFAULT_PSPEC if (len(pargspecs) == 1 and pargspecs[0].legacy and not npargs_is_exact(pargspecs[0].nargs)): default_spec = pargspecs.pop(0) all_flags = list(default_spec.flags) for pspec in pargspecs: all_flags.extend(pspec.flags) kwarg_breakstack = breakstack + [ KwargBreaker(list(kwargs.keys()) + all_flags) ] while tokens: # If it is a whitespace token then put it directly in the parse tree at # the current depth if tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # If it's a comment, then add it at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): if comment_belongs_up_tree(ctx, tokens, tree, breakstack): break tree.children.append(CommentNode.consume(ctx, tokens)) continue # If it's a sentinel comment, then add it at the current depth if tokens[0].type in (lexer.TokenType.FORMAT_OFF, lexer.TokenType.FORMAT_ON): tree.children.append(OnOffNode.consume(ctx, tokens)) continue # Break if the next token belongs to a parent parser, i.e. if it # matches a keyword argument of something higher in the stack, or if # it closes a parent group. if should_break(tokens[0], breakstack): # NOTE(josh): if spec.nargs is an exact number of arguments, then we # shouldn't break on kwarg match from a parent parser. Instead, we # should consume that many tokens. This is a hack to deal with # ```install(RUNTIME COMPONENT runtime)``. In this case the second # occurance of "runtime" should not match the ``RUNTIME`` keyword # and should not break the positional parser. # TODO(josh): this is kind of hacky because it will force the positional # parser to consume a right parenthesis and will lead to parse errors # in the event of a missing positional argument. Such errors will be # difficult to debug for the user. if pargspecs: pspec = pargspecs[0] else: pspec = default_spec if not npargs_is_exact(pspec.nargs) or pspec.nargs == 0: break ntokens = len(tokens) word = get_normalized_kwarg(tokens[0]) if word in kwargs: with ctx.pusharg(tree): subtree = KeywordGroupNode.parse(ctx, tokens, word, kwargs[word], kwarg_breakstack) tree.kwarg_groups.append(subtree) else: if pargspecs: pspec = pargspecs.pop(0) else: pspec = default_spec other_flags = [] for otherspec in pargspecs: for flag in otherspec.flags: if flag in pspec.flags: continue other_flags.append(flag) positional_breakstack = breakstack + [ KwargBreaker(list(kwargs.keys()) + other_flags) ] with ctx.pusharg(tree): subtree = PositionalGroupNode.parse2( ctx, tokens, pspec, positional_breakstack) tree.parg_groups.append(subtree) if len(tokens) >= ntokens: raise InternalError( "parsed an empty subtree at {}:\n {}\n pspec: {}".format( tokens[0], dump_tree_tostr([tree]), pspec)) tree.children.append(subtree) return tree