Beispiel #1
0
    def parse(cls, ctx, tokens, _breakstack):
        """
    Consume a parenthetical group of arguments from `tokens` and return the
    parse subtree rooted at this group.  `argstack` contains a stack of all
    early break conditions that are currently "opened".
    """

        assert tokens[0].type == lex.TokenType.LEFT_PAREN
        tree = cls()
        lparen = TreeNode(NodeType.LPAREN)
        lparen.children.append(tokens.pop(0))
        tree.children.append(lparen)

        with ctx.pusharg(tree):
            subtree = ConditionalGroupNode.parse(ctx, tokens, [ParenBreaker()])
        tree.children.append(subtree)

        if tokens[0].type != lex.TokenType.RIGHT_PAREN:
            raise ValueError(
                "Unexpected {} token at {}, expecting r-paren, got {}".format(
                    tokens[0].type.name, tokens[0].get_location(),
                    tokens[0].content))
        rparen = TreeNode(NodeType.RPAREN)
        rparen.children.append(tokens.pop(0))
        tree.children.append(rparen)

        # NOTE(josh): parenthetical groups can have trailing comments because
        # they have closing punctuation
        CommentNode.consume_trailing(ctx, tokens, tree)

        return tree
Beispiel #2
0
def parse_foreach_in(ctx, tokens, breakstack):
    """
  ::

    foreach(loop_var IN [LISTS [<lists>]] [ITEMS [<items>]])
  """
    tree = ArgGroupNode()

    # If it is a whitespace token then put it directly in the parse tree at
    # the current depth
    while tokens and tokens[0].type in WHITESPACE_TOKENS:
        tree.children.append(tokens.pop(0))
        continue

    # Consume the loop variable and any attached comments
    ntokens = len(tokens)
    pargs = PositionalGroupNode.parse(ctx, tokens, 2, ["IN"], breakstack)
    assert len(tokens) < ntokens
    tree.children.append(pargs)

    kwargs = {"LISTS": PositionalParser("+"), "ITEMS": PositionalParser("+")}
    sub_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()))]

    while tokens:
        # Break if the next token belongs to a parent parser, i.e. if it
        # matches a keyword argument of something higher in the stack, or if
        # it closes a parent group.
        if should_break(tokens[0], breakstack):
            break

        # If it is a whitespace token then put it directly in the parse tree at
        # the current depth
        if tokens[0].type in WHITESPACE_TOKENS:
            tree.children.append(tokens.pop(0))
            continue

        # If it's a comment, then add it at the current depth
        if tokens[0].type in (lex.TokenType.COMMENT,
                              lex.TokenType.BRACKET_COMMENT):
            if not get_tag(tokens[0]) in ("sort", "sortable"):
                child = TreeNode(NodeType.COMMENT)
                tree.children.append(child)
                child.children.append(tokens.pop(0))
                continue

        ntokens = len(tokens)
        word = get_normalized_kwarg(tokens[0])
        subparser = kwargs.get(word, None)
        if subparser is not None:
            subtree = KeywordGroupNode.parse(ctx, tokens, word, subparser,
                                             sub_breakstack)
        else:
            logger.warning("Unexpected positional argument at %s",
                           tokens[0].location)
            subtree = PositionalGroupNode.parse(ctx, tokens, '+', [],
                                                sub_breakstack)

        assert len(tokens) < ntokens
        tree.children.append(subtree)
    return tree
    def parse(cls, ctx, tokens, flags, breakstack):
        """
    Parse a continuous sequence of flags
    """

        tree = cls()
        tree.spec = PositionalSpec('+', False, [], flags)
        while tokens:
            # Break if the next token belongs to a parent parser, i.e. if it
            # matches a keyword argument of something higher in the stack, or if
            # it closes a parent group.
            if should_break(tokens[0], breakstack):
                break

            # If it is a whitespace token then put it directly in the parse tree at
            # the current depth
            if tokens[0].type in WHITESPACE_TOKENS:
                tree.children.append(tokens.pop(0))
                continue

            # Break if the next token is not a known flag
            if tokens[0].spelling.upper() not in flags:
                break

            # Otherwise is it is a flag, so add it to the tree as such
            child = TreeNode(NodeType.FLAG)
            child.children.append(tokens.pop(0))
            CommentNode.consume_trailing(ctx, tokens, child)
            tree.children.append(child)

        return tree
Beispiel #4
0
def parse_add_library_object(ctx, tokens, breakstack):
    """
  ::
    add_library(<name> OBJECT <src>...)

    :see: https://cmake.org/cmake/help/latest/command/add_library.html#object-libraries
  """
    tree = ArgGroupNode()

    # If it is a whitespace token then put it directly in the parse tree at
    # the current depth
    while tokens and tokens[0].type in WHITESPACE_TOKENS:
        tree.children.append(tokens.pop(0))
        continue

    ntokens = len(tokens)
    subtree = PositionalGroupNode.parse(ctx, tokens, 2, ["OBJECT"], breakstack)
    assert len(tokens) < ntokens
    tree.children.append(subtree)

    while tokens:
        # Break if the next token belongs to a parent parser, i.e. if it
        # matches a keyword argument of something higher in the stack, or if
        # it closes a parent group.
        if should_break(tokens[0], breakstack):
            break

        # If it is a whitespace token then put it directly in the parse tree at
        # the current depth
        if tokens[0].type in WHITESPACE_TOKENS:
            tree.children.append(tokens.pop(0))
            continue

        # If it's a comment, then add it at the current depth
        if tokens[0].type in (TokenType.COMMENT, TokenType.BRACKET_COMMENT):
            if not get_tag(tokens[0]) in ("sort", "sortable"):
                child = TreeNode(NodeType.COMMENT)
                tree.children.append(child)
                child.children.append(tokens.pop(0))
                continue

        ntokens = len(tokens)
        subtree = PositionalGroupNode.parse(ctx, tokens, '+', [], breakstack)
        assert len(tokens) < ntokens
        tree.children.append(subtree)
    return tree
Beispiel #5
0
def parse_empty(ctx, tokens, breakstack):
    """
  ::

    break()
    continue()
    enable_testing()

  :see: https://cmake.org/cmake/help/latest/command/break.html
  :see: https://cmake.org/cmake/help/latest/command/continue.html
  :see: https://cmake.org/cmake/help/latest/command/enable_testing.html
  :see: https://cmake.org/cmake/help/latest/command/return.html
  """
    tree = ArgGroupNode()

    while tokens:
        # Break if the next token belongs to a parent parser, i.e. if it
        # matches a keyword argument of something higher in the stack, or if
        # it closes a parent group.
        if should_break(tokens[0], breakstack):
            break

        # If it is a whitespace token then put it directly in the parse tree at
        # the current depth
        if tokens[0].type in WHITESPACE_TOKENS:
            tree.children.append(tokens.pop(0))
            continue

        # If it's a comment, then add it at the current depth
        if tokens[0].type in (lex.TokenType.COMMENT,
                              lex.TokenType.BRACKET_COMMENT):
            child = TreeNode(NodeType.COMMENT)
            tree.children.append(child)
            child.children.append(tokens.pop(0))
            continue

        ctx.lint_ctx.record_lint("E1121", location=tokens[0].get_location())
        ntokens = len(tokens)
        subtree = PositionalGroupNode.parse(ctx, tokens, "+", [], breakstack)
        assert len(tokens) < ntokens
        tree.children.append(subtree)
    return tree
    def parse(cls, ctx, tokens, npargs, ntup, flags, breakstack):
        """Parse a continuous sequence of `npargs` positional argument pairs.
    If npargs is an integer we will consume exactly that many arguments.
    If it is not an integer then it is a string meaning:

    * "?": zero or one
    * "*": zero or more
    * "+": one or more
    """

        tree = cls()
        tree.spec = PositionalSpec(ntup, flags=flags, legacy=True)
        subtree = None
        active_depth = tree

        npargs_consumed = 0
        ntup_consumed = 0

        while tokens:
            # Break if we have consumed enough positional arguments
            if pargs_are_full(npargs, npargs_consumed):
                break

            # Break if the next token belongs to a parent parser, i.e. if it
            # matches a keyword argument of something higher in the stack, or if
            # it closes a parent group.
            if should_break(tokens[0], breakstack):
                break

            # Otherwise we will consume the token
            token = tokens.pop(0)

            # If it is a whitespace token then put it directly in the parse tree at
            # the current depth
            if token.type in WHITESPACE_TOKENS:
                active_depth.children.append(token)
                continue

            # If it's a comment token not associated with an argument, then put it
            # directly into the parse tree at the current depth
            if token.type in (lex.TokenType.COMMENT,
                              lex.TokenType.BRACKET_COMMENT):
                child = CommentNode()
                tree.children.append(child)
                child.children.append(token)
                continue

            # If it's a sentinel comment, then add it at the current depth
            if tokens[0].type in (lex.TokenType.FORMAT_OFF,
                                  lex.TokenType.FORMAT_ON):
                tree.children.append(OnOffNode.consume(ctx, tokens))
                continue

            if subtree is None:
                subtree = PositionalGroupNode()
                subtree.spec = PositionalSpec(2, False, [], flags)
                tree.children.append(subtree)
                ntup_consumed = 0

            # Otherwise is it is a positional argument, so add it to the tree as such
            if get_normalized_kwarg(token) in flags:
                child = TreeNode(NodeType.FLAG)
            else:
                child = TreeNode(NodeType.ARGUMENT)

            child.children.append(token)
            CommentNode.consume_trailing(ctx, tokens, child)
            subtree.children.append(child)
            ntup_consumed += 1

            if ntup_consumed >= ntup:
                npargs_consumed += 1
                subtree = None

        return tree
Beispiel #7
0
def parse_add_library_standard(ctx, tokens, breakstack, sortable):
    """
  ::

    add_library(<name> [STATIC | SHARED | MODULE]
               [EXCLUDE_FROM_ALL]
               source1 [source2 ...])

  :see: https://cmake.org/cmake/help/v3.0/command/add_library.html
  """
    # pylint: disable=too-many-statements

    parsing_name = 1
    parsing_type = 2
    parsing_flag = 3
    parsing_sources = 4

    tree = ArgGroupNode()

    # If it is a whitespace token then put it directly in the parse tree at
    # the current depth
    while tokens and tokens[0].type in WHITESPACE_TOKENS:
        tree.children.append(tokens.pop(0))
        continue

    state_ = parsing_name
    parg_group = None
    src_group = None
    active_depth = tree

    flags = ("STATIC", "SHARED", "MODULE", "OBJECT")

    while tokens:
        # This parse function breakson the first right paren, since parenthetical
        # groups are not allowed. A parenthesis might exist in a filename, but
        # if so that filename should be quoted so it wont show up as a RIGHT_PAREN
        # token.
        if tokens[0].type is TokenType.RIGHT_PAREN:
            break

        # If it is a whitespace token then put it directly in the parse tree at
        # the current depth
        if tokens[0].type in WHITESPACE_TOKENS:
            active_depth.children.append(tokens.pop(0))
            continue

        # If it's a comment token not associated with an argument, then put it
        # directly into the parse tree at the current depth
        if tokens[0].type in (TokenType.COMMENT, TokenType.BRACKET_COMMENT):
            if state_ > parsing_name:
                if get_tag(tokens[0]) in ("unsort", "unsortable"):
                    sortable = False
                elif get_tag(tokens[0]) in ("sort", "sortable"):
                    sortable = True
            child = TreeNode(NodeType.COMMENT)
            active_depth.children.append(child)
            child.children.append(tokens.pop(0))
            continue

        if state_ is parsing_name:
            token = tokens.pop(0)
            parg_group = PositionalGroupNode()
            parg_group.spec = PositionalSpec("+")
            active_depth = parg_group
            tree.children.append(parg_group)
            child = TreeNode(NodeType.ARGUMENT)
            child.children.append(token)
            CommentNode.consume_trailing(ctx, tokens, child)
            parg_group.children.append(child)
            state_ += 1
        elif state_ is parsing_type:
            if get_normalized_kwarg(tokens[0]) in flags:
                token = tokens.pop(0)
                child = TreeNode(NodeType.FLAG)
                child.children.append(token)
                CommentNode.consume_trailing(ctx, tokens, child)
                parg_group.children.append(child)
            state_ += 1
        elif state_ is parsing_flag:
            if get_normalized_kwarg(tokens[0]) == "EXCLUDE_FROM_ALL":
                token = tokens.pop(0)
                child = TreeNode(NodeType.FLAG)
                child.children.append(token)
                CommentNode.consume_trailing(ctx, tokens, child)
                parg_group.children.append(child)
            state_ += 1
            src_group = PositionalGroupNode(sortable=sortable,
                                            tags=["file-list"])
            src_group.spec = PositionalSpec("+")
            active_depth = src_group
            tree.children.append(src_group)
        elif state_ is parsing_sources:
            token = tokens.pop(0)
            child = TreeNode(NodeType.ARGUMENT)
            child.children.append(token)
            CommentNode.consume_trailing(ctx, tokens, child)
            src_group.children.append(child)

            if only_comments_and_whitespace_remain(tokens, breakstack):
                active_depth = tree

    return tree
Beispiel #8
0
def parse_add_custom_target(ctx, tokens, breakstack):
    """
  ::
    add_custom_target(Name [ALL] [command1 [args1...]]
                      [COMMAND command2 [args2...] ...]
                      [DEPENDS depend depend depend ... ]
                      [BYPRODUCTS [files...]]
                      [WORKING_DIRECTORY dir]
                      [COMMENT comment]
                      [JOB_POOL job_pool]
                      [VERBATIM] [USES_TERMINAL]
                      [COMMAND_EXPAND_LISTS]
                      [SOURCES src1 [src2...]])

  :see: https://cmake.org/cmake/help/latest/command/add_custom_target.html
  """
    kwargs = {
        "BYPRODUCTS": PositionalParser("+"),
        "COMMAND": ShellCommandNode.parse,
        "COMMENT": PositionalParser(1),
        "DEPENDS": PositionalParser("+"),
        "JOB_POOL": PositionalParser(1),
        "SOURCES": PositionalParser("+"),
        "WORKING_DIRECTORY": PositionalParser(1),
    }

    required_kwargs = {
        # Required by convention
        "COMMENT": "C0113"
    }

    flags = ("VERBATIM", "USES_TERMINAL", "COMMAND_EXPAND_LISTS")
    tree = ArgGroupNode()

    # If it is a whitespace token then put it directly in the parse tree at
    # the current depth
    while tokens and tokens[0].type in WHITESPACE_TOKENS:
        tree.children.append(tokens.pop(0))
        continue

    breaker = KwargBreaker(list(kwargs.keys()) + list(flags))
    child_breakstack = breakstack + [breaker]

    nametree = None
    state = "name"

    while tokens:
        # Break if the next token belongs to a parent parser, i.e. if it
        # matches a keyword argument of something higher in the stack, or if
        # it closes a parent group.
        if should_break(tokens[0], breakstack):
            break

        # If it is a whitespace token then put it directly in the parse tree at
        # the current depth
        if tokens[0].type in WHITESPACE_TOKENS:
            tree.children.append(tokens.pop(0))
            continue

        # If it's a comment, then add it at the current depth
        if tokens[0].type in (TokenType.COMMENT, TokenType.BRACKET_COMMENT):
            child = TreeNode(NodeType.COMMENT)
            tree.children.append(child)
            child.children.append(tokens.pop(0))
            continue

        ntokens = len(tokens)
        if state == "name":
            next_semantic = get_first_semantic_token(tokens[1:])
            if (next_semantic is not None
                    and get_normalized_kwarg(next_semantic) == "ALL"):
                npargs = 2
            else:
                npargs = 1

            nametree = PositionalGroupNode.parse(ctx, tokens, npargs, ["ALL"],
                                                 child_breakstack)
            assert len(tokens) < ntokens
            tree.children.append(nametree)
            state = "first-command"
            continue

        word = get_normalized_kwarg(tokens[0])
        if state == "first-command":
            if not (word in kwargs or word in flags):
                subtree = PositionalGroupNode.parse(ctx, tokens, '+', [],
                                                    child_breakstack)
                tree.children.append(subtree)
                assert len(tokens) < ntokens
            state = "kwargs"
            continue

        if word in flags:
            subtree = FlagGroupNode.parse(
                ctx, tokens, flags,
                breakstack + [KwargBreaker(list(kwargs.keys()))])
            assert len(tokens) < ntokens
            tree.children.append(subtree)
            continue

        if word in kwargs:
            required_kwargs.pop(word, None)
            subtree = KeywordGroupNode.parse(ctx, tokens, word, kwargs[word],
                                             child_breakstack)
            assert len(tokens) < ntokens
            tree.children.append(subtree)
            continue

        ctx.lint_ctx.record_lint("E1122", location=tokens[0].get_location())
        # logger.warning(
        #     "Unexpected positional argument %s at %s",
        #     tokens[0].spelling, tokens[0].location())
        subtree = PositionalGroupNode.parse(ctx, tokens, '+', [],
                                            child_breakstack)
        assert len(tokens) < ntokens
        tree.children.append(subtree)
        continue

    if required_kwargs:
        location = ()
        for token in tree.get_semantic_tokens():
            location = token.get_location()
            break

        missing_kwargs = sorted(
            (lintid, word) for word, lintid in sorted(required_kwargs.items()))
        for lintid, word in missing_kwargs:
            ctx.lint_ctx.record_lint(lintid, word, location=location)
    return tree
Beispiel #9
0
    def parse2(cls, ctx, tokens, spec, breakstack):
        """
    Parse a continuous sequence of `npargs` positional arguments. If npargs is
    an integer we will consume exactly that many arguments. If it is not an
    integer then it is a string meaning:

    * "?": zero or one
    * "*": zero or more
    * "+": one or more
    """

        tree = cls(sortable=spec.sortable, tags=spec.tags)
        tree.spec = spec
        nconsumed = 0

        # Strip off any preceeding whitespace (note that in most cases this has
        # already been done but in some cases (such ask kwarg subparser) where
        # it hasn't
        while tokens and tokens[0].type in WHITESPACE_TOKENS:
            tree.children.append(tokens.pop(0))

        # If the first non-whitespace token is a cmake-format tag annotating
        # sortability, then parse it out here and record the annotation
        if tokens and get_tag(tokens[0]) in ("sortable", "sort"):
            tree.sortable = True
        elif tokens and get_tag(tokens[0]) in ("unsortable", "unsort"):
            tree.sortable = False

        while tokens:

            # Break if we have consumed   enough positional arguments
            if pargs_are_full(spec.nargs, nconsumed):
                break

            # Break if the next token belongs to a parent parser, i.e. if it
            # matches a keyword argument of something higher in the stack, or if
            # it closes a parent group.
            if should_break(tokens[0], breakstack):
                # NOTE(josh): if spec.nargs is an exact number of arguments, then we
                # shouldn't break on kwarg match from a parent parser. Instead, we
                # should consume the token. This is a hack to deal with
                # ```install(RUNTIME COMPONENT runtime)``. In this case the second
                # occurance of "runtime" should not match the ``RUNTIME`` keyword
                # and should not break the positional parser.
                # TODO(josh): this is kind of hacky because it will force the positional
                # parser to consume a right parenthesis and will lead to parse errors
                # in the event of a missing positional argument. Such errors will be
                # difficult to debug for the user.
                if not npargs_is_exact(spec.nargs):
                    break

                if tokens[0].type == lex.TokenType.RIGHT_PAREN:
                    break

            # If this is the start of a parenthetical group, then parse the group
            # NOTE(josh): syntatically this probably shouldn't be allowed here, but
            # cmake seems to accept it so we probably should too.
            if tokens[0].type == lex.TokenType.LEFT_PAREN:
                with ctx.pusharg(tree):
                    subtree = ParenGroupNode.parse(ctx, tokens, breakstack)
                tree.children.append(subtree)
                continue

            # If it is a whitespace token then put it directly in the parse tree at
            # the current depth
            if tokens[0].type in WHITESPACE_TOKENS:
                tree.children.append(tokens.pop(0))
                continue

            # If it's a comment token not associated with an argument, then put it
            # directly into the parse tree at the current depth
            if tokens[0].type in (lex.TokenType.COMMENT,
                                  lex.TokenType.BRACKET_COMMENT):
                if comment_belongs_up_tree(ctx, tokens, tree, breakstack):
                    break

                child = CommentNode.consume(ctx, tokens)
                tree.children.append(child)
                continue

            # If it's a sentinel comment, then add it at the current depth
            if tokens[0].type in (lex.TokenType.FORMAT_OFF,
                                  lex.TokenType.FORMAT_ON):
                tree.children.append(OnOffNode.consume(ctx, tokens))
                continue

            # Otherwise is it is a positional argument, so add it to the tree as such
            if get_normalized_kwarg(tokens[0]) in spec.flags:
                child = TreeNode(NodeType.FLAG)
            else:
                child = TreeNode(NodeType.ARGUMENT)

            child.children.append(tokens.pop(0))
            CommentNode.consume_trailing(ctx, tokens, child)
            tree.children.append(child)
            nconsumed += 1

        return tree
Beispiel #10
0
    def consume(cls, ctx, tokens):
        """
    Consume a complete statement, removing tokens from the input list and
    returning a STATEMENT node.
    """
        node = cls()

        # Consume the function name
        fnname = tokens[0].spelling.lower()
        node.funnode = funnode = FunctionNameNode.parse(ctx, tokens)
        node.children.append(funnode)

        # Consume whitespace up to the parenthesis
        while tokens and tokens[0].type in WHITESPACE_TOKENS:
            node.children.append(tokens.pop(0))

        # TODO(josh): should the parens belong to the statement node or the
        # group node?
        if tokens[0].type != lex.TokenType.LEFT_PAREN:
            raise ValueError(
                "Unexpected {} token at {}, expecting l-paren, got {}".format(
                    tokens[0].type.name, tokens[0].get_location(),
                    repr(tokens[0].content)))

        lparen = TreeNode(NodeType.LPAREN)
        lparen.children.append(tokens.pop(0))
        node.children.append(lparen)

        while tokens and tokens[0].type in WHITESPACE_TOKENS:
            node.children.append(tokens.pop(0))
            continue

        breakstack = [ParenBreaker()]

        parse_fun = ctx.parse_db.get(fnname, None)
        if parse_fun is None:
            # If the parse_db provides a "_default" then use that. Otherwise use the
            # standard parser with no kwargs or flags.
            parse_fun = ctx.parse_db.get("_default", StandardParser())

        if isinstance(parse_fun, StandardParser2):
            node.cmdspec = parse_fun.cmdspec

        node.argtree = subtree = parse_fun(ctx, tokens, breakstack)
        node.children.append(subtree)

        # NOTE(josh): technically we may have a statement specification with
        # an exact number of arguments. At this point we have broken out of that
        # statement but we might have some comments or whitespace to consume
        while tokens and tokens[0].type != lex.TokenType.RIGHT_PAREN:
            if tokens[0].type in WHITESPACE_TOKENS:
                node.children.append(tokens.pop(0))
                continue

            if tokens[0].type in COMMENT_TOKENS:
                cnode = CommentNode.consume(ctx, tokens)
                node.children.append(cnode)
                continue

            raise UserError(
                "Unexpected {} token at {}, expecting r-paren, got {}".format(
                    tokens[0].type.name, tokens[0].get_location(),
                    repr(tokens[0].content)))

        if not tokens:
            raise UserError(
                "Unexpected end of token stream while parsing statement:\n {}".
                format(tree_string([node])))

        if tokens[0].type != lex.TokenType.RIGHT_PAREN:
            raise UserError(
                "Unexpected {} token at {}, expecting r-paren, got {}".format(
                    tokens[0].type.name, tokens[0].get_location(),
                    repr(tokens[0].content)))

        rparen = TreeNode(NodeType.RPAREN)
        rparen.children.append(tokens.pop(0))
        node.children.append(rparen)
        CommentNode.consume_trailing(ctx, tokens, node)

        return node
Beispiel #11
0
def parse_set(ctx, tokens, breakstack):
  """
  ::

    set(<variable> <value>
        [[CACHE <type> <docstring> [FORCE]] | PARENT_SCOPE])

  :see: https://cmake.org/cmake/help/v3.0/command/set.html?
  """
  tree = SetFnNode()

  # If it is a whitespace token then put it directly in the parse tree at
  # the current depth
  while tokens and tokens[0].type in WHITESPACE_TOKENS:
    tree.children.append(tokens.pop(0))
    continue

  kwargs = {
      "CACHE": PositionalParser('2+', flags=["FORCE"])
  }
  flags = ["PARENT_SCOPE"]
  kwarg_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()) + flags)]
  positional_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()))]

  ntokens = len(tokens)
  subtree = PositionalGroupNode.parse(
      ctx, tokens, 1, flags, positional_breakstack)
  assert len(tokens) < ntokens
  tree.children.append(subtree)
  for token in subtree.get_semantic_tokens():
    tree.varname = token
    break

  while tokens:
    # Break if the next token belongs to a parent parser, i.e. if it
    # matches a keyword argument of something higher in the stack, or if
    # it closes a parent group.
    if should_break(tokens[0], breakstack):
      break

    # If it is a whitespace token then put it directly in the parse tree at
    # the current depth
    if tokens[0].type in WHITESPACE_TOKENS:
      tree.children.append(tokens.pop(0))
      continue

    # If it's a comment, then add it at the current depth
    if tokens[0].type in (lex.TokenType.COMMENT,
                          lex.TokenType.BRACKET_COMMENT):
      if not get_tag(tokens[0]) in ("sort", "sortable"):
        child = TreeNode(NodeType.COMMENT)
        tree.children.append(child)
        child.children.append(tokens.pop(0))
        continue

    ntokens = len(tokens)

    word = get_normalized_kwarg(tokens[0])
    if word == "CACHE":
      with ctx.pusharg(tree):
        subtree = KeywordGroupNode.parse(
            ctx, tokens, word, kwargs[word], kwarg_breakstack)
      cache_tokens = subtree.get_semantic_tokens()
      cache_tokens.pop(0)
      cache_args = [None, None, False]
      for idx, token in enumerate(cache_tokens[:2]):
        cache_args[idx] = token.spelling

      if not cache_tokens:
        ctx.lint_ctx.record_lint(
            "E1120", "<variable>", location=tokens[0].get_location())
      elif len(cache_tokens) < 2:
        ctx.lint_ctx.record_lint(
            "E1120", "<value>", location=tokens[0].get_location())
      elif len(cache_tokens) < 3:
        pass
      elif len(cache_tokens) > 4:
        ctx.lint_ctx.record_lint(
            "E1121", location=cache_tokens[2].get_location())
      else:
        if cache_tokens[2].spelling.upper() == "FORCE":
          cache_args[2] = True
        else:
          ctx.lint_ctx.record_lint(
              "E1121", location=cache_tokens[2].get_location())
      tree.cache = CacheTuple(*cache_args)

    elif word == "PARENT_SCOPE":
      with ctx.pusharg(tree):
        subtree = PositionalGroupNode.parse(
            ctx, tokens, '+', ["PARENT_SCOPE"], positional_breakstack)
      tree.parent_scope = True
    else:
      with ctx.pusharg(tree):
        subtree = PositionalGroupNode.parse(
            ctx, tokens, '+', [], kwarg_breakstack)
      for pattern, tags in ctx.config.parse.vartags_:
        if pattern.match(tree.varname.spelling):
          subtree.tags.extend(tags)
      tree.value_group = subtree

    assert len(tokens) < ntokens
    tree.children.append(subtree)
  return tree