def parse_foreach_in(tokens, breakstack): """ :: foreach(loop_var IN [LISTS [<lists>]] [ITEMS [<items>]]) """ tree = TreeNode(NodeType.ARGGROUP) # If it is a whitespace token then put it directly in the parse tree at # the current depth while tokens and tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # Consume the loop variable and any attached comments ntokens = len(tokens) pargs = parse_positionals(tokens, 2, ["IN"], breakstack) assert len(tokens) < ntokens tree.children.append(pargs) kwargs = {"LISTS": PositionalParser("+"), "ITEMS": PositionalParser("+")} sub_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()))] while tokens: # Break if the next token belongs to a parent parser, i.e. if it # matches a keyword argument of something higher in the stack, or if # it closes a parent group. if should_break(tokens[0], breakstack): break # If it is a whitespace token then put it directly in the parse tree at # the current depth if tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # If it's a comment, then add it at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): if not get_tag(tokens[0]) in ("sort", "sortable"): child = TreeNode(NodeType.COMMENT) tree.children.append(child) child.children.append(tokens.pop(0)) continue ntokens = len(tokens) word = get_normalized_kwarg(tokens[0]) subparser = kwargs.get(word, None) if subparser is not None: subtree = parse_kwarg(tokens, word, subparser, sub_breakstack) else: logger.warning("Unexpected positional argument at %s", tokens[0].location) subtree = parse_positionals(tokens, '+', [], sub_breakstack) assert len(tokens) < ntokens tree.children.append(subtree) return tree
def parse_add_custom_command(tokens, breakstack): """ There are two forms of `add_custom_command`. This is the dispatcher between the two forms. """ descriminator_token = get_first_semantic_token(tokens) descriminator_word = get_normalized_kwarg(descriminator_token) if descriminator_word == "TARGET": return parse_add_custom_command_events(tokens, breakstack) if descriminator_word == "OUTPUT": return parse_add_custom_command_standard(tokens, breakstack) logger.warning( "Indeterminate form of add_custom_command at %s", descriminator_token.location) return parse_add_custom_command_standard(tokens, breakstack)
def parse_add_executable_standard(tokens, breakstack, sortable): """ :: add_executable(<name> [WIN32] [MACOSX_BUNDLE] [EXCLUDE_FROM_ALL] [source1] [source2 ...]) :see: https://cmake.org/cmake/help/latest/command/add_executable.html#command:add_executable """ # pylint: disable=too-many-statements parsing_name = 1 parsing_flags = 2 parsing_sources = 3 tree = TreeNode(NodeType.ARGGROUP) # If it is a whitespace token then put it directly in the parse tree at # the current depth while tokens and tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue state_ = parsing_name parg_group = None src_group = None active_depth = tree while tokens: # This parse function breaks on the first right paren, since parenthetical # groups are not allowed. A parenthesis might exist in a filename, but # if so that filename should be quoted so it wont show up as a RIGHT_PAREN # token. if tokens[0].type is lexer.TokenType.RIGHT_PAREN: break # If it is a whitespace token then put it directly in the parse tree at # the current depth if tokens[0].type in WHITESPACE_TOKENS: active_depth.children.append(tokens.pop(0)) continue # If it's a comment token not associated with an argument, then put it # directly into the parse tree at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): if state_ > parsing_name: if get_tag(tokens[0]) in ("unsort", "unsortable"): sortable = False elif get_tag(tokens[0]) in ("unsort", "unsortable"): sortable = True child = TreeNode(NodeType.COMMENT) active_depth.children.append(child) child.children.append(tokens.pop(0)) continue if state_ is parsing_name: token = tokens.pop(0) parg_group = TreeNode(NodeType.PARGGROUP) active_depth = parg_group tree.children.append(parg_group) child = TreeNode(NodeType.ARGUMENT) child.children.append(token) consume_trailing_comment(child, tokens) parg_group.children.append(child) state_ += 1 elif state_ is parsing_flags: if get_normalized_kwarg(tokens[0]) in ( "WIN32", "MACOSX_BUNDLE", "EXCLUDE_FROM_ALL"): token = tokens.pop(0) child = TreeNode(NodeType.FLAG) child.children.append(token) consume_trailing_comment(child, tokens) parg_group.children.append(child) else: state_ += 1 src_group = TreeNode(NodeType.PARGGROUP, sortable=sortable) active_depth = src_group tree.children.append(src_group) elif state_ is parsing_sources: token = tokens.pop(0) child = TreeNode(NodeType.ARGUMENT) child.children.append(token) consume_trailing_comment(child, tokens) src_group.children.append(child) if only_comments_and_whitespace_remain(tokens, breakstack): active_depth = tree return tree
def _reflow(self, config, cursor, passno): """ Compute the size of a keyword argument group which is nominally allocated `linewidth` columns for packing. `linewidth` is only considered for hpacking of consecutive scalar arguments """ # TODO(josh): Much of this code is the same as update_statement_size_, so # dedup the two start_cursor = np.array(cursor) children = list(self.children) assert children child = children.pop(0) assert child.type == NodeType.KEYWORD cursor = child.reflow(config, cursor, passno) keyword = parser.get_normalized_kwarg(child.pnode.children[0]) self._colextent = child.colextent if self._wrap == WrapAlgo.HPACK: if len(children) > config.max_subargs_per_line: self._colextent = config.linewidth + 100 while children: cursor[1] += len(' ') child = children.pop(0) if child.type == NodeType.COMMENT: self._colextent = config.linewidth + 100 cursor = child.reflow(config, cursor, passno) self._colextent = max(self._colextent, child.colextent) return cursor elif self._wrap == WrapAlgo.VPACK: cursor[1] += len(' ') column_cursor = np.array(cursor) elif self._wrap in (WrapAlgo.KWNVPACK, WrapAlgo.PNVPACK): column_cursor = start_cursor + np.array((1, config.tab_size)) else: raise RuntimeError("Unexepected wrap algorithm") # NOTE(josh): this logic is common to both VPACK and NVPACK, the only # difference is the starting position of the column_cursor prev = None cursor = np.array(column_cursor) scalar_seq = analyze_scalar_sequence(children) while children: if (prev is None) or (prev.type not in SCALAR_TYPES): scalar_seq = analyze_scalar_sequence(children) child = children.pop(0) # If both the previous and current nodes are scalar nodes and the two # are not part of a particularly long (by a configurable margin) sequence # of scalar nodes, then advance the cursor horizontally by one space and # try to pack the next child on the same line as the current one if prev is None: cursor = child.reflow(config, cursor, passno) elif (prev.type in SCALAR_TYPES and child.type in SCALAR_TYPES and (scalar_seq.length <= config.max_subargs_per_line or keyword == 'COMMAND' or keyword.startswith('--')) and not scalar_seq.has_comment): cursor[1] += len(' ') # But if the cursor has overflowed the line width allocation, then # we cannot cursor = child.reflow(config, cursor, passno) if child.colextent > config.linewidth: column_cursor[0] += 1 cursor = np.array(column_cursor) cursor = child.reflow(config, cursor, passno) # Otherwise we fall back to vpack else: column_cursor[0] += 1 cursor = np.array(column_cursor) cursor = child.reflow(config, cursor, passno) self._colextent = max(self._colextent, child.colextent) column_cursor[0] = cursor[0] prev = child return cursor
def parse_set(tokens, breakstack): """ :: set(<variable> <value> [[CACHE <type> <docstring> [FORCE]] | PARENT_SCOPE]) :see: https://cmake.org/cmake/help/v3.0/command/set.html? """ tree = TreeNode(NodeType.ARGGROUP) # If it is a whitespace token then put it directly in the parse tree at # the current depth while tokens and tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue kwargs = {"CACHE": PositionalParser(3, flags=["FORCE"])} flags = ["PARENT_SCOPE"] kwarg_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()) + flags)] positional_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()))] ntokens = len(tokens) subtree = parse_positionals(tokens, 1, flags, positional_breakstack) assert len(tokens) < ntokens tree.children.append(subtree) while tokens: # Break if the next token belongs to a parent parser, i.e. if it # matches a keyword argument of something higher in the stack, or if # it closes a parent group. if should_break(tokens[0], breakstack): break # If it is a whitespace token then put it directly in the parse tree at # the current depth if tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # If it's a comment, then add it at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): if not get_tag(tokens[0]) in ("sort", "sortable"): child = TreeNode(NodeType.COMMENT) tree.children.append(child) child.children.append(tokens.pop(0)) continue ntokens = len(tokens) # NOTE(josh): each flag is also stored in kwargs as with a positional parser # of size zero. This is a legacy thing that should be removed, but for now # just make sure we check flags first. word = get_normalized_kwarg(tokens[0]) if word == "CACHE": subtree = parse_kwarg(tokens, word, kwargs[word], kwarg_breakstack) elif word == "PARENT_SCOPE": subtree = parse_positionals(tokens, '+', ["PARENT_SCOPE"], positional_breakstack) else: subtree = parse_positionals(tokens, '+', [], kwarg_breakstack) assert len(tokens) < ntokens tree.children.append(subtree) return tree
def _reflow(self, config, cursor, passno): """ Compute the size of a keyword argument group which is nominally allocated `linewidth` columns for packing. `linewidth` is only considered for hpacking of consecutive scalar arguments """ # TODO(josh): Much of this code is the same as update_statement_size_, so # dedup the two start_cursor = Cursor(*cursor) children = list(self.children) assert children child = children.pop(0) assert child.type == NodeType.KEYWORD cursor = child.reflow(config, cursor, passno) keyword = parser.get_normalized_kwarg(child.pnode.children[0]) self._colextent = child.colextent if self._wrap == WrapAlgo.HPACK: if len(children) > config.max_subargs_per_line: self._colextent = config.linewidth + 100 while children: cursor[1] += len(' ') child = children.pop(0) if child.type == NodeType.COMMENT: self._colextent = config.linewidth + 100 cursor = child.reflow(config, cursor, passno) self._colextent = max(self._colextent, child.colextent) return cursor elif self._wrap == WrapAlgo.VPACK: cursor[1] += len(' ') column_cursor = Cursor(*cursor) elif self._wrap in (WrapAlgo.KWNVPACK, WrapAlgo.PNVPACK): column_cursor = start_cursor + Cursor(1, config.tab_size) else: raise RuntimeError("Unexepected wrap algorithm") # NOTE(josh): this logic is common to both VPACK and NVPACK, the only # difference is the starting position of the column_cursor prev = None cursor = Cursor(*column_cursor) scalar_seq = analyze_scalar_sequence(children) while children: if (prev is None) or (prev.type not in SCALAR_TYPES): scalar_seq = analyze_scalar_sequence(children) child = children.pop(0) # If both the previous and current nodes are scalar nodes and the two # are not part of a particularly long (by a configurable margin) sequence # of scalar nodes, then advance the cursor horizontally by one space and # try to pack the next child on the same line as the current one if prev is None: cursor = child.reflow(config, cursor, passno) elif (prev.type in SCALAR_TYPES and child.type in SCALAR_TYPES and (scalar_seq.length <= config.max_subargs_per_line or keyword == 'COMMAND' or keyword.startswith('--')) and not scalar_seq.has_comment): cursor[1] += len(' ') # But if the cursor has overflowed the line width allocation, then # we cannot cursor = child.reflow(config, cursor, passno) if child.colextent > config.linewidth: column_cursor[0] += 1 cursor = Cursor(*column_cursor) cursor = child.reflow(config, cursor, passno) # Otherwise we fall back to vpack else: column_cursor[0] += 1 cursor = Cursor(*column_cursor) cursor = child.reflow(config, cursor, passno) self._colextent = max(self._colextent, child.colextent) column_cursor[0] = cursor[0] prev = child return cursor
def parse_add_custom_target(tokens, breakstack): """ :: add_custom_target(Name [ALL] [command1 [args1...]] [COMMAND command2 [args2...] ...] [DEPENDS depend depend depend ... ] [BYPRODUCTS [files...]] [WORKING_DIRECTORY dir] [COMMENT comment] [JOB_POOL job_pool] [VERBATIM] [USES_TERMINAL] [COMMAND_EXPAND_LISTS] [SOURCES src1 [src2...]]) :see: https://cmake.org/cmake/help/latest/command/add_custom_target.html """ kwargs = { "BYPRODUCTS": PositionalParser("+"), "COMMAND": parse_shell_command, "COMMENT": PositionalParser(1), "DEPENDS": PositionalParser("+"), "JOB_POOL": PositionalParser(1), "SOURCES": PositionalParser("+"), "WORKING_DIRECTORY": PositionalParser(1), } flags = ("VERBATIM", "USES_TERMINAL", "COMMAND_EXPAND_LISTS") tree = TreeNode(NodeType.ARGGROUP) # If it is a whitespace token then put it directly in the parse tree at # the current depth while tokens and tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue breaker = KwargBreaker(list(kwargs.keys()) + list(flags)) child_breakstack = breakstack + [breaker] nametree = None state = "name" while tokens: # Break if the next token belongs to a parent parser, i.e. if it # matches a keyword argument of something higher in the stack, or if # it closes a parent group. if should_break(tokens[0], breakstack): break # If it is a whitespace token then put it directly in the parse tree at # the current depth if tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # If it's a comment, then add it at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): child = TreeNode(NodeType.COMMENT) tree.children.append(child) child.children.append(tokens.pop(0)) continue ntokens = len(tokens) if state == "name": next_semantic = get_first_semantic_token(tokens[1:]) if (next_semantic is not None and get_normalized_kwarg(next_semantic) == "ALL"): npargs = 2 else: npargs = 1 nametree = parse_positionals(tokens, npargs, ["ALL"], child_breakstack) assert len(tokens) < ntokens tree.children.append(nametree) state = "first-command" continue word = get_normalized_kwarg(tokens[0]) if state == "first-command": if not(word in kwargs or word in flags): subtree = parse_positionals(tokens, '+', [], child_breakstack) tree.children.append(subtree) assert len(tokens) < ntokens state = "kwargs" continue if word in flags: subtree = parse_flags( tokens, flags, breakstack + [KwargBreaker(list(kwargs.keys()))]) assert len(tokens) < ntokens tree.children.append(subtree) continue if word in kwargs: subtree = parse_kwarg(tokens, word, kwargs[word], child_breakstack) assert len(tokens) < ntokens tree.children.append(subtree) continue logger.warning( "Unexpected positional argument %s at %s", tokens[0].spelling, tokens[0].location()) subtree = parse_positionals(tokens, '+', [], child_breakstack) assert len(tokens) < ntokens tree.children.append(subtree) continue return tree
def parse_install_targets(tokens, breakstack): """ :: install(TARGETS targets... [EXPORT <export-name>] [[ARCHIVE|LIBRARY|RUNTIME|OBJECTS|FRAMEWORK|BUNDLE| PRIVATE_HEADER|PUBLIC_HEADER|RESOURCE] [DESTINATION <dir>] [PERMISSIONS permissions...] [CONFIGURATIONS [Debug|Release|...]] [COMPONENT <component>] [NAMELINK_COMPONENT <component>] [OPTIONAL] [EXCLUDE_FROM_ALL] [NAMELINK_ONLY|NAMELINK_SKIP] ] [...] [INCLUDES DESTINATION [<dir> ...]] ) :see: https://cmake.org/cmake/help/v3.14/command/install.html#targets """ kwargs = { "TARGETS": PositionalParser('+'), "EXPORT": PositionalParser(1), "INCLUDES": PositionalParser('+', flags=["DESTINATION"]), # Common kwargs "DESTINATION": PositionalParser(1), "PERMISSIONS": PositionalParser('+'), "CONFIGURATIONS": PositionalParser('+'), "COMPONENT": PositionalParser(1), "NAMELINK_COMPONENT": PositionalParser(1), } flags = ( "OPTIONAL", "EXCLUDE_FROM_ALL", "NAMELINK_ONLY", "NAMELINK_SKIP" ) designated_kwargs = ( "ARCHIVE", "LIBRARY", "RUNTIME", "OBJECTS", "FRAMEWORK", "BUNDLE", "PRIVATE_HEADER", "PUBLIC_HEADER", "RESOURCE" ) # NOTE(josh): from here on, code is essentially parse_standard(), except that # we cannot break on the common subset of kwargs in the breakstack because # they are valid kwargs for the subtrees (ARCHIVE, LIBRARY, etc) as well as # the primary tree tree = TreeNode(NodeType.ARGGROUP) # If it is a whitespace token then put it directly in the parse tree at # the current depth while tokens and tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # ARCHIVE, LIBRARY, RUNTIME, subtrees etc only break on the start of # another subtree, or on "INCLUDES DESTINATION" subtree_breakstack = breakstack + [KwargBreaker( list(designated_kwargs) + ["INCLUDES"] )] # kwargs at this tree depth break on other kwargs or flags kwarg_breakstack = breakstack + [KwargBreaker( list(kwargs.keys()) + list(designated_kwargs) + list(flags) )] # and flags at this depth break only on kwargs positional_breakstack = breakstack + [KwargBreaker( list(kwargs.keys()) + list(designated_kwargs) )] while tokens: # Break if the next token belongs to a parent parser, i.e. if it # matches a keyword argument of something higher in the stack, or if # it closes a parent group. if should_break(tokens[0], breakstack): break # If it is a whitespace token then put it directly in the parse tree at # the current depth if tokens[0].type in WHITESPACE_TOKENS: tree.children.append(tokens.pop(0)) continue # If it's a comment, then add it at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): child = TreeNode(NodeType.COMMENT) tree.children.append(child) child.children.append(tokens.pop(0)) continue ntokens = len(tokens) # NOTE(josh): each flag is also stored in kwargs as with a positional parser # of size zero. This is a legacy thing that should be removed, but for now # just make sure we check flags first. word = get_normalized_kwarg(tokens[0]) if word in designated_kwargs: subtree = parse_kwarg( tokens, word, parse_install_targets, subtree_breakstack) elif word in kwargs: subtree = parse_kwarg(tokens, word, kwargs[word], kwarg_breakstack) else: subtree = parse_positionals(tokens, '+', flags, positional_breakstack) assert len(tokens) < ntokens tree.children.append(subtree) return tree