def parse_foreach_in(tokens, breakstack):
"""
::
foreach(loop_var IN [LISTS [<lists>]] [ITEMS [<items>]])
"""
tree = TreeNode(NodeType.ARGGROUP)
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
while tokens and tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
# Consume the loop variable and any attached comments
ntokens = len(tokens)
pargs = parse_positionals(tokens, 2, ["IN"], breakstack)
assert len(tokens) < ntokens
tree.children.append(pargs)
kwargs = {"LISTS": PositionalParser("+"), "ITEMS": PositionalParser("+")}
sub_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()))]
while tokens:
# Break if the next token belongs to a parent parser, i.e. if it
# matches a keyword argument of something higher in the stack, or if
# it closes a parent group.
if should_break(tokens[0], breakstack):
break
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
if tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
# If it's a comment, then add it at the current depth
if tokens[0].type in (lexer.TokenType.COMMENT,
lexer.TokenType.BRACKET_COMMENT):
if not get_tag(tokens[0]) in ("sort", "sortable"):
child = TreeNode(NodeType.COMMENT)
tree.children.append(child)
child.children.append(tokens.pop(0))
continue
ntokens = len(tokens)
word = get_normalized_kwarg(tokens[0])
subparser = kwargs.get(word, None)
if subparser is not None:
subtree = parse_kwarg(tokens, word, subparser, sub_breakstack)
else:
logger.warning("Unexpected positional argument at %s",
tokens[0].location)
subtree = parse_positionals(tokens, '+', [], sub_breakstack)
assert len(tokens) < ntokens
tree.children.append(subtree)
return tree
def parse_add_custom_command(tokens, breakstack):
"""
There are two forms of `add_custom_command`. This is the dispatcher between
the two forms.
"""
descriminator_token = get_first_semantic_token(tokens)
descriminator_word = get_normalized_kwarg(descriminator_token)
if descriminator_word == "TARGET":
return parse_add_custom_command_events(tokens, breakstack)
if descriminator_word == "OUTPUT":
return parse_add_custom_command_standard(tokens, breakstack)
logger.warning(
"Indeterminate form of add_custom_command at %s",
descriminator_token.location)
return parse_add_custom_command_standard(tokens, breakstack)
def parse_add_executable_standard(tokens, breakstack, sortable):
"""
::
add_executable(<name> [WIN32] [MACOSX_BUNDLE]
[EXCLUDE_FROM_ALL]
[source1] [source2 ...])
:see: https://cmake.org/cmake/help/latest/command/add_executable.html#command:add_executable
"""
# pylint: disable=too-many-statements
parsing_name = 1
parsing_flags = 2
parsing_sources = 3
tree = TreeNode(NodeType.ARGGROUP)
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
while tokens and tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
state_ = parsing_name
parg_group = None
src_group = None
active_depth = tree
while tokens:
# This parse function breaks on the first right paren, since parenthetical
# groups are not allowed. A parenthesis might exist in a filename, but
# if so that filename should be quoted so it wont show up as a RIGHT_PAREN
# token.
if tokens[0].type is lexer.TokenType.RIGHT_PAREN:
break
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
if tokens[0].type in WHITESPACE_TOKENS:
active_depth.children.append(tokens.pop(0))
continue
# If it's a comment token not associated with an argument, then put it
# directly into the parse tree at the current depth
if tokens[0].type in (lexer.TokenType.COMMENT,
lexer.TokenType.BRACKET_COMMENT):
if state_ > parsing_name:
if get_tag(tokens[0]) in ("unsort", "unsortable"):
sortable = False
elif get_tag(tokens[0]) in ("unsort", "unsortable"):
sortable = True
child = TreeNode(NodeType.COMMENT)
active_depth.children.append(child)
child.children.append(tokens.pop(0))
continue
if state_ is parsing_name:
token = tokens.pop(0)
parg_group = TreeNode(NodeType.PARGGROUP)
active_depth = parg_group
tree.children.append(parg_group)
child = TreeNode(NodeType.ARGUMENT)
child.children.append(token)
consume_trailing_comment(child, tokens)
parg_group.children.append(child)
state_ += 1
elif state_ is parsing_flags:
if get_normalized_kwarg(tokens[0]) in (
"WIN32", "MACOSX_BUNDLE", "EXCLUDE_FROM_ALL"):
token = tokens.pop(0)
child = TreeNode(NodeType.FLAG)
child.children.append(token)
consume_trailing_comment(child, tokens)
parg_group.children.append(child)
else:
state_ += 1
src_group = TreeNode(NodeType.PARGGROUP, sortable=sortable)
active_depth = src_group
tree.children.append(src_group)
elif state_ is parsing_sources:
token = tokens.pop(0)
child = TreeNode(NodeType.ARGUMENT)
child.children.append(token)
consume_trailing_comment(child, tokens)
src_group.children.append(child)
if only_comments_and_whitespace_remain(tokens, breakstack):
active_depth = tree
return tree
def _reflow(self, config, cursor, passno):
"""
Compute the size of a keyword argument group which is nominally allocated
`linewidth` columns for packing. `linewidth` is only considered for hpacking
of consecutive scalar arguments
"""
# TODO(josh): Much of this code is the same as update_statement_size_, so
# dedup the two
start_cursor = np.array(cursor)
children = list(self.children)
assert children
child = children.pop(0)
assert child.type == NodeType.KEYWORD
cursor = child.reflow(config, cursor, passno)
keyword = parser.get_normalized_kwarg(child.pnode.children[0])
self._colextent = child.colextent
if self._wrap == WrapAlgo.HPACK:
if len(children) > config.max_subargs_per_line:
self._colextent = config.linewidth + 100
while children:
cursor[1] += len(' ')
child = children.pop(0)
if child.type == NodeType.COMMENT:
self._colextent = config.linewidth + 100
cursor = child.reflow(config, cursor, passno)
self._colextent = max(self._colextent, child.colextent)
return cursor
elif self._wrap == WrapAlgo.VPACK:
cursor[1] += len(' ')
column_cursor = np.array(cursor)
elif self._wrap in (WrapAlgo.KWNVPACK, WrapAlgo.PNVPACK):
column_cursor = start_cursor + np.array((1, config.tab_size))
else:
raise RuntimeError("Unexepected wrap algorithm")
# NOTE(josh): this logic is common to both VPACK and NVPACK, the only
# difference is the starting position of the column_cursor
prev = None
cursor = np.array(column_cursor)
scalar_seq = analyze_scalar_sequence(children)
while children:
if (prev is None) or (prev.type not in SCALAR_TYPES):
scalar_seq = analyze_scalar_sequence(children)
child = children.pop(0)
# If both the previous and current nodes are scalar nodes and the two
# are not part of a particularly long (by a configurable margin) sequence
# of scalar nodes, then advance the cursor horizontally by one space and
# try to pack the next child on the same line as the current one
if prev is None:
cursor = child.reflow(config, cursor, passno)
elif (prev.type in SCALAR_TYPES
and child.type in SCALAR_TYPES
and (scalar_seq.length <= config.max_subargs_per_line
or keyword == 'COMMAND'
or keyword.startswith('--'))
and not scalar_seq.has_comment):
cursor[1] += len(' ')
# But if the cursor has overflowed the line width allocation, then
# we cannot
cursor = child.reflow(config, cursor, passno)
if child.colextent > config.linewidth:
column_cursor[0] += 1
cursor = np.array(column_cursor)
cursor = child.reflow(config, cursor, passno)
# Otherwise we fall back to vpack
else:
column_cursor[0] += 1
cursor = np.array(column_cursor)
cursor = child.reflow(config, cursor, passno)
self._colextent = max(self._colextent, child.colextent)
column_cursor[0] = cursor[0]
prev = child
return cursor
def parse_set(tokens, breakstack):
"""
::
set(<variable> <value>
[[CACHE <type> <docstring> [FORCE]] | PARENT_SCOPE])
:see: https://cmake.org/cmake/help/v3.0/command/set.html?
"""
tree = TreeNode(NodeType.ARGGROUP)
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
while tokens and tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
kwargs = {"CACHE": PositionalParser(3, flags=["FORCE"])}
flags = ["PARENT_SCOPE"]
kwarg_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()) + flags)]
positional_breakstack = breakstack + [KwargBreaker(list(kwargs.keys()))]
ntokens = len(tokens)
subtree = parse_positionals(tokens, 1, flags, positional_breakstack)
assert len(tokens) < ntokens
tree.children.append(subtree)
while tokens:
# Break if the next token belongs to a parent parser, i.e. if it
# matches a keyword argument of something higher in the stack, or if
# it closes a parent group.
if should_break(tokens[0], breakstack):
break
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
if tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
# If it's a comment, then add it at the current depth
if tokens[0].type in (lexer.TokenType.COMMENT,
lexer.TokenType.BRACKET_COMMENT):
if not get_tag(tokens[0]) in ("sort", "sortable"):
child = TreeNode(NodeType.COMMENT)
tree.children.append(child)
child.children.append(tokens.pop(0))
continue
ntokens = len(tokens)
# NOTE(josh): each flag is also stored in kwargs as with a positional parser
# of size zero. This is a legacy thing that should be removed, but for now
# just make sure we check flags first.
word = get_normalized_kwarg(tokens[0])
if word == "CACHE":
subtree = parse_kwarg(tokens, word, kwargs[word], kwarg_breakstack)
elif word == "PARENT_SCOPE":
subtree = parse_positionals(tokens, '+', ["PARENT_SCOPE"],
positional_breakstack)
else:
subtree = parse_positionals(tokens, '+', [], kwarg_breakstack)
assert len(tokens) < ntokens
tree.children.append(subtree)
return tree
def _reflow(self, config, cursor, passno):
"""
Compute the size of a keyword argument group which is nominally allocated
`linewidth` columns for packing. `linewidth` is only considered for hpacking
of consecutive scalar arguments
"""
# TODO(josh): Much of this code is the same as update_statement_size_, so
# dedup the two
start_cursor = Cursor(*cursor)
children = list(self.children)
assert children
child = children.pop(0)
assert child.type == NodeType.KEYWORD
cursor = child.reflow(config, cursor, passno)
keyword = parser.get_normalized_kwarg(child.pnode.children[0])
self._colextent = child.colextent
if self._wrap == WrapAlgo.HPACK:
if len(children) > config.max_subargs_per_line:
self._colextent = config.linewidth + 100
while children:
cursor[1] += len(' ')
child = children.pop(0)
if child.type == NodeType.COMMENT:
self._colextent = config.linewidth + 100
cursor = child.reflow(config, cursor, passno)
self._colextent = max(self._colextent, child.colextent)
return cursor
elif self._wrap == WrapAlgo.VPACK:
cursor[1] += len(' ')
column_cursor = Cursor(*cursor)
elif self._wrap in (WrapAlgo.KWNVPACK, WrapAlgo.PNVPACK):
column_cursor = start_cursor + Cursor(1, config.tab_size)
else:
raise RuntimeError("Unexepected wrap algorithm")
# NOTE(josh): this logic is common to both VPACK and NVPACK, the only
# difference is the starting position of the column_cursor
prev = None
cursor = Cursor(*column_cursor)
scalar_seq = analyze_scalar_sequence(children)
while children:
if (prev is None) or (prev.type not in SCALAR_TYPES):
scalar_seq = analyze_scalar_sequence(children)
child = children.pop(0)
# If both the previous and current nodes are scalar nodes and the two
# are not part of a particularly long (by a configurable margin) sequence
# of scalar nodes, then advance the cursor horizontally by one space and
# try to pack the next child on the same line as the current one
if prev is None:
cursor = child.reflow(config, cursor, passno)
elif (prev.type in SCALAR_TYPES
and child.type in SCALAR_TYPES
and (scalar_seq.length <= config.max_subargs_per_line
or keyword == 'COMMAND'
or keyword.startswith('--'))
and not scalar_seq.has_comment):
cursor[1] += len(' ')
# But if the cursor has overflowed the line width allocation, then
# we cannot
cursor = child.reflow(config, cursor, passno)
if child.colextent > config.linewidth:
column_cursor[0] += 1
cursor = Cursor(*column_cursor)
cursor = child.reflow(config, cursor, passno)
# Otherwise we fall back to vpack
else:
column_cursor[0] += 1
cursor = Cursor(*column_cursor)
cursor = child.reflow(config, cursor, passno)
self._colextent = max(self._colextent, child.colextent)
column_cursor[0] = cursor[0]
prev = child
return cursor
def parse_add_custom_target(tokens, breakstack):
"""
::
add_custom_target(Name [ALL] [command1 [args1...]]
[COMMAND command2 [args2...] ...]
[DEPENDS depend depend depend ... ]
[BYPRODUCTS [files...]]
[WORKING_DIRECTORY dir]
[COMMENT comment]
[JOB_POOL job_pool]
[VERBATIM] [USES_TERMINAL]
[COMMAND_EXPAND_LISTS]
[SOURCES src1 [src2...]])
:see: https://cmake.org/cmake/help/latest/command/add_custom_target.html
"""
kwargs = {
"BYPRODUCTS": PositionalParser("+"),
"COMMAND": parse_shell_command,
"COMMENT": PositionalParser(1),
"DEPENDS": PositionalParser("+"),
"JOB_POOL": PositionalParser(1),
"SOURCES": PositionalParser("+"),
"WORKING_DIRECTORY": PositionalParser(1),
}
flags = ("VERBATIM", "USES_TERMINAL", "COMMAND_EXPAND_LISTS")
tree = TreeNode(NodeType.ARGGROUP)
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
while tokens and tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
breaker = KwargBreaker(list(kwargs.keys()) + list(flags))
child_breakstack = breakstack + [breaker]
nametree = None
state = "name"
while tokens:
# Break if the next token belongs to a parent parser, i.e. if it
# matches a keyword argument of something higher in the stack, or if
# it closes a parent group.
if should_break(tokens[0], breakstack):
break
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
if tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
# If it's a comment, then add it at the current depth
if tokens[0].type in (lexer.TokenType.COMMENT,
lexer.TokenType.BRACKET_COMMENT):
child = TreeNode(NodeType.COMMENT)
tree.children.append(child)
child.children.append(tokens.pop(0))
continue
ntokens = len(tokens)
if state == "name":
next_semantic = get_first_semantic_token(tokens[1:])
if (next_semantic is not None and
get_normalized_kwarg(next_semantic) == "ALL"):
npargs = 2
else:
npargs = 1
nametree = parse_positionals(tokens, npargs, ["ALL"], child_breakstack)
assert len(tokens) < ntokens
tree.children.append(nametree)
state = "first-command"
continue
word = get_normalized_kwarg(tokens[0])
if state == "first-command":
if not(word in kwargs or word in flags):
subtree = parse_positionals(tokens, '+', [], child_breakstack)
tree.children.append(subtree)
assert len(tokens) < ntokens
state = "kwargs"
continue
if word in flags:
subtree = parse_flags(
tokens, flags, breakstack + [KwargBreaker(list(kwargs.keys()))])
assert len(tokens) < ntokens
tree.children.append(subtree)
continue
if word in kwargs:
subtree = parse_kwarg(tokens, word, kwargs[word], child_breakstack)
assert len(tokens) < ntokens
tree.children.append(subtree)
continue
logger.warning(
"Unexpected positional argument %s at %s",
tokens[0].spelling, tokens[0].location())
subtree = parse_positionals(tokens, '+', [], child_breakstack)
assert len(tokens) < ntokens
tree.children.append(subtree)
continue
return tree
def parse_install_targets(tokens, breakstack):
"""
::
install(TARGETS targets... [EXPORT <export-name>]
[[ARCHIVE|LIBRARY|RUNTIME|OBJECTS|FRAMEWORK|BUNDLE|
PRIVATE_HEADER|PUBLIC_HEADER|RESOURCE]
[DESTINATION <dir>]
[PERMISSIONS permissions...]
[CONFIGURATIONS [Debug|Release|...]]
[COMPONENT <component>]
[NAMELINK_COMPONENT <component>]
[OPTIONAL] [EXCLUDE_FROM_ALL]
[NAMELINK_ONLY|NAMELINK_SKIP]
] [...]
[INCLUDES DESTINATION [<dir> ...]]
)
:see: https://cmake.org/cmake/help/v3.14/command/install.html#targets
"""
kwargs = {
"TARGETS": PositionalParser('+'),
"EXPORT": PositionalParser(1),
"INCLUDES": PositionalParser('+', flags=["DESTINATION"]),
# Common kwargs
"DESTINATION": PositionalParser(1),
"PERMISSIONS": PositionalParser('+'),
"CONFIGURATIONS": PositionalParser('+'),
"COMPONENT": PositionalParser(1),
"NAMELINK_COMPONENT": PositionalParser(1),
}
flags = (
"OPTIONAL",
"EXCLUDE_FROM_ALL",
"NAMELINK_ONLY",
"NAMELINK_SKIP"
)
designated_kwargs = (
"ARCHIVE", "LIBRARY", "RUNTIME", "OBJECTS", "FRAMEWORK",
"BUNDLE", "PRIVATE_HEADER", "PUBLIC_HEADER", "RESOURCE"
)
# NOTE(josh): from here on, code is essentially parse_standard(), except that
# we cannot break on the common subset of kwargs in the breakstack because
# they are valid kwargs for the subtrees (ARCHIVE, LIBRARY, etc) as well as
# the primary tree
tree = TreeNode(NodeType.ARGGROUP)
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
while tokens and tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
# ARCHIVE, LIBRARY, RUNTIME, subtrees etc only break on the start of
# another subtree, or on "INCLUDES DESTINATION"
subtree_breakstack = breakstack + [KwargBreaker(
list(designated_kwargs) + ["INCLUDES"]
)]
# kwargs at this tree depth break on other kwargs or flags
kwarg_breakstack = breakstack + [KwargBreaker(
list(kwargs.keys()) + list(designated_kwargs) + list(flags)
)]
# and flags at this depth break only on kwargs
positional_breakstack = breakstack + [KwargBreaker(
list(kwargs.keys()) + list(designated_kwargs)
)]
while tokens:
# Break if the next token belongs to a parent parser, i.e. if it
# matches a keyword argument of something higher in the stack, or if
# it closes a parent group.
if should_break(tokens[0], breakstack):
break
# If it is a whitespace token then put it directly in the parse tree at
# the current depth
if tokens[0].type in WHITESPACE_TOKENS:
tree.children.append(tokens.pop(0))
continue
# If it's a comment, then add it at the current depth
if tokens[0].type in (lexer.TokenType.COMMENT,
lexer.TokenType.BRACKET_COMMENT):
child = TreeNode(NodeType.COMMENT)
tree.children.append(child)
child.children.append(tokens.pop(0))
continue
ntokens = len(tokens)
# NOTE(josh): each flag is also stored in kwargs as with a positional parser
# of size zero. This is a legacy thing that should be removed, but for now
# just make sure we check flags first.
word = get_normalized_kwarg(tokens[0])
if word in designated_kwargs:
subtree = parse_kwarg(
tokens, word, parse_install_targets, subtree_breakstack)
elif word in kwargs:
subtree = parse_kwarg(tokens, word, kwargs[word], kwarg_breakstack)
else:
subtree = parse_positionals(tokens, '+', flags, positional_breakstack)
assert len(tokens) < ntokens
tree.children.append(subtree)
return tree