def convert_simple_stmt_partial(config: ParserConfig,
children: Sequence[Any]) -> Any:
*statements, trailing_whitespace = children
last_stmt = len(statements) / 2
body = []
for i, (stmt_body, semi) in enumerate(grouper(statements, 2)):
if semi is not None:
if i == (last_stmt - 1):
# Trailing semicolons only own the whitespace before.
semi = Semicolon(
whitespace_before=parse_simple_whitespace(
config, semi.whitespace_before),
whitespace_after=SimpleWhitespace(""),
)
else:
# Middle semicolons own the whitespace before and after.
semi = Semicolon(
whitespace_before=parse_simple_whitespace(
config, semi.whitespace_before),
whitespace_after=parse_simple_whitespace(
config, semi.whitespace_after),
)
else:
semi = MaybeSentinel.DEFAULT
body.append(stmt_body.value.with_changes(semicolon=semi))
return SimpleStatementPartial(
body,
whitespace_before=statements[0].whitespace_before,
trailing_whitespace=trailing_whitespace,
)
def convert_annassign(config: ParserConfig, children: Sequence[Any]) -> Any:
if len(children) == 2:
# Variable annotation only
colon, annotation = children
annotation = annotation.value
equal = None
value = None
elif len(children) == 4:
# Variable annotation and assignment
colon, annotation, equal, value = children
annotation = annotation.value
value = value.value
equal = AssignEqual(
whitespace_before=parse_simple_whitespace(config,
equal.whitespace_before),
whitespace_after=parse_simple_whitespace(config,
equal.whitespace_after),
)
else:
raise Exception("Invalid parser state!")
return AnnAssignPartial(
annotation=Annotation(
whitespace_before_indicator=parse_simple_whitespace(
config, colon.whitespace_before),
whitespace_after_indicator=parse_simple_whitespace(
config, colon.whitespace_after),
annotation=annotation,
),
equal=equal,
value=value,
)
def convert_classdef(config: ParserConfig, children: Sequence[Any]) -> Any:
classdef, name, *arglist, colon, suite = children
# First, parse out the comments and empty lines before the statement.
leading_lines = parse_empty_lines(config, classdef.whitespace_before)
# Compute common whitespace and nodes
whitespace_after_class = parse_simple_whitespace(config,
classdef.whitespace_after)
namenode = Name(name.string)
whitespace_after_name = parse_simple_whitespace(config,
name.whitespace_after)
# Now, construct the classdef node itself
if not arglist:
# No arglist, so no arguments to this class
return ClassDef(
leading_lines=leading_lines,
lines_after_decorators=(),
whitespace_after_class=whitespace_after_class,
name=namenode,
whitespace_after_name=whitespace_after_name,
body=suite,
)
else:
# Unwrap arglist partial, because its valid to not have any
lpar, *args, rpar = arglist
args = args[0].args if args else []
bases: List[Arg] = []
keywords: List[Arg] = []
current_arg = bases
for arg in args:
if arg.star == "**" or arg.keyword is not None:
current_arg = keywords
# Some quick validation
if current_arg is keywords and (arg.star == "*" or
(arg.star == ""
and arg.keyword is None)):
raise PartialParserSyntaxError(
"Positional argument follows keyword argument.")
current_arg.append(arg)
return ClassDef(
leading_lines=leading_lines,
lines_after_decorators=(),
whitespace_after_class=whitespace_after_class,
name=namenode,
whitespace_after_name=whitespace_after_name,
lpar=LeftParen(whitespace_after=parse_parenthesizable_whitespace(
config, lpar.whitespace_after)),
bases=bases,
keywords=keywords,
rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace(
config, rpar.whitespace_before)),
whitespace_before_colon=parse_simple_whitespace(
config, colon.whitespace_before),
body=suite,
)
def convert_decorator(config: ParserConfig, children: Sequence[Any]) -> Any:
atsign, name, *arglist, newline = children
if not arglist:
# This is either a name or an attribute node, so just extract it.
decoratornode = name
else:
# This needs to be converted into a call node, and we have the
# arglist partial.
lpar, *args, rpar = arglist
args = args[0].args if args else []
# If the trailing argument doesn't have a comma, then it owns the
# trailing whitespace before the rpar. Otherwise, the comma owns
# it.
if len(args) > 0 and args[-1].comma == MaybeSentinel.DEFAULT:
args[-1] = args[-1].with_changes(
whitespace_after_arg=parse_parenthesizable_whitespace(
config, rpar.whitespace_before))
decoratornode = Call(
func=name,
whitespace_after_func=parse_simple_whitespace(
config, lpar.whitespace_before),
whitespace_before_args=parse_parenthesizable_whitespace(
config, lpar.whitespace_after),
args=tuple(args),
)
return Decorator(
leading_lines=parse_empty_lines(config, atsign.whitespace_before),
whitespace_after_at=parse_simple_whitespace(config,
atsign.whitespace_after),
decorator=decoratornode,
trailing_whitespace=newline,
)
def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
(with_token, *items, colon_token, suite) = children
item_nodes: List[WithItem] = []
for with_item, maybe_comma in grouper(items, 2):
if maybe_comma is not None:
item_nodes.append(
with_item.with_changes(comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, maybe_comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, maybe_comma.whitespace_after),
)))
else:
item_nodes.append(with_item)
return WithLeadingWhitespace(
With(
whitespace_after_with=parse_simple_whitespace(
config, with_token.whitespace_after),
items=tuple(item_nodes),
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before),
body=suite,
),
with_token.whitespace_before,
)
def convert_except_clause(config: ParserConfig,
children: Sequence[Any]) -> Any:
if len(children) == 1:
(except_token, ) = children
whitespace_after_except = SimpleWhitespace("")
test = None
name = None
elif len(children) == 2:
(except_token, test_node) = children
whitespace_after_except = parse_simple_whitespace(
config, except_token.whitespace_after)
test = test_node.value
name = None
else:
(except_token, test_node, as_token, name_token) = children
whitespace_after_except = parse_simple_whitespace(
config, except_token.whitespace_after)
test = test_node.value
name = AsName(
whitespace_before_as=parse_simple_whitespace(
config, as_token.whitespace_before),
whitespace_after_as=parse_simple_whitespace(
config, as_token.whitespace_after),
name=Name(name_token.string),
)
return ExceptClausePartial(
leading_lines=parse_empty_lines(config,
except_token.whitespace_before),
whitespace_after_except=whitespace_after_except,
type=test,
name=name,
)
def convert_while_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
while_token, test, while_colon_token, while_suite, *else_block = children
if len(else_block) > 0:
(else_token, else_colon_token, else_suite) = else_block
orelse = Else(
leading_lines=parse_empty_lines(config,
else_token.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, else_colon_token.whitespace_before),
body=else_suite,
)
else:
orelse = None
return While(
leading_lines=parse_empty_lines(config, while_token.whitespace_before),
whitespace_after_while=parse_simple_whitespace(
config, while_token.whitespace_after),
test=test.value,
whitespace_before_colon=parse_simple_whitespace(
config, while_colon_token.whitespace_before),
body=while_suite,
orelse=orelse,
)
def convert_assert_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
if len(children) == 2:
(assert_token, test) = children
assert_node = Assert(
whitespace_after_assert=parse_simple_whitespace(
config, test.whitespace_before),
test=test.value,
msg=None,
)
else:
(assert_token, test, comma_token, msg) = children
assert_node = Assert(
whitespace_after_assert=parse_simple_whitespace(
config, test.whitespace_before),
test=test.value,
comma=Comma(
whitespace_before=parse_simple_whitespace(
config, comma_token.whitespace_before),
whitespace_after=parse_simple_whitespace(
config, msg.whitespace_before),
),
msg=msg.value,
)
return WithLeadingWhitespace(assert_node, assert_token.whitespace_before)
def convert_raise_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
if len(children) == 1:
(raise_token, ) = children
whitespace_after_raise = MaybeSentinel.DEFAULT
exc = None
cause = None
elif len(children) == 2:
(raise_token, test) = children
whitespace_after_raise = parse_simple_whitespace(
config, test.whitespace_before)
exc = test.value
cause = None
elif len(children) == 4:
(raise_token, test, from_token, source) = children
whitespace_after_raise = parse_simple_whitespace(
config, test.whitespace_before)
exc = test.value
cause = From(
whitespace_before_from=parse_simple_whitespace(
config, from_token.whitespace_before),
whitespace_after_from=parse_simple_whitespace(
config, source.whitespace_before),
item=source.value,
)
else:
raise Exception("Logic error!")
return WithLeadingWhitespace(
Raise(whitespace_after_raise=whitespace_after_raise,
exc=exc,
cause=cause),
raise_token.whitespace_before,
)
def convert_import_relative(config: ParserConfig,
children: Sequence[Any]) -> Any:
dots = []
dotted_name = None
for child in children:
if isinstance(child, Token):
# Special case for "...", which is part of the grammar
if child.string == "...":
dots.extend([
Dot(),
Dot(),
Dot(whitespace_after=parse_simple_whitespace(
config, child.whitespace_after)),
])
else:
dots.append(
Dot(whitespace_after=parse_simple_whitespace(
config, child.whitespace_after)))
else:
# This should be the dotted name, and we can't get more than
# one, but lets be sure anyway
if dotted_name is not None:
raise Exception("Logic error!")
dotted_name = child
return ImportRelativePartial(relative=tuple(dots), module=dotted_name)
def convert_assign(config: ParserConfig, children: Sequence[Any]) -> Any:
equal, expr = children
return AssignPartial(
equal=AssignEqual(
whitespace_before=parse_simple_whitespace(config, equal.whitespace_before),
whitespace_after=parse_simple_whitespace(config, equal.whitespace_after),
),
value=expr.value,
)
def convert_try_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
trytoken, try_colon_token, try_suite, *rest = children
handlers: List[ExceptHandler] = []
orelse: Optional[Else] = None
finalbody: Optional[Finally] = None
for clause, colon_token, suite in grouper(rest, 3):
if isinstance(clause, Token):
if clause.string == "else":
if orelse is not None:
raise Exception("Logic error!")
orelse = Else(
leading_lines=parse_empty_lines(config, clause.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
body=suite,
)
elif clause.string == "finally":
if finalbody is not None:
raise Exception("Logic error!")
finalbody = Finally(
leading_lines=parse_empty_lines(config, clause.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
body=suite,
)
else:
raise Exception("Logic error!")
elif isinstance(clause, ExceptClausePartial):
handlers.append(
ExceptHandler(
body=suite,
type=clause.type,
name=clause.name,
leading_lines=clause.leading_lines,
whitespace_after_except=clause.whitespace_after_except,
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
)
)
else:
raise Exception("Logic error!")
return Try(
leading_lines=parse_empty_lines(config, trytoken.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, try_colon_token.whitespace_before
),
body=try_suite,
handlers=tuple(handlers),
orelse=orelse,
finalbody=finalbody,
)
def convert_funcdef(config: ParserConfig, children: Sequence[Any]) -> Any:
defnode, namenode, param_partial, *annotation, colon, suite = children
# If the trailing paremeter doesn't have a comma, then it owns the trailing
# whitespace before the rpar. Otherwise, the comma owns it (and will have
# already parsed it). We don't check/update ParamStar because if it exists
# then we are guaranteed have at least one kwonly_param.
parameters = param_partial.params
if parameters.star_kwarg is not None:
if parameters.star_kwarg.comma == MaybeSentinel.DEFAULT:
parameters = parameters.with_changes(
star_kwarg=parameters.star_kwarg.with_changes(
whitespace_after_param=param_partial.rpar.whitespace_before
))
elif parameters.kwonly_params:
if parameters.kwonly_params[-1].comma == MaybeSentinel.DEFAULT:
parameters = parameters.with_changes(kwonly_params=(
*parameters.kwonly_params[:-1],
parameters.kwonly_params[-1].with_changes(
whitespace_after_param=param_partial.rpar.whitespace_before
),
))
elif isinstance(parameters.star_arg, Param):
if parameters.star_arg.comma == MaybeSentinel.DEFAULT:
parameters = parameters.with_changes(
star_arg=parameters.star_arg.with_changes(
whitespace_after_param=param_partial.rpar.whitespace_before
))
elif parameters.params:
if parameters.params[-1].comma == MaybeSentinel.DEFAULT:
parameters = parameters.with_changes(params=(
*parameters.params[:-1],
parameters.params[-1].with_changes(
whitespace_after_param=param_partial.rpar.whitespace_before
),
))
return WithLeadingWhitespace(
FunctionDef(
whitespace_after_def=parse_simple_whitespace(
config, defnode.whitespace_after),
name=Name(namenode.string),
whitespace_after_name=parse_simple_whitespace(
config, namenode.whitespace_after),
whitespace_before_params=param_partial.lpar.whitespace_after,
params=parameters,
returns=None if not annotation else annotation[0],
whitespace_before_colon=parse_simple_whitespace(
config, colon.whitespace_before),
body=suite,
),
defnode.whitespace_before,
)
def convert_augassign(config: ParserConfig, children: Sequence[Any]) -> Any:
op, expr = children
if op.string not in AUGOP_TOKEN_LUT:
raise Exception(f"Unexpected token '{op.string}'!")
return AugAssignPartial(
# pyre-ignore Pyre seems to think that the value of this LUT is CSTNode
operator=AUGOP_TOKEN_LUT[op.string](
whitespace_before=parse_simple_whitespace(config, op.whitespace_before),
whitespace_after=parse_simple_whitespace(config, op.whitespace_after),
),
value=expr.value,
)
def convert_import_from(config: ParserConfig, children: Sequence[Any]) -> Any:
fromtoken, import_relative, importtoken, *importlist = children
if len(importlist) == 1:
(possible_star, ) = importlist
if isinstance(possible_star, Token):
# Its a "*" import, so we must construct this node.
names = ImportStar()
else:
# Its an import as names partial, grab the names from that.
names = possible_star.names
lpar = None
rpar = None
else:
# Its an import as names partial with parens
lpartoken, namespartial, rpartoken = importlist
lpar = LeftParen(whitespace_after=parse_parenthesizable_whitespace(
config, lpartoken.whitespace_after))
names = namespartial.names
rpar = RightParen(whitespace_before=parse_parenthesizable_whitespace(
config, rpartoken.whitespace_before))
# If we have a relative-only import, then we need to relocate the space
# after the final dot to be owned by the import token.
if len(import_relative.relative) > 0 and import_relative.module is None:
whitespace_before_import = import_relative.relative[
-1].whitespace_after
relative = (
*import_relative.relative[:-1],
import_relative.relative[-1].with_changes(
whitespace_after=SimpleWhitespace("")),
)
else:
whitespace_before_import = parse_simple_whitespace(
config, importtoken.whitespace_before)
relative = import_relative.relative
return WithLeadingWhitespace(
ImportFrom(
whitespace_after_from=parse_simple_whitespace(
config, fromtoken.whitespace_after),
relative=relative,
module=import_relative.module,
whitespace_before_import=whitespace_before_import,
whitespace_after_import=parse_simple_whitespace(
config, importtoken.whitespace_after),
lpar=lpar,
names=names,
rpar=rpar,
),
fromtoken.whitespace_before,
)
def convert_with_item(config: ParserConfig, children: Sequence[Any]) -> Any:
if len(children) == 3:
(test, as_token, expr_node) = children
test_node = test.value
asname = AsName(
whitespace_before_as=parse_simple_whitespace(
config, as_token.whitespace_before),
whitespace_after_as=parse_simple_whitespace(
config, as_token.whitespace_after),
name=expr_node.value,
)
else:
(test, ) = children
test_node = test.value
asname = None
return WithItem(item=test_node, asname=asname)
def convert_if_stmt_else(config: ParserConfig, children: Sequence[Any]) -> Any:
else_tok, colon_tok, suite = children
return Else(
leading_lines=parse_empty_lines(config, else_tok.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, colon_tok.whitespace_before),
body=suite,
)
def convert_import_as_name(config: ParserConfig,
children: Sequence[Any]) -> Any:
if len(children) == 1:
(dotted_name, ) = children
return ImportAlias(name=Name(dotted_name.string), asname=None)
else:
dotted_name, astoken, name = children
return ImportAlias(
name=Name(dotted_name.string),
asname=AsName(
whitespace_before_as=parse_simple_whitespace(
config, astoken.whitespace_before),
whitespace_after_as=parse_simple_whitespace(
config, astoken.whitespace_after),
name=Name(name.string),
),
)
def convert_if_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
if_tok, test, colon_tok, suite, *tail = children
if len(tail) > 0:
(orelse,) = tail
else:
orelse = None
return If(
leading_lines=parse_empty_lines(config, if_tok.whitespace_before),
whitespace_before_test=parse_simple_whitespace(config, if_tok.whitespace_after),
test=test.value,
whitespace_after_test=parse_simple_whitespace(
config, colon_tok.whitespace_before
),
body=suite,
orelse=orelse,
)
def _construct_nameitems(config: ParserConfig,
names: Sequence[Any]) -> List[NameItem]:
nameitems: List[NameItem] = []
for name, maybe_comma in grouper(names, 2):
if maybe_comma is None:
nameitems.append(NameItem(Name(name.string)))
else:
nameitems.append(
NameItem(
Name(name.string),
comma=Comma(
whitespace_before=parse_simple_whitespace(
config, maybe_comma.whitespace_before),
whitespace_after=parse_simple_whitespace(
config, maybe_comma.whitespace_after),
),
))
return nameitems
def convert_for_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
(
for_token,
expr,
in_token,
test,
for_colon_token,
for_suite,
*else_block,
) = children
if len(else_block) > 0:
(else_token, else_colon_token, else_suite) = else_block
orelse = Else(
leading_lines=parse_empty_lines(config, else_token.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, else_colon_token.whitespace_before
),
body=else_suite,
)
else:
orelse = None
return WithLeadingWhitespace(
For(
whitespace_after_for=parse_simple_whitespace(
config, for_token.whitespace_after
),
target=expr.value,
whitespace_before_in=parse_simple_whitespace(
config, in_token.whitespace_before
),
whitespace_after_in=parse_simple_whitespace(
config, in_token.whitespace_after
),
iter=test.value,
whitespace_before_colon=parse_simple_whitespace(
config, for_colon_token.whitespace_before
),
body=for_suite,
orelse=orelse,
),
for_token.whitespace_before,
)
def convert_import_name(config: ParserConfig, children: Sequence[Any]) -> Any:
importtoken, names = children
return WithLeadingWhitespace(
Import(
names=names.names,
whitespace_after_import=parse_simple_whitespace(
config, importtoken.whitespace_after),
),
importtoken.whitespace_before,
)
def convert_simple_stmt_suite(config: ParserConfig, children: Sequence[Any]) -> Any:
"""
This function is similar to convert_simple_stmt_line, but yields a different type
"""
(partial,) = children
return SimpleStatementSuite(
partial.body,
leading_whitespace=parse_simple_whitespace(config, partial.whitespace_before),
trailing_whitespace=partial.trailing_whitespace,
)
def convert_global_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
(global_token, *names) = children
return WithLeadingWhitespace(
Global(
names=tuple(_construct_nameitems(config, names)),
whitespace_after_global=parse_simple_whitespace(
config, names[0].whitespace_before),
),
global_token.whitespace_before,
)
def convert_del_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
(del_name, exprlist) = children
return WithLeadingWhitespace(
Del(
target=exprlist.value,
whitespace_after_del=parse_simple_whitespace(
config, del_name.whitespace_after),
),
del_name.whitespace_before,
)
def _extract_async(
config: ParserConfig, children: Sequence[Any]
) -> Tuple[List[EmptyLine], Optional[Asynchronous], Any]:
if len(children) == 1:
(stmt, ) = children
whitespace_before = stmt.whitespace_before
asyncnode = None
else:
asynctoken, stmt = children
whitespace_before = asynctoken.whitespace_before
asyncnode = Asynchronous(whitespace_after=parse_simple_whitespace(
config, asynctoken.whitespace_after))
return (parse_empty_lines(config,
whitespace_before), asyncnode, stmt.value)
def convert_return_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
if len(children) == 1:
(keyword, ) = children
return WithLeadingWhitespace(
Return(whitespace_after_return=SimpleWhitespace("")),
keyword.whitespace_before,
)
else:
(keyword, testlist) = children
return WithLeadingWhitespace(
Return(
value=testlist.value,
whitespace_after_return=parse_simple_whitespace(
config, keyword.whitespace_after),
),
keyword.whitespace_before,
)
