def convert_test(
config: ParserConfig, children: typing.Sequence[typing.Any]
) -> typing.Any:
if len(children) == 1:
(child,) = children
return child
else:
(body, if_token, test, else_token, orelse) = children
return WithLeadingWhitespace(
IfExp(
body=body.value,
test=test.value,
orelse=orelse.value,
whitespace_before_if=parse_parenthesizable_whitespace(
config, if_token.whitespace_before
),
whitespace_after_if=parse_parenthesizable_whitespace(
config, if_token.whitespace_after
),
whitespace_before_else=parse_parenthesizable_whitespace(
config, else_token.whitespace_before
),
whitespace_after_else=parse_parenthesizable_whitespace(
config, else_token.whitespace_after
),
),
body.whitespace_before,
)
def convert_factor(
config: ParserConfig, children: typing.Sequence[typing.Any]
) -> typing.Any:
if len(children) == 1:
(child,) = children
return child
op, factor = children
# First, tokenize the unary operator
if op.string == "+":
opnode = Plus(
whitespace_after=parse_parenthesizable_whitespace(
config, op.whitespace_after
)
)
elif op.string == "-":
opnode = Minus(
whitespace_after=parse_parenthesizable_whitespace(
config, op.whitespace_after
)
)
elif op.string == "~":
opnode = BitInvert(
whitespace_after=parse_parenthesizable_whitespace(
config, op.whitespace_after
)
)
else:
raise Exception(f"Unexpected token '{op.string}'!")
return WithLeadingWhitespace(
UnaryOperation(operator=opnode, expression=factor.value), op.whitespace_before
)
def convert_sync_comp_for(
config: ParserConfig, children: typing.Sequence[typing.Any]
) -> typing.Any:
# unpack
for_tok, target, in_tok, iter, *trailing = children
if len(trailing) and isinstance(trailing[-1], CompFor):
*ifs, inner_for_in = trailing
else:
ifs, inner_for_in = trailing, None
return CompFor(
target=target.value,
iter=iter.value,
ifs=ifs,
inner_for_in=inner_for_in,
whitespace_before=parse_parenthesizable_whitespace(
config, for_tok.whitespace_before
),
whitespace_after_for=parse_parenthesizable_whitespace(
config, for_tok.whitespace_after
),
whitespace_before_in=parse_parenthesizable_whitespace(
config, in_tok.whitespace_before
),
whitespace_after_in=parse_parenthesizable_whitespace(
config, in_tok.whitespace_after
),
)
def convert_subscriptlist(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
if len(children) > 1:
# This is a list of ExtSlice, so construct as such by grouping every
# subscript with an optional comma and adding to a list.
extslices = []
for slice, comma in grouper(children, 2):
if comma is None:
extslices.append(ExtSlice(slice=slice.value))
else:
extslices.append(
ExtSlice(
slice=slice.value,
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
),
))
return WithLeadingWhitespace(extslices, children[0].whitespace_before)
else:
# This is an Index or Slice, as parsed in the child.
(index_or_slice, ) = children
return index_or_slice
def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
(with_token, *items, colon_token, suite) = children
item_nodes: List[WithItem] = []
for with_item, maybe_comma in grouper(items, 2):
if maybe_comma is not None:
item_nodes.append(
with_item.with_changes(
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, maybe_comma.whitespace_before
),
whitespace_after=parse_parenthesizable_whitespace(
config, maybe_comma.whitespace_after
),
)
)
)
else:
item_nodes.append(with_item)
return WithLeadingWhitespace(
With(
whitespace_after_with=parse_simple_whitespace(
config, with_token.whitespace_after
),
items=tuple(item_nodes),
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
body=suite,
),
with_token.whitespace_before,
)
def convert_binop(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
leftexpr, *rightexprs = children
if len(rightexprs) == 0:
return leftexpr
whitespace_before = leftexpr.whitespace_before
leftexpr = leftexpr.value
# Convert all of the operations that have no precedence in a loop
for op, rightexpr in grouper(rightexprs, 2):
if op.string not in BINOP_TOKEN_LUT:
raise Exception(f"Unexpected token '{op.string}'!")
leftexpr = BinaryOperation(
left=leftexpr,
# pyre-ignore Pyre thinks that the type of the LUT is CSTNode.
operator=BINOP_TOKEN_LUT[op.string](
whitespace_before=parse_parenthesizable_whitespace(
config, op.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, op.whitespace_after),
),
right=rightexpr.value,
)
return WithLeadingWhitespace(leftexpr, whitespace_before)
def convert_arg_assign_comp_for(
config: ParserConfig, children: typing.Sequence[typing.Any]
) -> typing.Any:
if len(children) == 1:
# Simple test
(child,) = children
return Arg(value=child.value)
elif len(children) == 2:
elt, for_in = children
return Arg(value=GeneratorExp(elt.value, for_in, lpar=(), rpar=()))
else:
# "key = value" assignment argument
lhs, equal, rhs = children
return Arg(
keyword=lhs.value,
equal=AssignEqual(
whitespace_before=parse_parenthesizable_whitespace(
config, equal.whitespace_before
),
whitespace_after=parse_parenthesizable_whitespace(
config, equal.whitespace_after
),
),
value=rhs.value,
)
def convert_classdef(config: ParserConfig, children: Sequence[Any]) -> Any:
classdef, name, *arglist, colon, suite = children
# First, parse out the comments and empty lines before the statement.
leading_lines = parse_empty_lines(config, classdef.whitespace_before)
# Compute common whitespace and nodes
whitespace_after_class = parse_simple_whitespace(config,
classdef.whitespace_after)
namenode = Name(name.string)
whitespace_after_name = parse_simple_whitespace(config,
name.whitespace_after)
# Now, construct the classdef node itself
if not arglist:
# No arglist, so no arguments to this class
return ClassDef(
leading_lines=leading_lines,
lines_after_decorators=(),
whitespace_after_class=whitespace_after_class,
name=namenode,
whitespace_after_name=whitespace_after_name,
body=suite,
)
else:
# Unwrap arglist partial, because its valid to not have any
lpar, *args, rpar = arglist
args = args[0].args if args else []
bases: List[Arg] = []
keywords: List[Arg] = []
current_arg = bases
for arg in args:
if arg.star == "**" or arg.keyword is not None:
current_arg = keywords
# Some quick validation
if current_arg is keywords and (arg.star == "*" or
(arg.star == ""
and arg.keyword is None)):
raise PartialParserSyntaxError(
"Positional argument follows keyword argument.")
current_arg.append(arg)
return ClassDef(
leading_lines=leading_lines,
lines_after_decorators=(),
whitespace_after_class=whitespace_after_class,
name=namenode,
whitespace_after_name=whitespace_after_name,
lpar=LeftParen(whitespace_after=parse_parenthesizable_whitespace(
config, lpar.whitespace_after)),
bases=bases,
keywords=keywords,
rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace(
config, rpar.whitespace_before)),
whitespace_before_colon=parse_simple_whitespace(
config, colon.whitespace_before),
body=suite,
)
def convert_decorator(config: ParserConfig, children: Sequence[Any]) -> Any:
atsign, name, *arglist, newline = children
if not arglist:
# This is either a name or an attribute node, so just extract it.
decoratornode = name
else:
# This needs to be converted into a call node, and we have the
# arglist partial.
lpar, *args, rpar = arglist
args = args[0].args if args else []
# If the trailing argument doesn't have a comma, then it owns the
# trailing whitespace before the rpar. Otherwise, the comma owns
# it.
if len(args) > 0 and args[-1].comma == MaybeSentinel.DEFAULT:
args[-1] = args[-1].with_changes(
whitespace_after_arg=parse_parenthesizable_whitespace(
config, rpar.whitespace_before))
decoratornode = Call(
func=name,
whitespace_after_func=parse_simple_whitespace(
config, lpar.whitespace_before),
whitespace_before_args=parse_parenthesizable_whitespace(
config, lpar.whitespace_after),
args=tuple(args),
)
return Decorator(
leading_lines=parse_empty_lines(config, atsign.whitespace_before),
whitespace_after_at=parse_simple_whitespace(config,
atsign.whitespace_after),
decorator=decoratornode,
trailing_whitespace=newline,
)
def convert_atom_parens(
config: ParserConfig, children: typing.Sequence[typing.Any]
) -> typing.Any:
lpar_tok, *atoms, rpar_tok = children
lpar = LeftParen(
whitespace_after=parse_parenthesizable_whitespace(
config, lpar_tok.whitespace_after
)
)
rpar = RightParen(
whitespace_before=parse_parenthesizable_whitespace(
config, rpar_tok.whitespace_before
)
)
if len(atoms) == 1:
# inner_atom is a _BaseParenthesizedNode
inner_atom = atoms[0].value
return WithLeadingWhitespace(
inner_atom.with_changes(
lpar=(lpar, *inner_atom.lpar), rpar=(*inner_atom.rpar, rpar)
),
lpar_tok.whitespace_before,
)
else:
return WithLeadingWhitespace(
Tuple((), lpar=(lpar,), rpar=(rpar,)), lpar_tok.whitespace_before
)
def convert_fstring_expr(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
openbrkt, testlist, *conversions, closebrkt = children
# Extract any optional conversion
if len(conversions) > 0 and isinstance(conversions[0],
FormattedStringConversionPartial):
conversion = conversions[0].value
conversions = conversions[1:]
else:
conversion = None
# Extract any optional format spec
if len(conversions) > 0:
format_spec = conversions[0].values
else:
format_spec = None
return FormattedStringExpression(
whitespace_before_expression=parse_parenthesizable_whitespace(
config, testlist.whitespace_before),
expression=testlist.value,
whitespace_after_expression=parse_parenthesizable_whitespace(
config, children[2].whitespace_before),
conversion=conversion,
format_spec=format_spec,
)
def convert_parameters(config: ParserConfig, children: Sequence[Any]) -> Any:
lpar, *paramlist, rpar = children
return FuncdefPartial(
lpar=LeftParen(whitespace_after=parse_parenthesizable_whitespace(
config, lpar.whitespace_after)),
params=Parameters() if not paramlist else paramlist[0],
rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace(
config, rpar.whitespace_before)),
)
def convert_subscript(
config: ParserConfig, children: typing.Sequence[typing.Any]
) -> typing.Any:
if len(children) == 1 and not isinstance(children[0], Token):
# This is just an index node
(test,) = children
return WithLeadingWhitespace(Index(test.value), test.whitespace_before)
if isinstance(children[-1], SlicePartial):
# We got a partial slice as the final param. Extract the final
# bits of the full subscript.
*others, sliceop = children
whitespace_before = others[0].whitespace_before
second_colon = sliceop.second_colon
step = sliceop.step
else:
# We can just parse this below, without taking extras from the
# partial child.
others = children
whitespace_before = others[0].whitespace_before
second_colon = MaybeSentinel.DEFAULT
step = None
# We need to create a partial slice to pass up. So, align so we have
# a list that's always [Optional[Test], Colon, Optional[Test]].
if isinstance(others[0], Token):
# First token is a colon, so insert an empty test on the LHS. We
# know the RHS is a test since it's not a sliceop.
slicechildren = [None, *others]
else:
# First token is non-colon, so its a test.
slicechildren = [*others]
if len(slicechildren) < 3:
# Now, we have to fill in the RHS. We know its two long
# at this point if its not already 3.
slicechildren = [*slicechildren, None]
lower, first_colon, upper = slicechildren
return WithLeadingWhitespace(
Slice(
lower=lower.value if lower is not None else None,
first_colon=Colon(
whitespace_before=parse_parenthesizable_whitespace(
config, first_colon.whitespace_before
),
whitespace_after=parse_parenthesizable_whitespace(
config, first_colon.whitespace_after
),
),
upper=upper.value if upper is not None else None,
second_colon=second_colon,
step=step,
),
whitespace_before=whitespace_before,
)
def convert_funcdef_annotation(config: ParserConfig,
children: Sequence[Any]) -> Any:
arrow, typehint = children
return Annotation(
whitespace_before_indicator=parse_parenthesizable_whitespace(
config, arrow.whitespace_before),
whitespace_after_indicator=parse_parenthesizable_whitespace(
config, arrow.whitespace_after),
annotation=typehint.value,
)
def convert_comp_if(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
if_tok, test = children
return CompIf(
test.value,
whitespace_before=parse_parenthesizable_whitespace(
config, if_tok.whitespace_before),
whitespace_before_test=parse_parenthesizable_whitespace(
config, test.whitespace_before),
)
def convert_import_from(config: ParserConfig, children: Sequence[Any]) -> Any:
fromtoken, import_relative, importtoken, *importlist = children
if len(importlist) == 1:
(possible_star, ) = importlist
if isinstance(possible_star, Token):
# Its a "*" import, so we must construct this node.
names = ImportStar()
else:
# Its an import as names partial, grab the names from that.
names = possible_star.names
lpar = None
rpar = None
else:
# Its an import as names partial with parens
lpartoken, namespartial, rpartoken = importlist
lpar = LeftParen(whitespace_after=parse_parenthesizable_whitespace(
config, lpartoken.whitespace_after))
names = namespartial.names
rpar = RightParen(whitespace_before=parse_parenthesizable_whitespace(
config, rpartoken.whitespace_before))
# If we have a relative-only import, then we need to relocate the space
# after the final dot to be owned by the import token.
if len(import_relative.relative) > 0 and import_relative.module is None:
whitespace_before_import = import_relative.relative[
-1].whitespace_after
relative = (
*import_relative.relative[:-1],
import_relative.relative[-1].with_changes(
whitespace_after=SimpleWhitespace("")),
)
else:
whitespace_before_import = parse_simple_whitespace(
config, importtoken.whitespace_before)
relative = import_relative.relative
return WithLeadingWhitespace(
ImportFrom(
whitespace_after_from=parse_simple_whitespace(
config, fromtoken.whitespace_after),
relative=relative,
module=import_relative.module,
whitespace_before_import=whitespace_before_import,
whitespace_after_import=parse_simple_whitespace(
config, importtoken.whitespace_after),
lpar=lpar,
names=names,
rpar=rpar,
),
fromtoken.whitespace_before,
)
def convert_trailer_attribute(
config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
dot, name = children
return AttributePartial(
dot=Dot(
whitespace_before=parse_parenthesizable_whitespace(
config, dot.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, dot.whitespace_after),
),
attr=Name(name.string),
)
def convert_comp_op(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
if len(children) == 1:
(op, ) = children
if op.string in COMPOP_TOKEN_LUT:
# A regular comparison containing one token
# pyre-ignore Pyre thinks that the type of the LUT is CSTNode.
return COMPOP_TOKEN_LUT[op.string](
whitespace_before=parse_parenthesizable_whitespace(
config, op.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, op.whitespace_after),
)
elif op.string in ["!=", "<>"]:
# Not equal, which can take two forms in some cases
return NotEqual(
whitespace_before=parse_parenthesizable_whitespace(
config, op.whitespace_before),
value=op.string,
whitespace_after=parse_parenthesizable_whitespace(
config, op.whitespace_after),
)
else:
# this should be unreachable
raise Exception(f"Unexpected token '{op.string}'!")
else:
# A two-token comparison
leftcomp, rightcomp = children
if leftcomp.string == "not" and rightcomp.string == "in":
return NotIn(
whitespace_before=parse_parenthesizable_whitespace(
config, leftcomp.whitespace_before),
whitespace_between=parse_parenthesizable_whitespace(
config, leftcomp.whitespace_after),
whitespace_after=parse_parenthesizable_whitespace(
config, rightcomp.whitespace_after),
)
elif leftcomp.string == "is" and rightcomp.string == "not":
return IsNot(
whitespace_before=parse_parenthesizable_whitespace(
config, leftcomp.whitespace_before),
whitespace_between=parse_parenthesizable_whitespace(
config, leftcomp.whitespace_after),
whitespace_after=parse_parenthesizable_whitespace(
config, rightcomp.whitespace_after),
)
else:
# this should be unreachable
raise Exception(
f"Unexpected token '{leftcomp.string} {rightcomp.string}'!")
def _convert_dict_element(
config: ParserConfig,
children_iter: typing.Iterator[typing.Any],
last_child: typing.Any,
) -> typing.Union[DictElement, StarredDictElement]:
first = next(children_iter)
if isinstance(first, Token) and first.string == "**":
expr = next(children_iter)
element = StarredDictElement(
expr.value,
whitespace_before_value=parse_parenthesizable_whitespace(
config, expr.whitespace_before
),
)
else:
key = first
colon_tok = next(children_iter)
value = next(children_iter)
element = DictElement(
key.value,
value.value,
whitespace_before_colon=parse_parenthesizable_whitespace(
config, colon_tok.whitespace_before
),
whitespace_after_colon=parse_parenthesizable_whitespace(
config, colon_tok.whitespace_after
),
)
# Handle the trailing comma (if there is one)
try:
comma_token = next(children_iter)
element = element.with_changes(
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma_token.whitespace_before
),
# Only compute whitespace_after if we're not a trailing comma.
# If we're a trailing comma, that whitespace should be consumed by the
# RightBracket.
whitespace_after=(
parse_parenthesizable_whitespace(
config, comma_token.whitespace_after
)
if comma_token is not last_child
else SimpleWhitespace("")
),
)
)
except StopIteration:
pass
return element
def convert_atom_expr_trailer(
config: ParserConfig, children: typing.Sequence[typing.Any]
) -> typing.Any:
atom, *trailers = children
whitespace_before = atom.whitespace_before
atom = atom.value
# Need to walk through all trailers from left to right and construct
# a series of nodes based on each partial type. We can't do this with
# left recursion due to limits in the parser.
for trailer in trailers:
if isinstance(trailer, SubscriptPartial):
atom = Subscript(
value=atom,
whitespace_after_value=parse_parenthesizable_whitespace(
config, trailer.whitespace_before
),
lbracket=trailer.lbracket,
slice=trailer.slice,
rbracket=trailer.rbracket,
)
elif isinstance(trailer, AttributePartial):
atom = Attribute(value=atom, dot=trailer.dot, attr=trailer.attr)
elif isinstance(trailer, CallPartial):
# If the trailing argument doesn't have a comma, then it owns the
# trailing whitespace before the rpar. Otherwise, the comma owns
# it.
if (
len(trailer.args) > 0
and trailer.args[-1].comma == MaybeSentinel.DEFAULT
):
args = (
*trailer.args[:-1],
trailer.args[-1].with_changes(
whitespace_after_arg=trailer.rpar.whitespace_before
),
)
else:
args = trailer.args
atom = Call(
func=atom,
whitespace_after_func=parse_parenthesizable_whitespace(
config, trailer.lpar.whitespace_before
),
whitespace_before_args=trailer.lpar.value.whitespace_after,
args=tuple(args),
)
else:
# This is an invalid trailer, so lets give up
raise Exception("Logic error!")
return WithLeadingWhitespace(atom, whitespace_before)
def convert_trailer_arglist(
config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
lpar, *arglist, rpar = children
return CallPartial(
lpar=WithLeadingWhitespace(
LeftParen(whitespace_after=parse_parenthesizable_whitespace(
config, lpar.whitespace_after)),
lpar.whitespace_before,
),
args=() if not arglist else arglist[0].args,
rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace(
config, rpar.whitespace_before)),
)
def convert_trailer_subscriptlist(
config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
(lbracket, subscriptlist, rbracket) = children
return SubscriptPartial(
lbracket=LeftSquareBracket(
whitespace_after=parse_parenthesizable_whitespace(
config, lbracket.whitespace_after)),
slice=subscriptlist.value,
rbracket=RightSquareBracket(
whitespace_before=parse_parenthesizable_whitespace(
config, rbracket.whitespace_before)),
whitespace_before=lbracket.whitespace_before,
)
def convert_fpdef_assign(config: ParserConfig, children: Sequence[Any]) -> Any:
if len(children) == 1:
(child, ) = children
return child
param, equal, default = children
return param.with_changes(
equal=AssignEqual(
whitespace_before=parse_parenthesizable_whitespace(
config, equal.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, equal.whitespace_after),
),
default=default.value,
)
def convert_arglist(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
args = []
for argument, comma in grouper(children, 2):
if comma is None:
args.append(argument)
else:
args.append(
argument.with_changes(comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
)))
return ArglistPartial(args)
def _gather_import_names(config: ParserConfig,
children: Sequence[Any]) -> ImportPartial:
names = []
for name, comma in grouper(children, 2):
if comma is None:
names.append(name)
else:
names.append(
name.with_changes(comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
)))
return ImportPartial(names=names)
def _convert_dict_comp(config,
children: typing.Sequence[typing.Any]) -> typing.Any:
key, colon_token, value, comp_for = children
return WithLeadingWhitespace(
DictComp(
key.value,
value.value,
comp_for,
# lbrace, rbrace, lpar, and rpar will be attached as-needed by the atom grammar
whitespace_before_colon=parse_parenthesizable_whitespace(
config, colon_token.whitespace_before),
whitespace_after_colon=parse_parenthesizable_whitespace(
config, colon_token.whitespace_after),
),
key.whitespace_before,
)
def convert_fpdef_starstar(config: ParserConfig,
children: Sequence[Any]) -> Any:
starstar, param = children
return param.with_changes(
star=starstar.string,
whitespace_after_star=parse_parenthesizable_whitespace(
config, starstar.whitespace_after),
)
def convert_dotted_name(config: ParserConfig, children: Sequence[Any]) -> Any:
left, *rest = children
node = Name(left.string)
for dot, right in grouper(rest, 2):
node = Attribute(
value=node,
dot=Dot(
whitespace_before=parse_parenthesizable_whitespace(
config, dot.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, dot.whitespace_after),
),
attr=Name(right.string),
)
return node
def convert_dotted_as_name(config: ParserConfig,
children: Sequence[Any]) -> Any:
if len(children) == 1:
(dotted_name, ) = children
return ImportAlias(name=dotted_name, asname=None)
else:
dotted_name, astoken, name = children
return ImportAlias(
name=dotted_name,
asname=AsName(
whitespace_before_as=parse_parenthesizable_whitespace(
config, astoken.whitespace_before),
whitespace_after_as=parse_parenthesizable_whitespace(
config, astoken.whitespace_after),
name=Name(name.string),
),
)
def convert_sliceop(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
if len(children) == 2:
colon, test = children
step = test.value
else:
(colon, ) = children
step = None
return SlicePartial(
second_colon=Colon(
whitespace_before=parse_parenthesizable_whitespace(
config, colon.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, colon.whitespace_after),
),
step=step,
)