def convert_subscriptlist(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
if len(children) > 1:
# This is a list of ExtSlice, so construct as such by grouping every
# subscript with an optional comma and adding to a list.
extslices = []
for slice, comma in grouper(children, 2):
if comma is None:
extslices.append(ExtSlice(slice=slice.value))
else:
extslices.append(
ExtSlice(
slice=slice.value,
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
),
))
return WithLeadingWhitespace(extslices, children[0].whitespace_before)
else:
# This is an Index or Slice, as parsed in the child.
(index_or_slice, ) = children
return index_or_slice
def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
(with_token, *items, colon_token, suite) = children
item_nodes: List[WithItem] = []
for with_item, maybe_comma in grouper(items, 2):
if maybe_comma is not None:
item_nodes.append(
with_item.with_changes(
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, maybe_comma.whitespace_before
),
whitespace_after=parse_parenthesizable_whitespace(
config, maybe_comma.whitespace_after
),
)
)
)
else:
item_nodes.append(with_item)
return WithLeadingWhitespace(
With(
whitespace_after_with=parse_simple_whitespace(
config, with_token.whitespace_after
),
items=tuple(item_nodes),
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
body=suite,
),
with_token.whitespace_before,
)
def convert_simple_stmt_partial(config: ParserConfig, children: Sequence[Any]) -> Any:
*statements, trailing_whitespace = children
last_stmt = len(statements) / 2
body = []
for i, (stmt_body, semi) in enumerate(grouper(statements, 2)):
if semi is not None:
if i == (last_stmt - 1):
# Trailing semicolons only own the whitespace before.
semi = Semicolon(
whitespace_before=parse_simple_whitespace(
config, semi.whitespace_before
),
whitespace_after=SimpleWhitespace(""),
)
else:
# Middle semicolons own the whitespace before and after.
semi = Semicolon(
whitespace_before=parse_simple_whitespace(
config, semi.whitespace_before
),
whitespace_after=parse_simple_whitespace(
config, semi.whitespace_after
),
)
else:
semi = MaybeSentinel.DEFAULT
body.append(stmt_body.value.with_changes(semicolon=semi))
return SimpleStatementPartial(
body,
whitespace_before=statements[0].whitespace_before,
trailing_whitespace=trailing_whitespace,
)
def convert_binop(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
leftexpr, *rightexprs = children
if len(rightexprs) == 0:
return leftexpr
whitespace_before = leftexpr.whitespace_before
leftexpr = leftexpr.value
# Convert all of the operations that have no precedence in a loop
for op, rightexpr in grouper(rightexprs, 2):
if op.string not in BINOP_TOKEN_LUT:
raise Exception(f"Unexpected token '{op.string}'!")
leftexpr = BinaryOperation(
left=leftexpr,
# pyre-ignore Pyre thinks that the type of the LUT is CSTNode.
operator=BINOP_TOKEN_LUT[op.string](
whitespace_before=parse_parenthesizable_whitespace(
config, op.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, op.whitespace_after),
),
right=rightexpr.value,
)
return WithLeadingWhitespace(leftexpr, whitespace_before)
def convert_try_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
trytoken, try_colon_token, try_suite, *rest = children
handlers: List[ExceptHandler] = []
orelse: Optional[Else] = None
finalbody: Optional[Finally] = None
for clause, colon_token, suite in grouper(rest, 3):
if isinstance(clause, Token):
if clause.string == "else":
if orelse is not None:
raise Exception("Logic error!")
orelse = Else(
leading_lines=parse_empty_lines(config, clause.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
body=suite,
)
elif clause.string == "finally":
if finalbody is not None:
raise Exception("Logic error!")
finalbody = Finally(
leading_lines=parse_empty_lines(config, clause.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
body=suite,
)
else:
raise Exception("Logic error!")
elif isinstance(clause, ExceptClausePartial):
handlers.append(
ExceptHandler(
body=suite,
type=clause.type,
name=clause.name,
leading_lines=clause.leading_lines,
whitespace_after_except=clause.whitespace_after_except,
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
)
)
else:
raise Exception("Logic error!")
return Try(
leading_lines=parse_empty_lines(config, trytoken.whitespace_before),
whitespace_before_colon=parse_simple_whitespace(
config, try_colon_token.whitespace_before
),
body=try_suite,
handlers=tuple(handlers),
orelse=orelse,
finalbody=finalbody,
)
def convert_arglist(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
args = []
for argument, comma in grouper(children, 2):
if comma is None:
args.append(argument)
else:
args.append(
argument.with_changes(comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
)))
return ArglistPartial(args)
def _gather_import_names(config: ParserConfig,
children: Sequence[Any]) -> ImportPartial:
names = []
for name, comma in grouper(children, 2):
if comma is None:
names.append(name)
else:
names.append(
name.with_changes(comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
)))
return ImportPartial(names=names)
def convert_dotted_name(config: ParserConfig, children: Sequence[Any]) -> Any:
left, *rest = children
node = Name(left.string)
for dot, right in grouper(rest, 2):
node = Attribute(
value=node,
dot=Dot(
whitespace_before=parse_parenthesizable_whitespace(
config, dot.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, dot.whitespace_after),
),
attr=Name(right.string),
)
return node
def convert_comparison(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
if len(children) == 1:
(child, ) = children
return child
lhs, *rest = children
comparisons: typing.List[ComparisonTarget] = []
for operator, comparator in grouper(rest, 2):
comparisons.append(
ComparisonTarget(operator=operator, comparator=comparator.value))
return WithLeadingWhitespace(
Comparison(left=lhs.value, comparisons=tuple(comparisons)),
lhs.whitespace_before,
)
def _convert_sequencelike(
config: ParserConfig,
children: typing.Sequence[typing.Any],
single_child_is_sequence: bool,
sequence_type: typing.Union[
typing.Type[Tuple], typing.Type[List], typing.Type[Set]
],
) -> typing.Any:
if not single_child_is_sequence and len(children) == 1:
return children[0]
# N.B. The parent node (e.g. atom) is responsible for computing and attaching
# whitespace information on any parenthesis, square brackets, or curly braces
elements = []
for wrapped_expr_or_starred_element, comma_token in grouper(children, 2):
expr_or_starred_element = wrapped_expr_or_starred_element.value
if comma_token is None:
comma = MaybeSentinel.DEFAULT
else:
comma = Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma_token.whitespace_before
),
# Only compute whitespace_after if we're not a trailing comma.
# If we're a trailing comma, that whitespace should be consumed by the
# TrailingWhitespace, parenthesis, etc.
whitespace_after=(
parse_parenthesizable_whitespace(
config, comma_token.whitespace_after
)
if comma_token is not children[-1]
else SimpleWhitespace("")
),
)
if isinstance(expr_or_starred_element, StarredElement):
starred_element = expr_or_starred_element
elements.append(starred_element.with_changes(comma=comma))
else:
expr = expr_or_starred_element
elements.append(Element(value=expr, comma=comma))
# lpar/rpar are the responsibility of our parent
return WithLeadingWhitespace(
sequence_type(elements, lpar=(), rpar=()), children[0].whitespace_before
)
def _construct_nameitems(config: ParserConfig,
names: Sequence[Any]) -> List[NameItem]:
nameitems: List[NameItem] = []
for name, maybe_comma in grouper(names, 2):
if maybe_comma is None:
nameitems.append(NameItem(Name(name.string)))
else:
nameitems.append(
NameItem(
Name(name.string),
comma=Comma(
whitespace_before=parse_simple_whitespace(
config, maybe_comma.whitespace_before),
whitespace_after=parse_simple_whitespace(
config, maybe_comma.whitespace_after),
),
))
return nameitems
def convert_subscriptlist(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
# This is a list of SubscriptElement, so construct as such by grouping every
# subscript with an optional comma and adding to a list.
elements = []
for slice, comma in grouper(children, 2):
if comma is None:
elements.append(SubscriptElement(slice=slice.value))
else:
elements.append(
SubscriptElement(
slice=slice.value,
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
),
))
return WithLeadingWhitespace(elements, children[0].whitespace_before)
def convert_argslist(config: ParserConfig, children: Sequence[Any]) -> Any:
params: List[Param] = []
default_params: List[Param] = []
star_arg: Union[Param, ParamStar, MaybeSentinel] = MaybeSentinel.DEFAULT
kwonly_params: List[Param] = []
star_kwarg: Optional[Param] = None
def add_param(current_param: Optional[List[Param]],
param: Union[Param, ParamStar]) -> Optional[List[Param]]:
nonlocal star_arg
nonlocal star_kwarg
if isinstance(param, ParamStar):
# Only can add this if we don't already have a "*" or a "*param".
if current_param in [params, default_params]:
star_arg = param
current_param = kwonly_params
else:
# Example code:
# def fn(*abc, *): ...
# This should be unreachable, the grammar already disallows it.
raise Exception(
"Cannot have multiple star ('*') markers in a single argument "
+ "list.")
# pyre-ignore Pyre seems to think param.star.__eq__ is not callable
elif isinstance(param.star,
str) and param.star == "" and param.default is None:
# Can only add this if we're in the params or kwonly_params section
if current_param is params:
params.append(param)
elif current_param is kwonly_params:
kwonly_params.append(param)
else:
# Example code:
# def fn(first=None, second): ...
# This code is reachable, so we should use a PartialParserSyntaxError.
raise PartialParserSyntaxError(
"Cannot have a non-default argument following a default argument."
)
elif (isinstance(param.star, str)
# pyre-ignore Pyre seems to think param.star.__eq__ is not callable
and param.star == "" and param.default is not None):
if current_param is params:
current_param = default_params
# Can only add this if we're not yet at star args.
if current_param is default_params:
default_params.append(param)
elif current_param is kwonly_params:
kwonly_params.append(param)
else:
# Example code:
# def fn(**kwargs, trailing=None)
# This should be unreachable, the grammar already disallows it.
raise Exception(
"Cannot have any arguments after a kwargs expansion.")
elif (isinstance(param.star, str)
# pyre-ignore Pyre seems to think param.star.__eq__ is not callable
and param.star == "*" and param.default is None):
# Can only add this if we're in params/default_params, since
# we only allow one of "*" or "*param".
if current_param in [params, default_params]:
star_arg = param
current_param = kwonly_params
else:
# Example code:
# def fn(*first, *second): ...
# This should be unreachable, the grammar already disallows it.
raise Exception(
"Expected a keyword argument but found a starred positional "
+ "argument expansion.")
elif (isinstance(param.star, str)
# pyre-ignore Pyre seems to think param.star.__eq__ is not callable
and param.star == "**" and param.default is None):
# Can add this in all cases where we don't have a star_kwarg
# yet.
if current_param is not None:
star_kwarg = param
current_param = None
else:
# Example code:
# def fn(**first, **second)
# This should be unreachable, the grammar already disallows it.
raise Exception(
"Multiple starred keyword argument expansions are not allowed in a "
+ "single argument list")
else:
# The state machine should never end up here.
raise Exception("Logic error!")
return current_param
# The parameter list we are adding to
current: Optional[List[Param]] = params
# We should have every other item in the group as a param or a comma by now,
# so split them up, add commas and then put them in the appropriate group.
for parameter, comma in grouper(children, 2):
if comma is None:
if isinstance(parameter, ParamStarPartial):
# Example:
# def fn(abc, *): ...
#
# There's also the case where we have bare * with a trailing comma.
# That's handled later.
#
# It's not valid to construct a ParamStar object without a comma, so we
# need to catch the non-comma case separately.
raise PartialParserSyntaxError(
"Named (keyword) arguments must follow a bare *.")
else:
current = add_param(current, parameter)
else:
comma = Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
)
if isinstance(parameter, ParamStarPartial):
current = add_param(current, ParamStar(comma=comma))
else:
current = add_param(current,
parameter.with_changes(comma=comma))
if isinstance(star_arg, ParamStar) and len(kwonly_params) == 0:
# Example:
# def fn(abc, *,): ...
#
# This will raise a validation error, but we want to make sure to raise a syntax
# error instead.
#
# The case where there's no trailing comma is already handled by this point, so
# this conditional is only for the case where we have a trailing comma.
raise PartialParserSyntaxError(
"Named (keyword) arguments must follow a bare *.")
return Parameters(
params=tuple(params),
default_params=tuple(default_params),
star_arg=star_arg,
kwonly_params=tuple(kwonly_params),
star_kwarg=star_kwarg,
)
