def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
(with_token, *items, colon_token, suite) = children
item_nodes: List[WithItem] = []
for with_item, maybe_comma in grouper(items, 2):
if maybe_comma is not None:
item_nodes.append(
with_item.with_changes(
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, maybe_comma.whitespace_before
),
whitespace_after=parse_parenthesizable_whitespace(
config, maybe_comma.whitespace_after
),
)
)
)
else:
item_nodes.append(with_item)
return WithLeadingWhitespace(
With(
whitespace_after_with=parse_simple_whitespace(
config, with_token.whitespace_after
),
items=tuple(item_nodes),
whitespace_before_colon=parse_simple_whitespace(
config, colon_token.whitespace_before
),
body=suite,
),
with_token.whitespace_before,
)
def convert_subscriptlist(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
if len(children) > 1:
# This is a list of ExtSlice, so construct as such by grouping every
# subscript with an optional comma and adding to a list.
extslices = []
for slice, comma in grouper(children, 2):
if comma is None:
extslices.append(ExtSlice(slice=slice.value))
else:
extslices.append(
ExtSlice(
slice=slice.value,
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
),
))
return WithLeadingWhitespace(extslices, children[0].whitespace_before)
else:
# This is an Index or Slice, as parsed in the child.
(index_or_slice, ) = children
return index_or_slice
def convert_assert_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
if len(children) == 2:
(assert_token, test) = children
assert_node = Assert(
whitespace_after_assert=parse_simple_whitespace(
config, test.whitespace_before
),
test=test.value,
msg=None,
)
else:
(assert_token, test, comma_token, msg) = children
assert_node = Assert(
whitespace_after_assert=parse_simple_whitespace(
config, test.whitespace_before
),
test=test.value,
comma=Comma(
whitespace_before=parse_simple_whitespace(
config, comma_token.whitespace_before
),
whitespace_after=parse_simple_whitespace(config, msg.whitespace_before),
),
msg=msg.value,
)
return WithLeadingWhitespace(assert_node, assert_token.whitespace_before)
def _convert_dict_element(
config: ParserConfig,
children_iter: typing.Iterator[typing.Any],
last_child: typing.Any,
) -> typing.Union[DictElement, StarredDictElement]:
first = next(children_iter)
if isinstance(first, Token) and first.string == "**":
expr = next(children_iter)
element = StarredDictElement(
expr.value,
whitespace_before_value=parse_parenthesizable_whitespace(
config, expr.whitespace_before
),
)
else:
key = first
colon_tok = next(children_iter)
value = next(children_iter)
element = DictElement(
key.value,
value.value,
whitespace_before_colon=parse_parenthesizable_whitespace(
config, colon_tok.whitespace_before
),
whitespace_after_colon=parse_parenthesizable_whitespace(
config, colon_tok.whitespace_after
),
)
# Handle the trailing comma (if there is one)
try:
comma_token = next(children_iter)
element = element.with_changes(
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma_token.whitespace_before
),
# Only compute whitespace_after if we're not a trailing comma.
# If we're a trailing comma, that whitespace should be consumed by the
# RightBracket.
whitespace_after=(
parse_parenthesizable_whitespace(
config, comma_token.whitespace_after
)
if comma_token is not last_child
else SimpleWhitespace("")
),
)
)
except StopIteration:
pass
return element
def convert_arglist(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
args = []
for argument, comma in grouper(children, 2):
if comma is None:
args.append(argument)
else:
args.append(
argument.with_changes(comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
)))
return ArglistPartial(args)
def _gather_import_names(config: ParserConfig,
children: Sequence[Any]) -> ImportPartial:
names = []
for name, comma in grouper(children, 2):
if comma is None:
names.append(name)
else:
names.append(
name.with_changes(comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
)))
return ImportPartial(names=names)
def _convert_sequencelike(
config: ParserConfig,
children: typing.Sequence[typing.Any],
single_child_is_sequence: bool,
sequence_type: typing.Union[
typing.Type[Tuple], typing.Type[List], typing.Type[Set]
],
) -> typing.Any:
if not single_child_is_sequence and len(children) == 1:
return children[0]
# N.B. The parent node (e.g. atom) is responsible for computing and attaching
# whitespace information on any parenthesis, square brackets, or curly braces
elements = []
for wrapped_expr_or_starred_element, comma_token in grouper(children, 2):
expr_or_starred_element = wrapped_expr_or_starred_element.value
if comma_token is None:
comma = MaybeSentinel.DEFAULT
else:
comma = Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma_token.whitespace_before
),
# Only compute whitespace_after if we're not a trailing comma.
# If we're a trailing comma, that whitespace should be consumed by the
# TrailingWhitespace, parenthesis, etc.
whitespace_after=(
parse_parenthesizable_whitespace(
config, comma_token.whitespace_after
)
if comma_token is not children[-1]
else SimpleWhitespace("")
),
)
if isinstance(expr_or_starred_element, StarredElement):
starred_element = expr_or_starred_element
elements.append(starred_element.with_changes(comma=comma))
else:
expr = expr_or_starred_element
elements.append(Element(value=expr, comma=comma))
# lpar/rpar are the responsibility of our parent
return WithLeadingWhitespace(
sequence_type(elements, lpar=(), rpar=()), children[0].whitespace_before
)
def _construct_nameitems(config: ParserConfig,
names: Sequence[Any]) -> List[NameItem]:
nameitems: List[NameItem] = []
for name, maybe_comma in grouper(names, 2):
if maybe_comma is None:
nameitems.append(NameItem(Name(name.string)))
else:
nameitems.append(
NameItem(
Name(name.string),
comma=Comma(
whitespace_before=parse_simple_whitespace(
config, maybe_comma.whitespace_before),
whitespace_after=parse_simple_whitespace(
config, maybe_comma.whitespace_after),
),
))
return nameitems
def convert_subscriptlist(config: ParserConfig,
children: typing.Sequence[typing.Any]) -> typing.Any:
# This is a list of SubscriptElement, so construct as such by grouping every
# subscript with an optional comma and adding to a list.
elements = []
for slice, comma in grouper(children, 2):
if comma is None:
elements.append(SubscriptElement(slice=slice.value))
else:
elements.append(
SubscriptElement(
slice=slice.value,
comma=Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
),
))
return WithLeadingWhitespace(elements, children[0].whitespace_before)
def convert_argslist(config: ParserConfig, children: Sequence[Any]) -> Any:
params: List[Param] = []
default_params: List[Param] = []
star_arg: Union[Param, ParamStar, MaybeSentinel] = MaybeSentinel.DEFAULT
kwonly_params: List[Param] = []
star_kwarg: Optional[Param] = None
def add_param(current_param: Optional[List[Param]],
param: Union[Param, ParamStar]) -> Optional[List[Param]]:
nonlocal star_arg
nonlocal star_kwarg
if isinstance(param, ParamStar):
# Only can add this if we don't already have a "*" or a "*param".
if current_param in [params, default_params]:
star_arg = param
current_param = kwonly_params
else:
# Example code:
# def fn(*abc, *): ...
# This should be unreachable, the grammar already disallows it.
raise Exception(
"Cannot have multiple star ('*') markers in a single argument "
+ "list.")
# pyre-ignore Pyre seems to think param.star.__eq__ is not callable
elif isinstance(param.star,
str) and param.star == "" and param.default is None:
# Can only add this if we're in the params or kwonly_params section
if current_param is params:
params.append(param)
elif current_param is kwonly_params:
kwonly_params.append(param)
else:
# Example code:
# def fn(first=None, second): ...
# This code is reachable, so we should use a PartialParserSyntaxError.
raise PartialParserSyntaxError(
"Cannot have a non-default argument following a default argument."
)
elif (isinstance(param.star, str)
# pyre-ignore Pyre seems to think param.star.__eq__ is not callable
and param.star == "" and param.default is not None):
if current_param is params:
current_param = default_params
# Can only add this if we're not yet at star args.
if current_param is default_params:
default_params.append(param)
elif current_param is kwonly_params:
kwonly_params.append(param)
else:
# Example code:
# def fn(**kwargs, trailing=None)
# This should be unreachable, the grammar already disallows it.
raise Exception(
"Cannot have any arguments after a kwargs expansion.")
elif (isinstance(param.star, str)
# pyre-ignore Pyre seems to think param.star.__eq__ is not callable
and param.star == "*" and param.default is None):
# Can only add this if we're in params/default_params, since
# we only allow one of "*" or "*param".
if current_param in [params, default_params]:
star_arg = param
current_param = kwonly_params
else:
# Example code:
# def fn(*first, *second): ...
# This should be unreachable, the grammar already disallows it.
raise Exception(
"Expected a keyword argument but found a starred positional "
+ "argument expansion.")
elif (isinstance(param.star, str)
# pyre-ignore Pyre seems to think param.star.__eq__ is not callable
and param.star == "**" and param.default is None):
# Can add this in all cases where we don't have a star_kwarg
# yet.
if current_param is not None:
star_kwarg = param
current_param = None
else:
# Example code:
# def fn(**first, **second)
# This should be unreachable, the grammar already disallows it.
raise Exception(
"Multiple starred keyword argument expansions are not allowed in a "
+ "single argument list")
else:
# The state machine should never end up here.
raise Exception("Logic error!")
return current_param
# The parameter list we are adding to
current: Optional[List[Param]] = params
# We should have every other item in the group as a param or a comma by now,
# so split them up, add commas and then put them in the appropriate group.
for parameter, comma in grouper(children, 2):
if comma is None:
if isinstance(parameter, ParamStarPartial):
# Example:
# def fn(abc, *): ...
#
# There's also the case where we have bare * with a trailing comma.
# That's handled later.
#
# It's not valid to construct a ParamStar object without a comma, so we
# need to catch the non-comma case separately.
raise PartialParserSyntaxError(
"Named (keyword) arguments must follow a bare *.")
else:
current = add_param(current, parameter)
else:
comma = Comma(
whitespace_before=parse_parenthesizable_whitespace(
config, comma.whitespace_before),
whitespace_after=parse_parenthesizable_whitespace(
config, comma.whitespace_after),
)
if isinstance(parameter, ParamStarPartial):
current = add_param(current, ParamStar(comma=comma))
else:
current = add_param(current,
parameter.with_changes(comma=comma))
if isinstance(star_arg, ParamStar) and len(kwonly_params) == 0:
# Example:
# def fn(abc, *,): ...
#
# This will raise a validation error, but we want to make sure to raise a syntax
# error instead.
#
# The case where there's no trailing comma is already handled by this point, so
# this conditional is only for the case where we have a trailing comma.
raise PartialParserSyntaxError(
"Named (keyword) arguments must follow a bare *.")
return Parameters(
params=tuple(params),
default_params=tuple(default_params),
star_arg=star_arg,
kwonly_params=tuple(kwonly_params),
star_kwarg=star_kwarg,
)