def convert_fstring_expr(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: openbrkt, testlist, *conversions, closebrkt = children # Extract any optional conversion if len(conversions) > 0 and isinstance(conversions[0], FormattedStringConversionPartial): conversion = conversions[0].value conversions = conversions[1:] else: conversion = None # Extract any optional format spec if len(conversions) > 0: format_spec = conversions[0].values else: format_spec = None return FormattedStringExpression( whitespace_before_expression=parse_parenthesizable_whitespace( config, testlist.whitespace_before), expression=testlist.value, whitespace_after_expression=parse_parenthesizable_whitespace( config, children[2].whitespace_before), conversion=conversion, format_spec=format_spec, )
def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: (with_token, *items, colon_token, suite) = children item_nodes: List[WithItem] = [] for with_item, maybe_comma in grouper(items, 2): if maybe_comma is not None: item_nodes.append( with_item.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, maybe_comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, maybe_comma.whitespace_after), ))) else: item_nodes.append(with_item) return WithLeadingWhitespace( With( whitespace_after_with=parse_simple_whitespace( config, with_token.whitespace_after), items=tuple(item_nodes), whitespace_before_colon=parse_simple_whitespace( config, colon_token.whitespace_before), body=suite, ), with_token.whitespace_before, )
def convert_classdef(config: ParserConfig, children: Sequence[Any]) -> Any: classdef, name, *arglist, colon, suite = children # First, parse out the comments and empty lines before the statement. leading_lines = parse_empty_lines(config, classdef.whitespace_before) # Compute common whitespace and nodes whitespace_after_class = parse_simple_whitespace(config, classdef.whitespace_after) namenode = Name(name.string) whitespace_after_name = parse_simple_whitespace(config, name.whitespace_after) # Now, construct the classdef node itself if not arglist: # No arglist, so no arguments to this class return ClassDef( leading_lines=leading_lines, lines_after_decorators=(), whitespace_after_class=whitespace_after_class, name=namenode, whitespace_after_name=whitespace_after_name, body=suite, ) else: # Unwrap arglist partial, because its valid to not have any lpar, *args, rpar = arglist args = args[0].args if args else [] bases: List[Arg] = [] keywords: List[Arg] = [] current_arg = bases for arg in args: if arg.star == "**" or arg.keyword is not None: current_arg = keywords # Some quick validation if current_arg is keywords and (arg.star == "*" or (arg.star == "" and arg.keyword is None)): # TODO: Need a real syntax error here raise Exception("Syntax error!") current_arg.append(arg) return ClassDef( leading_lines=leading_lines, lines_after_decorators=(), whitespace_after_class=whitespace_after_class, name=namenode, whitespace_after_name=whitespace_after_name, lpar=LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar.whitespace_after)), bases=bases, keywords=keywords, rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar.whitespace_before)), whitespace_before_colon=parse_simple_whitespace( config, colon.whitespace_before), body=suite, )
def convert_decorator(config: ParserConfig, children: Sequence[Any]) -> Any: atsign, name, *arglist, newline = children if not arglist: # This is either a name or an attribute node, so just extract it. decoratornode = name else: # This needs to be converted into a call node, and we have the # arglist partial. lpar, *args, rpar = arglist args = args[0].args if args else [] # If the trailing argument doesn't have a comma, then it owns the # trailing whitespace before the rpar. Otherwise, the comma owns # it. if len(args) > 0 and args[-1].comma == MaybeSentinel.DEFAULT: args[-1] = args[-1].with_changes( whitespace_after_arg=parse_parenthesizable_whitespace( config, rpar.whitespace_before)) decoratornode = Call( func=name, whitespace_after_func=parse_simple_whitespace( config, lpar.whitespace_before), whitespace_before_args=parse_parenthesizable_whitespace( config, lpar.whitespace_after), args=tuple(args), ) return Decorator( leading_lines=parse_empty_lines(config, atsign.whitespace_before), whitespace_after_at=parse_simple_whitespace(config, atsign.whitespace_after), decorator=decoratornode, trailing_whitespace=newline, )
def convert_subscriptlist(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) > 1: # This is a list of ExtSlice, so construct as such by grouping every # subscript with an optional comma and adding to a list. extslices = [] for slice, comma in grouper(children, 2): if comma is None: extslices.append(ExtSlice(slice=slice.value)) else: extslices.append( ExtSlice( slice=slice.value, comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, comma.whitespace_after), ), )) return WithLeadingWhitespace(extslices, children[0].whitespace_before) else: # This is an Index or Slice, as parsed in the child. (index_or_slice, ) = children return index_or_slice
def convert_parameters(config: ParserConfig, children: Sequence[Any]) -> Any: lpar, *paramlist, rpar = children return FuncdefPartial( lpar=LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar.whitespace_after)), params=Parameters() if not paramlist else paramlist[0], rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar.whitespace_before)), )
def convert_funcdef_annotation(config: ParserConfig, children: Sequence[Any]) -> Any: arrow, typehint = children return Annotation( whitespace_before_indicator=parse_parenthesizable_whitespace( config, arrow.whitespace_before), whitespace_after_indicator=parse_parenthesizable_whitespace( config, arrow.whitespace_after), annotation=typehint.value, )
def convert_comp_if(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if_tok, test = children return CompIf( test.value, whitespace_before=parse_parenthesizable_whitespace( config, if_tok.whitespace_before), whitespace_before_test=parse_parenthesizable_whitespace( config, test.whitespace_before), )
def convert_subscript(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1 and not isinstance(children[0], Token): # This is just an index node (test, ) = children return WithLeadingWhitespace(Index(test.value), test.whitespace_before) if isinstance(children[-1], SlicePartial): # We got a partial slice as the final param. Extract the final # bits of the full subscript. *others, sliceop = children whitespace_before = others[0].whitespace_before second_colon = sliceop.second_colon step = sliceop.step else: # We can just parse this below, without taking extras from the # partial child. others = children whitespace_before = others[0].whitespace_before second_colon = MaybeSentinel.DEFAULT step = None # We need to create a partial slice to pass up. So, align so we have # a list that's always [Optional[Test], Colon, Optional[Test]]. if isinstance(others[0], Token): # First token is a colon, so insert an empty test on the LHS. We # know the RHS is a test since it's not a sliceop. slicechildren = [None, *others] else: # First token is non-colon, so its a test. slicechildren = [*others] if len(slicechildren) < 3: # Now, we have to fill in the RHS. We know its two long # at this point if its not already 3. slicechildren = [*slicechildren, None] lower, first_colon, upper = slicechildren return WithLeadingWhitespace( Slice( lower=lower.value if lower is not None else None, first_colon=Colon( whitespace_before=parse_parenthesizable_whitespace( config, first_colon.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, first_colon.whitespace_after), ), upper=upper.value if upper is not None else None, second_colon=second_colon, step=step, ), whitespace_before=whitespace_before, )
def convert_import_from(config: ParserConfig, children: Sequence[Any]) -> Any: fromtoken, import_relative, importtoken, *importlist = children if len(importlist) == 1: (possible_star, ) = importlist if isinstance(possible_star, Token): # Its a "*" import, so we must construct this node. names = ImportStar() else: # Its an import as names partial, grab the names from that. names = possible_star.names lpar = None rpar = None else: # Its an import as names partial with parens lpartoken, namespartial, rpartoken = importlist lpar = LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpartoken.whitespace_after)) names = namespartial.names rpar = RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpartoken.whitespace_before)) # If we have a relative-only import, then we need to relocate the space # after the final dot to be owned by the import token. if len(import_relative.relative) > 0 and import_relative.module is None: whitespace_before_import = import_relative.relative[ -1].whitespace_after relative = ( *import_relative.relative[:-1], import_relative.relative[-1].with_changes( whitespace_after=SimpleWhitespace("")), ) else: whitespace_before_import = parse_simple_whitespace( config, importtoken.whitespace_before) relative = import_relative.relative return WithLeadingWhitespace( ImportFrom( whitespace_after_from=parse_simple_whitespace( config, fromtoken.whitespace_after), relative=relative, module=import_relative.module, whitespace_before_import=whitespace_before_import, whitespace_after_import=parse_simple_whitespace( config, importtoken.whitespace_after), lpar=lpar, names=names, rpar=rpar, ), fromtoken.whitespace_before, )
def convert_trailer_attribute( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: dot, name = children return AttributePartial( dot=Dot( whitespace_before=parse_parenthesizable_whitespace( config, dot.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, dot.whitespace_after), ), attr=Name(name.string), )
def convert_comp_op(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: (op, ) = children if op.string in COMPOP_TOKEN_LUT: # A regular comparison containing one token return COMPOP_TOKEN_LUT[op.string]( whitespace_before=parse_parenthesizable_whitespace( config, op.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, op.whitespace_after), ) elif op.string in ["!=", "<>"]: # Not equal, which can take two forms in some cases return NotEqual( whitespace_before=parse_parenthesizable_whitespace( config, op.whitespace_before), value=op.string, whitespace_after=parse_parenthesizable_whitespace( config, op.whitespace_after), ) else: # TODO: Make this a ParserSyntaxError raise Exception(f"Unexpected token '{op.string}'!") else: # A two-token comparison leftcomp, rightcomp = children if leftcomp.string == "not" and rightcomp.string == "in": return NotIn( whitespace_before=parse_parenthesizable_whitespace( config, leftcomp.whitespace_before), whitespace_between=parse_parenthesizable_whitespace( config, leftcomp.whitespace_after), whitespace_after=parse_parenthesizable_whitespace( config, rightcomp.whitespace_after), ) elif leftcomp.string == "is" and rightcomp.string == "not": return IsNot( whitespace_before=parse_parenthesizable_whitespace( config, leftcomp.whitespace_before), whitespace_between=parse_parenthesizable_whitespace( config, leftcomp.whitespace_after), whitespace_after=parse_parenthesizable_whitespace( config, rightcomp.whitespace_after), ) else: # TODO: Make this a ParserSyntaxError raise Exception( f"Unexpected token '{leftcomp.string} {rightcomp.string}'!")
def convert_trailer_arglist( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lpar, *arglist, rpar = children return CallPartial( lpar=WithLeadingWhitespace( LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar.whitespace_after)), lpar.whitespace_before, ), args=() if not arglist else arglist[0].args, rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar.whitespace_before)), )
def convert_trailer_subscriptlist( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: (lbracket, subscriptlist, rbracket) = children return SubscriptPartial( lbracket=LeftSquareBracket( whitespace_after=parse_parenthesizable_whitespace( config, lbracket.whitespace_after)), slice=subscriptlist.value, rbracket=RightSquareBracket( whitespace_before=parse_parenthesizable_whitespace( config, rbracket.whitespace_before)), whitespace_before=lbracket.whitespace_before, )
def convert_arglist(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: args = [] for argument, comma in grouper(children, 2): if comma is None: args.append(argument) else: args.append( argument.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, comma.whitespace_after), ))) return ArglistPartial(args)
def convert_fpdef_assign(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (child, ) = children return child param, equal, default = children return param.with_changes( equal=AssignEqual( whitespace_before=parse_parenthesizable_whitespace( config, equal.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, equal.whitespace_after), ), default=default.value, )
def _gather_import_names(config: ParserConfig, children: Sequence[Any]) -> ImportPartial: names = [] for name, comma in grouper(children, 2): if comma is None: names.append(name) else: names.append( name.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, comma.whitespace_after), ))) return ImportPartial(names=names)
def _convert_dict_comp(config, children: typing.Sequence[typing.Any]) -> typing.Any: key, colon_token, value, comp_for = children return WithLeadingWhitespace( DictComp( key.value, value.value, comp_for, # lbrace, rbrace, lpar, and rpar will be attached as-needed by the atom grammar whitespace_before_colon=parse_parenthesizable_whitespace( config, colon_token.whitespace_before), whitespace_after_colon=parse_parenthesizable_whitespace( config, colon_token.whitespace_after), ), key.whitespace_before, )
def convert_sliceop(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 2: colon, test = children step = test.value else: (colon, ) = children step = None return SlicePartial( second_colon=Colon( whitespace_before=parse_parenthesizable_whitespace( config, colon.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, colon.whitespace_after), ), step=step, )
def convert_atom_expr_trailer( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: atom, *trailers = children whitespace_before = atom.whitespace_before atom = atom.value # Need to walk through all trailers from left to right and construct # a series of nodes based on each partial type. We can't do this with # left recursion due to limits in the parser. for trailer in trailers: if isinstance(trailer, SubscriptPartial): atom = Subscript( value=atom, whitespace_after_value=parse_parenthesizable_whitespace( config, trailer.whitespace_before), lbracket=trailer.lbracket, slice=trailer.slice, rbracket=trailer.rbracket, ) elif isinstance(trailer, AttributePartial): atom = Attribute(value=atom, dot=trailer.dot, attr=trailer.attr) elif isinstance(trailer, CallPartial): # If the trailing argument doesn't have a comma, then it owns the # trailing whitespace before the rpar. Otherwise, the comma owns # it. if (len(trailer.args) > 0 and trailer.args[-1].comma == MaybeSentinel.DEFAULT): args = ( *trailer.args[:-1], trailer.args[-1].with_changes( whitespace_after_arg=trailer.rpar.whitespace_before), ) else: args = trailer.args atom = Call( func=atom, whitespace_after_func=parse_parenthesizable_whitespace( config, trailer.lpar.whitespace_before), whitespace_before_args=trailer.lpar.value.whitespace_after, args=tuple(args), ) else: # This is an invalid trailer, so lets give up raise Exception("Logic error!") return WithLeadingWhitespace(atom, whitespace_before)
def convert_fpdef_starstar(config: ParserConfig, children: Sequence[Any]) -> Any: starstar, param = children return param.with_changes( star=starstar.string, whitespace_after_star=parse_parenthesizable_whitespace( config, starstar.whitespace_after), )
def convert_dotted_as_name(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (dotted_name, ) = children return ImportAlias(name=dotted_name, asname=None) else: dotted_name, astoken, name = children return ImportAlias( name=dotted_name, asname=AsName( whitespace_before_as=parse_parenthesizable_whitespace( config, astoken.whitespace_before), whitespace_after_as=parse_parenthesizable_whitespace( config, astoken.whitespace_after), name=Name(name.string), ), )
def convert_dotted_name(config: ParserConfig, children: Sequence[Any]) -> Any: left, *rest = children node = Name(left.string) for dot, right in grouper(rest, 2): node = Attribute( value=node, dot=Dot( whitespace_before=parse_parenthesizable_whitespace( config, dot.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, dot.whitespace_after), ), attr=Name(right.string), ) return node
def convert_star_arg(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: star, test = children return Arg( star=star.string, whitespace_after_star=parse_parenthesizable_whitespace( config, star.whitespace_after), value=test.value, )
def convert_atom_curlybraces( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lbrace_tok, *body, rbrace_tok = children lbrace = LeftCurlyBrace(whitespace_after=parse_parenthesizable_whitespace( config, lbrace_tok.whitespace_after)) rbrace = RightCurlyBrace( whitespace_before=parse_parenthesizable_whitespace( config, rbrace_tok.whitespace_before)) if len(body) == 0: dict_or_set_node = Dict((), lbrace=lbrace, rbrace=rbrace) else: # len(body) == 1 dict_or_set_node = body[0].value.with_changes(lbrace=lbrace, rbrace=rbrace) return WithLeadingWhitespace(dict_or_set_node, lbrace_tok.whitespace_before)
def convert_fpdef(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: # This is just a parameter (child, ) = children namenode = Name(child.string) annotation = None else: # This is a parameter with a type hint name, colon, typehint = children namenode = Name(name.string) annotation = Annotation( whitespace_before_indicator=parse_parenthesizable_whitespace( config, colon.whitespace_before), whitespace_after_indicator=parse_parenthesizable_whitespace( config, colon.whitespace_after), annotation=typehint.value, ) return Param(star="", name=namenode, annotation=annotation, default=None)
def convert_atom_squarebrackets( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lbracket_tok, *body, rbracket_tok = children lbracket = LeftSquareBracket( whitespace_after=parse_parenthesizable_whitespace( config, lbracket_tok.whitespace_after)) rbracket = RightSquareBracket( whitespace_before=parse_parenthesizable_whitespace( config, rbracket_tok.whitespace_before)) if len(body) == 0: list_node = List((), lbracket=lbracket, rbracket=rbracket) else: # len(body) == 1 # body[0] is a List or ListComp list_node = body[0].value.with_changes(lbracket=lbracket, rbracket=rbracket) return WithLeadingWhitespace(list_node, lbracket_tok.whitespace_before)
def convert_power(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: (child, ) = children return child left, power, right = children return WithLeadingWhitespace( BinaryOperation( left=left.value, operator=Power( whitespace_before=parse_parenthesizable_whitespace( config, power.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, power.whitespace_after), ), right=right.value, ), left.whitespace_before, )
def convert_fpdef_star(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (star, ) = children return ParamStarPartial() else: star, param = children return param.with_changes( star=star.string, whitespace_after_star=parse_parenthesizable_whitespace( config, star.whitespace_after), )
def _convert_dict_element( config: ParserConfig, children_iter: typing.Iterator[typing.Any], last_child: typing.Any, ) -> typing.Union[DictElement, StarredDictElement]: first = next(children_iter) if isinstance(first, Token) and first.string == "**": expr = next(children_iter) element = StarredDictElement( expr.value, whitespace_before_value=parse_parenthesizable_whitespace( config, expr.whitespace_before), ) else: key = first colon_tok = next(children_iter) value = next(children_iter) element = DictElement( key.value, value.value, whitespace_before_colon=parse_parenthesizable_whitespace( config, colon_tok.whitespace_before), whitespace_after_colon=parse_parenthesizable_whitespace( config, colon_tok.whitespace_after), ) # Handle the trailing comma (if there is one) try: comma_token = next(children_iter) element = element.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma_token.whitespace_before), # Only compute whitespace_after if we're not a trailing comma. # If we're a trailing comma, that whitespace should be consumed by the # RightBracket. whitespace_after=(parse_parenthesizable_whitespace( config, comma_token.whitespace_after ) if comma_token is not last_child else SimpleWhitespace("")), )) except StopIteration: pass return element