def convert_test( config: ParserConfig, children: typing.Sequence[typing.Any] ) -> typing.Any: if len(children) == 1: (child,) = children return child else: (body, if_token, test, else_token, orelse) = children return WithLeadingWhitespace( IfExp( body=body.value, test=test.value, orelse=orelse.value, whitespace_before_if=parse_parenthesizable_whitespace( config, if_token.whitespace_before ), whitespace_after_if=parse_parenthesizable_whitespace( config, if_token.whitespace_after ), whitespace_before_else=parse_parenthesizable_whitespace( config, else_token.whitespace_before ), whitespace_after_else=parse_parenthesizable_whitespace( config, else_token.whitespace_after ), ), body.whitespace_before, )
def convert_factor( config: ParserConfig, children: typing.Sequence[typing.Any] ) -> typing.Any: if len(children) == 1: (child,) = children return child op, factor = children # First, tokenize the unary operator if op.string == "+": opnode = Plus( whitespace_after=parse_parenthesizable_whitespace( config, op.whitespace_after ) ) elif op.string == "-": opnode = Minus( whitespace_after=parse_parenthesizable_whitespace( config, op.whitespace_after ) ) elif op.string == "~": opnode = BitInvert( whitespace_after=parse_parenthesizable_whitespace( config, op.whitespace_after ) ) else: raise Exception(f"Unexpected token '{op.string}'!") return WithLeadingWhitespace( UnaryOperation(operator=opnode, expression=factor.value), op.whitespace_before )
def convert_sync_comp_for( config: ParserConfig, children: typing.Sequence[typing.Any] ) -> typing.Any: # unpack for_tok, target, in_tok, iter, *trailing = children if len(trailing) and isinstance(trailing[-1], CompFor): *ifs, inner_for_in = trailing else: ifs, inner_for_in = trailing, None return CompFor( target=target.value, iter=iter.value, ifs=ifs, inner_for_in=inner_for_in, whitespace_before=parse_parenthesizable_whitespace( config, for_tok.whitespace_before ), whitespace_after_for=parse_parenthesizable_whitespace( config, for_tok.whitespace_after ), whitespace_before_in=parse_parenthesizable_whitespace( config, in_tok.whitespace_before ), whitespace_after_in=parse_parenthesizable_whitespace( config, in_tok.whitespace_after ), )
def convert_subscriptlist(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) > 1: # This is a list of ExtSlice, so construct as such by grouping every # subscript with an optional comma and adding to a list. extslices = [] for slice, comma in grouper(children, 2): if comma is None: extslices.append(ExtSlice(slice=slice.value)) else: extslices.append( ExtSlice( slice=slice.value, comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, comma.whitespace_after), ), )) return WithLeadingWhitespace(extslices, children[0].whitespace_before) else: # This is an Index or Slice, as parsed in the child. (index_or_slice, ) = children return index_or_slice
def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: (with_token, *items, colon_token, suite) = children item_nodes: List[WithItem] = [] for with_item, maybe_comma in grouper(items, 2): if maybe_comma is not None: item_nodes.append( with_item.with_changes( comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, maybe_comma.whitespace_before ), whitespace_after=parse_parenthesizable_whitespace( config, maybe_comma.whitespace_after ), ) ) ) else: item_nodes.append(with_item) return WithLeadingWhitespace( With( whitespace_after_with=parse_simple_whitespace( config, with_token.whitespace_after ), items=tuple(item_nodes), whitespace_before_colon=parse_simple_whitespace( config, colon_token.whitespace_before ), body=suite, ), with_token.whitespace_before, )
def convert_binop(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: leftexpr, *rightexprs = children if len(rightexprs) == 0: return leftexpr whitespace_before = leftexpr.whitespace_before leftexpr = leftexpr.value # Convert all of the operations that have no precedence in a loop for op, rightexpr in grouper(rightexprs, 2): if op.string not in BINOP_TOKEN_LUT: raise Exception(f"Unexpected token '{op.string}'!") leftexpr = BinaryOperation( left=leftexpr, # pyre-ignore Pyre thinks that the type of the LUT is CSTNode. operator=BINOP_TOKEN_LUT[op.string]( whitespace_before=parse_parenthesizable_whitespace( config, op.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, op.whitespace_after), ), right=rightexpr.value, ) return WithLeadingWhitespace(leftexpr, whitespace_before)
def convert_arg_assign_comp_for( config: ParserConfig, children: typing.Sequence[typing.Any] ) -> typing.Any: if len(children) == 1: # Simple test (child,) = children return Arg(value=child.value) elif len(children) == 2: elt, for_in = children return Arg(value=GeneratorExp(elt.value, for_in, lpar=(), rpar=())) else: # "key = value" assignment argument lhs, equal, rhs = children return Arg( keyword=lhs.value, equal=AssignEqual( whitespace_before=parse_parenthesizable_whitespace( config, equal.whitespace_before ), whitespace_after=parse_parenthesizable_whitespace( config, equal.whitespace_after ), ), value=rhs.value, )
def convert_classdef(config: ParserConfig, children: Sequence[Any]) -> Any: classdef, name, *arglist, colon, suite = children # First, parse out the comments and empty lines before the statement. leading_lines = parse_empty_lines(config, classdef.whitespace_before) # Compute common whitespace and nodes whitespace_after_class = parse_simple_whitespace(config, classdef.whitespace_after) namenode = Name(name.string) whitespace_after_name = parse_simple_whitespace(config, name.whitespace_after) # Now, construct the classdef node itself if not arglist: # No arglist, so no arguments to this class return ClassDef( leading_lines=leading_lines, lines_after_decorators=(), whitespace_after_class=whitespace_after_class, name=namenode, whitespace_after_name=whitespace_after_name, body=suite, ) else: # Unwrap arglist partial, because its valid to not have any lpar, *args, rpar = arglist args = args[0].args if args else [] bases: List[Arg] = [] keywords: List[Arg] = [] current_arg = bases for arg in args: if arg.star == "**" or arg.keyword is not None: current_arg = keywords # Some quick validation if current_arg is keywords and (arg.star == "*" or (arg.star == "" and arg.keyword is None)): raise PartialParserSyntaxError( "Positional argument follows keyword argument.") current_arg.append(arg) return ClassDef( leading_lines=leading_lines, lines_after_decorators=(), whitespace_after_class=whitespace_after_class, name=namenode, whitespace_after_name=whitespace_after_name, lpar=LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar.whitespace_after)), bases=bases, keywords=keywords, rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar.whitespace_before)), whitespace_before_colon=parse_simple_whitespace( config, colon.whitespace_before), body=suite, )
def convert_decorator(config: ParserConfig, children: Sequence[Any]) -> Any: atsign, name, *arglist, newline = children if not arglist: # This is either a name or an attribute node, so just extract it. decoratornode = name else: # This needs to be converted into a call node, and we have the # arglist partial. lpar, *args, rpar = arglist args = args[0].args if args else [] # If the trailing argument doesn't have a comma, then it owns the # trailing whitespace before the rpar. Otherwise, the comma owns # it. if len(args) > 0 and args[-1].comma == MaybeSentinel.DEFAULT: args[-1] = args[-1].with_changes( whitespace_after_arg=parse_parenthesizable_whitespace( config, rpar.whitespace_before)) decoratornode = Call( func=name, whitespace_after_func=parse_simple_whitespace( config, lpar.whitespace_before), whitespace_before_args=parse_parenthesizable_whitespace( config, lpar.whitespace_after), args=tuple(args), ) return Decorator( leading_lines=parse_empty_lines(config, atsign.whitespace_before), whitespace_after_at=parse_simple_whitespace(config, atsign.whitespace_after), decorator=decoratornode, trailing_whitespace=newline, )
def convert_atom_parens( config: ParserConfig, children: typing.Sequence[typing.Any] ) -> typing.Any: lpar_tok, *atoms, rpar_tok = children lpar = LeftParen( whitespace_after=parse_parenthesizable_whitespace( config, lpar_tok.whitespace_after ) ) rpar = RightParen( whitespace_before=parse_parenthesizable_whitespace( config, rpar_tok.whitespace_before ) ) if len(atoms) == 1: # inner_atom is a _BaseParenthesizedNode inner_atom = atoms[0].value return WithLeadingWhitespace( inner_atom.with_changes( lpar=(lpar, *inner_atom.lpar), rpar=(*inner_atom.rpar, rpar) ), lpar_tok.whitespace_before, ) else: return WithLeadingWhitespace( Tuple((), lpar=(lpar,), rpar=(rpar,)), lpar_tok.whitespace_before )
def convert_fstring_expr(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: openbrkt, testlist, *conversions, closebrkt = children # Extract any optional conversion if len(conversions) > 0 and isinstance(conversions[0], FormattedStringConversionPartial): conversion = conversions[0].value conversions = conversions[1:] else: conversion = None # Extract any optional format spec if len(conversions) > 0: format_spec = conversions[0].values else: format_spec = None return FormattedStringExpression( whitespace_before_expression=parse_parenthesizable_whitespace( config, testlist.whitespace_before), expression=testlist.value, whitespace_after_expression=parse_parenthesizable_whitespace( config, children[2].whitespace_before), conversion=conversion, format_spec=format_spec, )
def convert_parameters(config: ParserConfig, children: Sequence[Any]) -> Any: lpar, *paramlist, rpar = children return FuncdefPartial( lpar=LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar.whitespace_after)), params=Parameters() if not paramlist else paramlist[0], rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar.whitespace_before)), )
def convert_subscript( config: ParserConfig, children: typing.Sequence[typing.Any] ) -> typing.Any: if len(children) == 1 and not isinstance(children[0], Token): # This is just an index node (test,) = children return WithLeadingWhitespace(Index(test.value), test.whitespace_before) if isinstance(children[-1], SlicePartial): # We got a partial slice as the final param. Extract the final # bits of the full subscript. *others, sliceop = children whitespace_before = others[0].whitespace_before second_colon = sliceop.second_colon step = sliceop.step else: # We can just parse this below, without taking extras from the # partial child. others = children whitespace_before = others[0].whitespace_before second_colon = MaybeSentinel.DEFAULT step = None # We need to create a partial slice to pass up. So, align so we have # a list that's always [Optional[Test], Colon, Optional[Test]]. if isinstance(others[0], Token): # First token is a colon, so insert an empty test on the LHS. We # know the RHS is a test since it's not a sliceop. slicechildren = [None, *others] else: # First token is non-colon, so its a test. slicechildren = [*others] if len(slicechildren) < 3: # Now, we have to fill in the RHS. We know its two long # at this point if its not already 3. slicechildren = [*slicechildren, None] lower, first_colon, upper = slicechildren return WithLeadingWhitespace( Slice( lower=lower.value if lower is not None else None, first_colon=Colon( whitespace_before=parse_parenthesizable_whitespace( config, first_colon.whitespace_before ), whitespace_after=parse_parenthesizable_whitespace( config, first_colon.whitespace_after ), ), upper=upper.value if upper is not None else None, second_colon=second_colon, step=step, ), whitespace_before=whitespace_before, )
def convert_funcdef_annotation(config: ParserConfig, children: Sequence[Any]) -> Any: arrow, typehint = children return Annotation( whitespace_before_indicator=parse_parenthesizable_whitespace( config, arrow.whitespace_before), whitespace_after_indicator=parse_parenthesizable_whitespace( config, arrow.whitespace_after), annotation=typehint.value, )
def convert_comp_if(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if_tok, test = children return CompIf( test.value, whitespace_before=parse_parenthesizable_whitespace( config, if_tok.whitespace_before), whitespace_before_test=parse_parenthesizable_whitespace( config, test.whitespace_before), )
def convert_import_from(config: ParserConfig, children: Sequence[Any]) -> Any: fromtoken, import_relative, importtoken, *importlist = children if len(importlist) == 1: (possible_star, ) = importlist if isinstance(possible_star, Token): # Its a "*" import, so we must construct this node. names = ImportStar() else: # Its an import as names partial, grab the names from that. names = possible_star.names lpar = None rpar = None else: # Its an import as names partial with parens lpartoken, namespartial, rpartoken = importlist lpar = LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpartoken.whitespace_after)) names = namespartial.names rpar = RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpartoken.whitespace_before)) # If we have a relative-only import, then we need to relocate the space # after the final dot to be owned by the import token. if len(import_relative.relative) > 0 and import_relative.module is None: whitespace_before_import = import_relative.relative[ -1].whitespace_after relative = ( *import_relative.relative[:-1], import_relative.relative[-1].with_changes( whitespace_after=SimpleWhitespace("")), ) else: whitespace_before_import = parse_simple_whitespace( config, importtoken.whitespace_before) relative = import_relative.relative return WithLeadingWhitespace( ImportFrom( whitespace_after_from=parse_simple_whitespace( config, fromtoken.whitespace_after), relative=relative, module=import_relative.module, whitespace_before_import=whitespace_before_import, whitespace_after_import=parse_simple_whitespace( config, importtoken.whitespace_after), lpar=lpar, names=names, rpar=rpar, ), fromtoken.whitespace_before, )
def convert_trailer_attribute( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: dot, name = children return AttributePartial( dot=Dot( whitespace_before=parse_parenthesizable_whitespace( config, dot.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, dot.whitespace_after), ), attr=Name(name.string), )
def convert_comp_op(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: (op, ) = children if op.string in COMPOP_TOKEN_LUT: # A regular comparison containing one token # pyre-ignore Pyre thinks that the type of the LUT is CSTNode. return COMPOP_TOKEN_LUT[op.string]( whitespace_before=parse_parenthesizable_whitespace( config, op.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, op.whitespace_after), ) elif op.string in ["!=", "<>"]: # Not equal, which can take two forms in some cases return NotEqual( whitespace_before=parse_parenthesizable_whitespace( config, op.whitespace_before), value=op.string, whitespace_after=parse_parenthesizable_whitespace( config, op.whitespace_after), ) else: # this should be unreachable raise Exception(f"Unexpected token '{op.string}'!") else: # A two-token comparison leftcomp, rightcomp = children if leftcomp.string == "not" and rightcomp.string == "in": return NotIn( whitespace_before=parse_parenthesizable_whitespace( config, leftcomp.whitespace_before), whitespace_between=parse_parenthesizable_whitespace( config, leftcomp.whitespace_after), whitespace_after=parse_parenthesizable_whitespace( config, rightcomp.whitespace_after), ) elif leftcomp.string == "is" and rightcomp.string == "not": return IsNot( whitespace_before=parse_parenthesizable_whitespace( config, leftcomp.whitespace_before), whitespace_between=parse_parenthesizable_whitespace( config, leftcomp.whitespace_after), whitespace_after=parse_parenthesizable_whitespace( config, rightcomp.whitespace_after), ) else: # this should be unreachable raise Exception( f"Unexpected token '{leftcomp.string} {rightcomp.string}'!")
def _convert_dict_element( config: ParserConfig, children_iter: typing.Iterator[typing.Any], last_child: typing.Any, ) -> typing.Union[DictElement, StarredDictElement]: first = next(children_iter) if isinstance(first, Token) and first.string == "**": expr = next(children_iter) element = StarredDictElement( expr.value, whitespace_before_value=parse_parenthesizable_whitespace( config, expr.whitespace_before ), ) else: key = first colon_tok = next(children_iter) value = next(children_iter) element = DictElement( key.value, value.value, whitespace_before_colon=parse_parenthesizable_whitespace( config, colon_tok.whitespace_before ), whitespace_after_colon=parse_parenthesizable_whitespace( config, colon_tok.whitespace_after ), ) # Handle the trailing comma (if there is one) try: comma_token = next(children_iter) element = element.with_changes( comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma_token.whitespace_before ), # Only compute whitespace_after if we're not a trailing comma. # If we're a trailing comma, that whitespace should be consumed by the # RightBracket. whitespace_after=( parse_parenthesizable_whitespace( config, comma_token.whitespace_after ) if comma_token is not last_child else SimpleWhitespace("") ), ) ) except StopIteration: pass return element
def convert_atom_expr_trailer( config: ParserConfig, children: typing.Sequence[typing.Any] ) -> typing.Any: atom, *trailers = children whitespace_before = atom.whitespace_before atom = atom.value # Need to walk through all trailers from left to right and construct # a series of nodes based on each partial type. We can't do this with # left recursion due to limits in the parser. for trailer in trailers: if isinstance(trailer, SubscriptPartial): atom = Subscript( value=atom, whitespace_after_value=parse_parenthesizable_whitespace( config, trailer.whitespace_before ), lbracket=trailer.lbracket, slice=trailer.slice, rbracket=trailer.rbracket, ) elif isinstance(trailer, AttributePartial): atom = Attribute(value=atom, dot=trailer.dot, attr=trailer.attr) elif isinstance(trailer, CallPartial): # If the trailing argument doesn't have a comma, then it owns the # trailing whitespace before the rpar. Otherwise, the comma owns # it. if ( len(trailer.args) > 0 and trailer.args[-1].comma == MaybeSentinel.DEFAULT ): args = ( *trailer.args[:-1], trailer.args[-1].with_changes( whitespace_after_arg=trailer.rpar.whitespace_before ), ) else: args = trailer.args atom = Call( func=atom, whitespace_after_func=parse_parenthesizable_whitespace( config, trailer.lpar.whitespace_before ), whitespace_before_args=trailer.lpar.value.whitespace_after, args=tuple(args), ) else: # This is an invalid trailer, so lets give up raise Exception("Logic error!") return WithLeadingWhitespace(atom, whitespace_before)
def convert_trailer_arglist( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lpar, *arglist, rpar = children return CallPartial( lpar=WithLeadingWhitespace( LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar.whitespace_after)), lpar.whitespace_before, ), args=() if not arglist else arglist[0].args, rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar.whitespace_before)), )
def convert_trailer_subscriptlist( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: (lbracket, subscriptlist, rbracket) = children return SubscriptPartial( lbracket=LeftSquareBracket( whitespace_after=parse_parenthesizable_whitespace( config, lbracket.whitespace_after)), slice=subscriptlist.value, rbracket=RightSquareBracket( whitespace_before=parse_parenthesizable_whitespace( config, rbracket.whitespace_before)), whitespace_before=lbracket.whitespace_before, )
def convert_fpdef_assign(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (child, ) = children return child param, equal, default = children return param.with_changes( equal=AssignEqual( whitespace_before=parse_parenthesizable_whitespace( config, equal.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, equal.whitespace_after), ), default=default.value, )
def convert_arglist(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: args = [] for argument, comma in grouper(children, 2): if comma is None: args.append(argument) else: args.append( argument.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, comma.whitespace_after), ))) return ArglistPartial(args)
def _gather_import_names(config: ParserConfig, children: Sequence[Any]) -> ImportPartial: names = [] for name, comma in grouper(children, 2): if comma is None: names.append(name) else: names.append( name.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, comma.whitespace_after), ))) return ImportPartial(names=names)
def _convert_dict_comp(config, children: typing.Sequence[typing.Any]) -> typing.Any: key, colon_token, value, comp_for = children return WithLeadingWhitespace( DictComp( key.value, value.value, comp_for, # lbrace, rbrace, lpar, and rpar will be attached as-needed by the atom grammar whitespace_before_colon=parse_parenthesizable_whitespace( config, colon_token.whitespace_before), whitespace_after_colon=parse_parenthesizable_whitespace( config, colon_token.whitespace_after), ), key.whitespace_before, )
def convert_fpdef_starstar(config: ParserConfig, children: Sequence[Any]) -> Any: starstar, param = children return param.with_changes( star=starstar.string, whitespace_after_star=parse_parenthesizable_whitespace( config, starstar.whitespace_after), )
def convert_dotted_name(config: ParserConfig, children: Sequence[Any]) -> Any: left, *rest = children node = Name(left.string) for dot, right in grouper(rest, 2): node = Attribute( value=node, dot=Dot( whitespace_before=parse_parenthesizable_whitespace( config, dot.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, dot.whitespace_after), ), attr=Name(right.string), ) return node
def convert_dotted_as_name(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (dotted_name, ) = children return ImportAlias(name=dotted_name, asname=None) else: dotted_name, astoken, name = children return ImportAlias( name=dotted_name, asname=AsName( whitespace_before_as=parse_parenthesizable_whitespace( config, astoken.whitespace_before), whitespace_after_as=parse_parenthesizable_whitespace( config, astoken.whitespace_after), name=Name(name.string), ), )
def convert_sliceop(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 2: colon, test = children step = test.value else: (colon, ) = children step = None return SlicePartial( second_colon=Colon( whitespace_before=parse_parenthesizable_whitespace( config, colon.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, colon.whitespace_after), ), step=step, )