def convert_subscript(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1 and not isinstance(children[0], Token): # This is just an index node (test, ) = children return WithLeadingWhitespace(Index(test.value), test.whitespace_before) if isinstance(children[-1], SlicePartial): # We got a partial slice as the final param. Extract the final # bits of the full subscript. *others, sliceop = children whitespace_before = others[0].whitespace_before second_colon = sliceop.second_colon step = sliceop.step else: # We can just parse this below, without taking extras from the # partial child. others = children whitespace_before = others[0].whitespace_before second_colon = MaybeSentinel.DEFAULT step = None # We need to create a partial slice to pass up. So, align so we have # a list that's always [Optional[Test], Colon, Optional[Test]]. if isinstance(others[0], Token): # First token is a colon, so insert an empty test on the LHS. We # know the RHS is a test since it's not a sliceop. slicechildren = [None, *others] else: # First token is non-colon, so its a test. slicechildren = [*others] if len(slicechildren) < 3: # Now, we have to fill in the RHS. We know its two long # at this point if its not already 3. slicechildren = [*slicechildren, None] lower, first_colon, upper = slicechildren return WithLeadingWhitespace( Slice( lower=lower.value if lower is not None else None, first_colon=Colon( whitespace_before=parse_parenthesizable_whitespace( config, first_colon.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, first_colon.whitespace_after), ), upper=upper.value if upper is not None else None, second_colon=second_colon, step=step, ), whitespace_before=whitespace_before, )
def convert_subscriptlist(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) > 1: # This is a list of ExtSlice, so construct as such by grouping every # subscript with an optional comma and adding to a list. extslices = [] for slice, comma in grouper(children, 2): if comma is None: extslices.append(ExtSlice(slice=slice.value)) else: extslices.append( ExtSlice( slice=slice.value, comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, comma.whitespace_after), ), )) return WithLeadingWhitespace(extslices, children[0].whitespace_before) else: # This is an Index or Slice, as parsed in the child. (index_or_slice, ) = children return index_or_slice
def convert_raise_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (raise_token, ) = children whitespace_after_raise = MaybeSentinel.DEFAULT exc = None cause = None elif len(children) == 2: (raise_token, test) = children whitespace_after_raise = parse_simple_whitespace( config, test.whitespace_before) exc = test.value cause = None elif len(children) == 4: (raise_token, test, from_token, source) = children whitespace_after_raise = parse_simple_whitespace( config, test.whitespace_before) exc = test.value cause = From( whitespace_before_from=parse_simple_whitespace( config, from_token.whitespace_before), whitespace_after_from=parse_simple_whitespace( config, source.whitespace_before), item=source.value, ) else: raise Exception("Logic error!") return WithLeadingWhitespace( Raise(whitespace_after_raise=whitespace_after_raise, exc=exc, cause=cause), raise_token.whitespace_before, )
def convert_assert_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 2: (assert_token, test) = children assert_node = Assert( whitespace_after_assert=parse_simple_whitespace( config, test.whitespace_before), test=test.value, msg=None, ) else: (assert_token, test, comma_token, msg) = children assert_node = Assert( whitespace_after_assert=parse_simple_whitespace( config, test.whitespace_before), test=test.value, comma=Comma( whitespace_before=parse_simple_whitespace( config, comma_token.whitespace_before), whitespace_after=parse_simple_whitespace( config, msg.whitespace_before), ), msg=msg.value, ) return WithLeadingWhitespace(assert_node, assert_token.whitespace_before)
def _convert_dict(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: is_first_starred = isinstance(children[0], Token) and children[0].string == "**" if is_first_starred: possible_comp_for = None if len(children) < 3 else children[2] else: possible_comp_for = None if len(children) < 4 else children[3] if isinstance(possible_comp_for, CompFor): if is_first_starred: # TODO: Make this a ParserSyntaxError raise Exception( "dict unpacking cannot be used in dict comprehension") return _convert_dict_comp(config, children) children_iter = iter(children) last_child = children[-1] elements = [] while True: try: elements.append( _convert_dict_element(config, children_iter, last_child)) except StopIteration: break # lbrace, rbrace, lpar, and rpar will be attached as-needed by the atom grammar return WithLeadingWhitespace(Dict(tuple(elements)), children[0].whitespace_before)
def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: (with_token, *items, colon_token, suite) = children item_nodes: List[WithItem] = [] for with_item, maybe_comma in grouper(items, 2): if maybe_comma is not None: item_nodes.append( with_item.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, maybe_comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, maybe_comma.whitespace_after), ))) else: item_nodes.append(with_item) return WithLeadingWhitespace( With( whitespace_after_with=parse_simple_whitespace( config, with_token.whitespace_after), items=tuple(item_nodes), whitespace_before_colon=parse_simple_whitespace( config, colon_token.whitespace_before), body=suite, ), with_token.whitespace_before, )
def convert_return_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (keyword, ) = children return WithLeadingWhitespace( Return(whitespace_after_return=SimpleWhitespace("")), keyword.whitespace_before, ) else: (keyword, testlist) = children return WithLeadingWhitespace( Return( value=testlist.value, whitespace_after_return=parse_simple_whitespace( config, keyword.whitespace_after), ), keyword.whitespace_before, )
def convert_expr_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: # This is an unassigned expr statement (like a function call) (test_node, ) = children return WithLeadingWhitespace(Expr(value=test_node.value), test_node.whitespace_before) elif len(children) == 2: lhs, rhs = children if isinstance(rhs, AnnAssignPartial): return WithLeadingWhitespace( AnnAssign( target=lhs.value, annotation=rhs.annotation, equal=MaybeSentinel.DEFAULT if rhs.equal is None else rhs.equal, value=rhs.value, ), lhs.whitespace_before, ) elif isinstance(rhs, AugAssignPartial): return WithLeadingWhitespace( AugAssign(target=lhs.value, operator=rhs.operator, value=rhs.value), lhs.whitespace_before, ) # The only thing it could be at this point is an assign with one or more targets. # So, walk the children moving the equals ownership back one and constructing a # list of AssignTargets. targets = [] for i in range(len(children) - 1): target = children[i].value equal = children[i + 1].equal targets.append( AssignTarget( target=target, whitespace_before_equal=equal.whitespace_before, whitespace_after_equal=equal.whitespace_after, )) return WithLeadingWhitespace( Assign(targets=tuple(targets), value=children[-1].value), children[0].whitespace_before, )
def convert_fstring(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: start, *content, end = children return WithLeadingWhitespace( FormattedString(start=start.string, parts=tuple(content), end=end.string), start.whitespace_before, )
def convert_global_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: (global_token, *names) = children return WithLeadingWhitespace( Global( names=tuple(_construct_nameitems(config, names)), whitespace_after_global=parse_simple_whitespace( config, names[0].whitespace_before), ), global_token.whitespace_before, )
def convert_import_name(config: ParserConfig, children: Sequence[Any]) -> Any: importtoken, names = children return WithLeadingWhitespace( Import( names=names.names, whitespace_after_import=parse_simple_whitespace( config, importtoken.whitespace_after), ), importtoken.whitespace_before, )
def convert_del_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: (del_name, exprlist) = children return WithLeadingWhitespace( Del( target=exprlist.value, whitespace_after_del=parse_simple_whitespace( config, del_name.whitespace_after), ), del_name.whitespace_before, )
def convert_atom_parens(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lpar_tok, *atoms, rpar_tok = children lpar = LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar_tok.whitespace_after)) rpar = RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar_tok.whitespace_before)) if len(atoms) == 1: # inner_atom is a _BaseParenthesizedNode inner_atom = atoms[0].value return WithLeadingWhitespace( inner_atom.with_changes(lpar=(lpar, *inner_atom.lpar), rpar=(*inner_atom.rpar, rpar)), lpar_tok.whitespace_before, ) else: return WithLeadingWhitespace(Tuple((), lpar=(lpar, ), rpar=(rpar, )), lpar_tok.whitespace_before)
def convert_atom_expr_await( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: keyword, expr = children return WithLeadingWhitespace( Await( whitespace_after_await=parse_parenthesizable_whitespace( config, keyword.whitespace_after), expression=expr.value, ), keyword.whitespace_before, )
def convert_import_from(config: ParserConfig, children: Sequence[Any]) -> Any: fromtoken, import_relative, importtoken, *importlist = children if len(importlist) == 1: (possible_star, ) = importlist if isinstance(possible_star, Token): # Its a "*" import, so we must construct this node. names = ImportStar() else: # Its an import as names partial, grab the names from that. names = possible_star.names lpar = None rpar = None else: # Its an import as names partial with parens lpartoken, namespartial, rpartoken = importlist lpar = LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpartoken.whitespace_after)) names = namespartial.names rpar = RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpartoken.whitespace_before)) # If we have a relative-only import, then we need to relocate the space # after the final dot to be owned by the import token. if len(import_relative.relative) > 0 and import_relative.module is None: whitespace_before_import = import_relative.relative[ -1].whitespace_after relative = ( *import_relative.relative[:-1], import_relative.relative[-1].with_changes( whitespace_after=SimpleWhitespace("")), ) else: whitespace_before_import = parse_simple_whitespace( config, importtoken.whitespace_before) relative = import_relative.relative return WithLeadingWhitespace( ImportFrom( whitespace_after_from=parse_simple_whitespace( config, fromtoken.whitespace_after), relative=relative, module=import_relative.module, whitespace_before_import=whitespace_before_import, whitespace_after_import=parse_simple_whitespace( config, importtoken.whitespace_after), lpar=lpar, names=names, rpar=rpar, ), fromtoken.whitespace_before, )
def convert_star_expr(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: star, expr = children return WithLeadingWhitespace( StarredElement( expr.value, whitespace_before_value=parse_parenthesizable_whitespace( config, expr.whitespace_before), # atom is responsible for parenthesis and trailing_whitespace if they exist # testlist_comp, exprlist, dictorsetmaker, etc are responsible for the comma # if it exists. ), whitespace_before=star.whitespace_before, )
def convert_trailer_arglist( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lpar, *arglist, rpar = children return CallPartial( lpar=WithLeadingWhitespace( LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar.whitespace_after)), lpar.whitespace_before, ), args=() if not arglist else arglist[0].args, rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar.whitespace_before)), )
def convert_not_test(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: (child, ) = children return child else: nottoken, nottest = children return WithLeadingWhitespace( UnaryOperation( operator=Not(whitespace_after=parse_parenthesizable_whitespace( config, nottoken.whitespace_after)), expression=nottest.value, ), nottoken.whitespace_before, )
def convert_atom_string(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: return children[0] else: left, right = children return WithLeadingWhitespace( ConcatenatedString( left=left.value, whitespace_between=parse_parenthesizable_whitespace( config, right.whitespace_before), right=right.value, ), left.whitespace_before, )
def convert_atom_basic(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: (child, ) = children if child.type.name == "NAME": # This also handles 'None', 'True', and 'False' directly, but we # keep it in the grammar to be more correct. return WithLeadingWhitespace(Name(child.string), child.whitespace_before) elif child.type.name == "NUMBER": # We must determine what type of number it is since we split node # types up this way. if re.fullmatch(INTNUMBER_RE, child.string): return WithLeadingWhitespace(Integer(child.string), child.whitespace_before) elif re.fullmatch(FLOATNUMBER_RE, child.string): return WithLeadingWhitespace(Float(child.string), child.whitespace_before) elif re.fullmatch(IMAGNUMBER_RE, child.string): return WithLeadingWhitespace(Imaginary(child.string), child.whitespace_before) else: raise Exception("Unparseable number {child.string}") else: raise Exception(f"Logic error, unexpected token {child.type.name}")
def convert_yield_expr(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: # Yielding implicit none (yield_token, ) = children yield_node = Yield(value=None) else: # Yielding explicit value (yield_token, yield_arg) = children yield_node = Yield( value=yield_arg.value, whitespace_after_yield=parse_parenthesizable_whitespace( config, yield_arg.whitespace_before), ) return WithLeadingWhitespace(yield_node, yield_token.whitespace_before)
def _convert_dict_comp(config, children: typing.Sequence[typing.Any]) -> typing.Any: key, colon_token, value, comp_for = children return WithLeadingWhitespace( DictComp( key.value, value.value, comp_for, # lbrace, rbrace, lpar, and rpar will be attached as-needed by the atom grammar whitespace_before_colon=parse_parenthesizable_whitespace( config, colon_token.whitespace_before), whitespace_after_colon=parse_parenthesizable_whitespace( config, colon_token.whitespace_after), ), key.whitespace_before, )
def convert_comparison(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: (child, ) = children return child lhs, *rest = children comparisons: typing.List[ComparisonTarget] = [] for operator, comparator in grouper(rest, 2): comparisons.append( ComparisonTarget(operator=operator, comparator=comparator.value)) return WithLeadingWhitespace( Comparison(left=lhs.value, comparisons=tuple(comparisons)), lhs.whitespace_before, )
def convert_atom_expr_trailer( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: atom, *trailers = children whitespace_before = atom.whitespace_before atom = atom.value # Need to walk through all trailers from left to right and construct # a series of nodes based on each partial type. We can't do this with # left recursion due to limits in the parser. for trailer in trailers: if isinstance(trailer, SubscriptPartial): atom = Subscript( value=atom, whitespace_after_value=parse_parenthesizable_whitespace( config, trailer.whitespace_before), lbracket=trailer.lbracket, slice=trailer.slice, rbracket=trailer.rbracket, ) elif isinstance(trailer, AttributePartial): atom = Attribute(value=atom, dot=trailer.dot, attr=trailer.attr) elif isinstance(trailer, CallPartial): # If the trailing argument doesn't have a comma, then it owns the # trailing whitespace before the rpar. Otherwise, the comma owns # it. if (len(trailer.args) > 0 and trailer.args[-1].comma == MaybeSentinel.DEFAULT): args = ( *trailer.args[:-1], trailer.args[-1].with_changes( whitespace_after_arg=trailer.rpar.whitespace_before), ) else: args = trailer.args atom = Call( func=atom, whitespace_after_func=parse_parenthesizable_whitespace( config, trailer.lpar.whitespace_before), whitespace_before_args=trailer.lpar.value.whitespace_after, args=tuple(args), ) else: # This is an invalid trailer, so lets give up raise Exception("Logic error!") return WithLeadingWhitespace(atom, whitespace_before)
def convert_yield_arg(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: # Just a regular testlist, pass it up (child, ) = children return child else: # Its a yield from (from_token, test) = children return WithLeadingWhitespace( From( item=test.value, whitespace_after_from=parse_parenthesizable_whitespace( config, test.whitespace_before), ), from_token.whitespace_before, )
def convert_for_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: ( for_token, expr, in_token, test, for_colon_token, for_suite, *else_block, ) = children if len(else_block) > 0: (else_token, else_colon_token, else_suite) = else_block orelse = Else( leading_lines=parse_empty_lines(config, else_token.whitespace_before), whitespace_before_colon=parse_simple_whitespace( config, else_colon_token.whitespace_before ), body=else_suite, ) else: orelse = None return WithLeadingWhitespace( For( whitespace_after_for=parse_simple_whitespace( config, for_token.whitespace_after ), target=expr.value, whitespace_before_in=parse_simple_whitespace( config, in_token.whitespace_before ), whitespace_after_in=parse_simple_whitespace( config, in_token.whitespace_after ), iter=test.value, whitespace_before_colon=parse_simple_whitespace( config, for_colon_token.whitespace_before ), body=for_suite, orelse=orelse, ), for_token.whitespace_before, )
def convert_atom_curlybraces( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lbrace_tok, *body, rbrace_tok = children lbrace = LeftCurlyBrace(whitespace_after=parse_parenthesizable_whitespace( config, lbrace_tok.whitespace_after)) rbrace = RightCurlyBrace( whitespace_before=parse_parenthesizable_whitespace( config, rbrace_tok.whitespace_before)) if len(body) == 0: dict_or_set_node = Dict((), lbrace=lbrace, rbrace=rbrace) else: # len(body) == 1 dict_or_set_node = body[0].value.with_changes(lbrace=lbrace, rbrace=rbrace) return WithLeadingWhitespace(dict_or_set_node, lbrace_tok.whitespace_before)
def convert_power(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: if len(children) == 1: (child, ) = children return child left, power, right = children return WithLeadingWhitespace( BinaryOperation( left=left.value, operator=Power( whitespace_before=parse_parenthesizable_whitespace( config, power.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, power.whitespace_after), ), right=right.value, ), left.whitespace_before, )
def convert_atom_squarebrackets( config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lbracket_tok, *body, rbracket_tok = children lbracket = LeftSquareBracket( whitespace_after=parse_parenthesizable_whitespace( config, lbracket_tok.whitespace_after)) rbracket = RightSquareBracket( whitespace_before=parse_parenthesizable_whitespace( config, rbracket_tok.whitespace_before)) if len(body) == 0: list_node = List((), lbracket=lbracket, rbracket=rbracket) else: # len(body) == 1 # body[0] is a List or ListComp list_node = body[0].value.with_changes(lbracket=lbracket, rbracket=rbracket) return WithLeadingWhitespace(list_node, lbracket_tok.whitespace_before)
def _convert_sequencelike( config: ParserConfig, children: typing.Sequence[typing.Any], single_child_is_sequence: bool, sequence_type: typing.Union[typing.Type[Tuple], typing.Type[List], typing.Type[Set]], ) -> typing.Any: if not single_child_is_sequence and len(children) == 1: return children[0] # N.B. The parent node (e.g. atom) is responsible for computing and attaching # whitespace information on any parenthesis, square brackets, or curly braces elements = [] for wrapped_expr_or_starred_element, comma_token in grouper(children, 2): expr_or_starred_element = wrapped_expr_or_starred_element.value if comma_token is None: comma = MaybeSentinel.DEFAULT else: comma = Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma_token.whitespace_before), # Only compute whitespace_after if we're not a trailing comma. # If we're a trailing comma, that whitespace should be consumed by the # TrailingWhitespace, parenthesis, etc. whitespace_after=(parse_parenthesizable_whitespace( config, comma_token.whitespace_after) if comma_token is not children[-1] else SimpleWhitespace("")), ) if isinstance(expr_or_starred_element, StarredElement): starred_element = expr_or_starred_element elements.append(starred_element.with_changes(comma=comma)) else: expr = expr_or_starred_element elements.append(Element(value=expr, comma=comma)) # lpar/rpar are the responsibility of our parent return WithLeadingWhitespace(sequence_type(elements, lpar=(), rpar=()), children[0].whitespace_before)