def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: (with_token, *items, colon_token, suite) = children item_nodes: List[WithItem] = [] for with_item, maybe_comma in grouper(items, 2): if maybe_comma is not None: item_nodes.append( with_item.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, maybe_comma.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, maybe_comma.whitespace_after), ))) else: item_nodes.append(with_item) return WithLeadingWhitespace( With( whitespace_after_with=parse_simple_whitespace( config, with_token.whitespace_after), items=tuple(item_nodes), whitespace_before_colon=parse_simple_whitespace( config, colon_token.whitespace_before), body=suite, ), with_token.whitespace_before, )
def convert_import_relative(config: ParserConfig, children: Sequence[Any]) -> Any: dots = [] dotted_name = None for child in children: if isinstance(child, Token): # Special case for "...", which is part of the grammar if child.string == "...": dots.extend([ Dot(), Dot(), Dot(whitespace_after=parse_simple_whitespace( config, child.whitespace_after)), ]) else: dots.append( Dot(whitespace_after=parse_simple_whitespace( config, child.whitespace_after))) else: # This should be the dotted name, and we can't get more than # one, but lets be sure anyway if dotted_name is not None: raise Exception("Logic error!") dotted_name = child return ImportRelativePartial(relative=tuple(dots), module=dotted_name)
def convert_assert_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 2: (assert_token, test) = children assert_node = Assert( whitespace_after_assert=parse_simple_whitespace( config, test.whitespace_before), test=test.value, msg=None, ) else: (assert_token, test, comma_token, msg) = children assert_node = Assert( whitespace_after_assert=parse_simple_whitespace( config, test.whitespace_before), test=test.value, comma=Comma( whitespace_before=parse_simple_whitespace( config, comma_token.whitespace_before), whitespace_after=parse_simple_whitespace( config, msg.whitespace_before), ), msg=msg.value, ) return WithLeadingWhitespace(assert_node, assert_token.whitespace_before)
def convert_raise_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (raise_token, ) = children whitespace_after_raise = MaybeSentinel.DEFAULT exc = None cause = None elif len(children) == 2: (raise_token, test) = children whitespace_after_raise = parse_simple_whitespace( config, test.whitespace_before) exc = test.value cause = None elif len(children) == 4: (raise_token, test, from_token, source) = children whitespace_after_raise = parse_simple_whitespace( config, test.whitespace_before) exc = test.value cause = From( whitespace_before_from=parse_simple_whitespace( config, from_token.whitespace_before), whitespace_after_from=parse_simple_whitespace( config, source.whitespace_before), item=source.value, ) else: raise Exception("Logic error!") return WithLeadingWhitespace( Raise(whitespace_after_raise=whitespace_after_raise, exc=exc, cause=cause), raise_token.whitespace_before, )
def convert_simple_stmt_partial(config: ParserConfig, children: Sequence[Any]) -> Any: *statements, trailing_whitespace = children last_stmt = len(statements) / 2 body = [] for i, (stmt_body, semi) in enumerate(grouper(statements, 2)): if semi is not None: if i == (last_stmt - 1): # Trailing semicolons only own the whitespace before. semi = Semicolon( whitespace_before=parse_simple_whitespace( config, semi.whitespace_before), whitespace_after=SimpleWhitespace(""), ) else: # Middle semicolons own the whitespace before and after. semi = Semicolon( whitespace_before=parse_simple_whitespace( config, semi.whitespace_before), whitespace_after=parse_simple_whitespace( config, semi.whitespace_after), ) else: semi = MaybeSentinel.DEFAULT body.append(stmt_body.value.with_changes(semicolon=semi)) return SimpleStatementPartial( body, whitespace_before=statements[0].whitespace_before, trailing_whitespace=trailing_whitespace, )
def convert_annassign(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 2: # Variable annotation only colon, annotation = children annotation = annotation.value equal = None value = None elif len(children) == 4: # Variable annotation and assignment colon, annotation, equal, value = children annotation = annotation.value value = value.value equal = AssignEqual( whitespace_before=parse_simple_whitespace(config, equal.whitespace_before), whitespace_after=parse_simple_whitespace(config, equal.whitespace_after), ) else: raise Exception("Invalid parser state!") return AnnAssignPartial( annotation=Annotation( whitespace_before_indicator=parse_simple_whitespace( config, colon.whitespace_before), whitespace_after_indicator=parse_simple_whitespace( config, colon.whitespace_after), annotation=annotation, ), equal=equal, value=value, )
def convert_classdef(config: ParserConfig, children: Sequence[Any]) -> Any: classdef, name, *arglist, colon, suite = children # First, parse out the comments and empty lines before the statement. leading_lines = parse_empty_lines(config, classdef.whitespace_before) # Compute common whitespace and nodes whitespace_after_class = parse_simple_whitespace(config, classdef.whitespace_after) namenode = Name(name.string) whitespace_after_name = parse_simple_whitespace(config, name.whitespace_after) # Now, construct the classdef node itself if not arglist: # No arglist, so no arguments to this class return ClassDef( leading_lines=leading_lines, lines_after_decorators=(), whitespace_after_class=whitespace_after_class, name=namenode, whitespace_after_name=whitespace_after_name, body=suite, ) else: # Unwrap arglist partial, because its valid to not have any lpar, *args, rpar = arglist args = args[0].args if args else [] bases: List[Arg] = [] keywords: List[Arg] = [] current_arg = bases for arg in args: if arg.star == "**" or arg.keyword is not None: current_arg = keywords # Some quick validation if current_arg is keywords and (arg.star == "*" or (arg.star == "" and arg.keyword is None)): # TODO: Need a real syntax error here raise Exception("Syntax error!") current_arg.append(arg) return ClassDef( leading_lines=leading_lines, lines_after_decorators=(), whitespace_after_class=whitespace_after_class, name=namenode, whitespace_after_name=whitespace_after_name, lpar=LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpar.whitespace_after)), bases=bases, keywords=keywords, rpar=RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpar.whitespace_before)), whitespace_before_colon=parse_simple_whitespace( config, colon.whitespace_before), body=suite, )
def convert_decorator(config: ParserConfig, children: Sequence[Any]) -> Any: atsign, name, *arglist, newline = children if not arglist: # This is either a name or an attribute node, so just extract it. decoratornode = name else: # This needs to be converted into a call node, and we have the # arglist partial. lpar, *args, rpar = arglist args = args[0].args if args else [] # If the trailing argument doesn't have a comma, then it owns the # trailing whitespace before the rpar. Otherwise, the comma owns # it. if len(args) > 0 and args[-1].comma == MaybeSentinel.DEFAULT: args[-1] = args[-1].with_changes( whitespace_after_arg=parse_parenthesizable_whitespace( config, rpar.whitespace_before)) decoratornode = Call( func=name, whitespace_after_func=parse_simple_whitespace( config, lpar.whitespace_before), whitespace_before_args=parse_parenthesizable_whitespace( config, lpar.whitespace_after), args=tuple(args), ) return Decorator( leading_lines=parse_empty_lines(config, atsign.whitespace_before), whitespace_after_at=parse_simple_whitespace(config, atsign.whitespace_after), decorator=decoratornode, trailing_whitespace=newline, )
def convert_while_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: while_token, test, while_colon_token, while_suite, *else_block = children if len(else_block) > 0: (else_token, else_colon_token, else_suite) = else_block orelse = Else( leading_lines=parse_empty_lines(config, else_token.whitespace_before), whitespace_before_colon=parse_simple_whitespace( config, else_colon_token.whitespace_before), body=else_suite, ) else: orelse = None return While( leading_lines=parse_empty_lines(config, while_token.whitespace_before), whitespace_after_while=parse_simple_whitespace( config, while_token.whitespace_after), test=test.value, whitespace_before_colon=parse_simple_whitespace( config, while_colon_token.whitespace_before), body=while_suite, orelse=orelse, )
def convert_except_clause(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (except_token, ) = children whitespace_after_except = SimpleWhitespace("") test = None name = None elif len(children) == 2: (except_token, test_node) = children whitespace_after_except = parse_simple_whitespace( config, except_token.whitespace_after) test = test_node.value name = None else: (except_token, test_node, as_token, name_token) = children whitespace_after_except = parse_simple_whitespace( config, except_token.whitespace_after) test = test_node.value name = AsName( whitespace_before_as=parse_simple_whitespace( config, as_token.whitespace_before), whitespace_after_as=parse_simple_whitespace( config, as_token.whitespace_after), name=Name(name_token.string), ) return ExceptClausePartial( leading_lines=parse_empty_lines(config, except_token.whitespace_before), whitespace_after_except=whitespace_after_except, type=test, name=name, )
def convert_assign(config: ParserConfig, children: Sequence[Any]) -> Any: equal, expr = children return AssignPartial( equal=AssignEqual( whitespace_before=parse_simple_whitespace(config, equal.whitespace_before), whitespace_after=parse_simple_whitespace(config, equal.whitespace_after), ), value=expr.value, )
def convert_try_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: trytoken, try_colon_token, try_suite, *rest = children handlers: List[ExceptHandler] = [] orelse: Optional[Else] = None finalbody: Optional[Finally] = None for clause, colon_token, suite in grouper(rest, 3): if isinstance(clause, Token): if clause.string == "else": if orelse is not None: raise Exception("Logic error!") orelse = Else( leading_lines=parse_empty_lines(config, clause.whitespace_before), whitespace_before_colon=parse_simple_whitespace( config, colon_token.whitespace_before ), body=suite, ) elif clause.string == "finally": if finalbody is not None: raise Exception("Logic error!") finalbody = Finally( leading_lines=parse_empty_lines(config, clause.whitespace_before), whitespace_before_colon=parse_simple_whitespace( config, colon_token.whitespace_before ), body=suite, ) else: raise Exception("Logic error!") elif isinstance(clause, ExceptClausePartial): handlers.append( ExceptHandler( body=suite, type=clause.type, name=clause.name, leading_lines=clause.leading_lines, whitespace_after_except=clause.whitespace_after_except, whitespace_before_colon=parse_simple_whitespace( config, colon_token.whitespace_before ), ) ) else: raise Exception("Logic error!") return Try( leading_lines=parse_empty_lines(config, trytoken.whitespace_before), whitespace_before_colon=parse_simple_whitespace( config, try_colon_token.whitespace_before ), body=try_suite, handlers=tuple(handlers), orelse=orelse, finalbody=finalbody, )
def convert_augassign(config: ParserConfig, children: Sequence[Any]) -> Any: op, expr = children if op.string not in AUGOP_TOKEN_LUT: raise Exception(f"Unexpected token '{op.string}'!") return AugAssignPartial( operator=AUGOP_TOKEN_LUT[op.string]( whitespace_before=parse_simple_whitespace(config, op.whitespace_before), whitespace_after=parse_simple_whitespace(config, op.whitespace_after), ), value=expr.value, )
def convert_import_from(config: ParserConfig, children: Sequence[Any]) -> Any: fromtoken, import_relative, importtoken, *importlist = children if len(importlist) == 1: (possible_star, ) = importlist if isinstance(possible_star, Token): # Its a "*" import, so we must construct this node. names = ImportStar() else: # Its an import as names partial, grab the names from that. names = possible_star.names lpar = None rpar = None else: # Its an import as names partial with parens lpartoken, namespartial, rpartoken = importlist lpar = LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpartoken.whitespace_after)) names = namespartial.names rpar = RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpartoken.whitespace_before)) # If we have a relative-only import, then we need to relocate the space # after the final dot to be owned by the import token. if len(import_relative.relative) > 0 and import_relative.module is None: whitespace_before_import = import_relative.relative[ -1].whitespace_after relative = ( *import_relative.relative[:-1], import_relative.relative[-1].with_changes( whitespace_after=SimpleWhitespace("")), ) else: whitespace_before_import = parse_simple_whitespace( config, importtoken.whitespace_before) relative = import_relative.relative return WithLeadingWhitespace( ImportFrom( whitespace_after_from=parse_simple_whitespace( config, fromtoken.whitespace_after), relative=relative, module=import_relative.module, whitespace_before_import=whitespace_before_import, whitespace_after_import=parse_simple_whitespace( config, importtoken.whitespace_after), lpar=lpar, names=names, rpar=rpar, ), fromtoken.whitespace_before, )
def convert_with_item(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 3: (test, as_token, expr_node) = children test_node = test.value asname = AsName( whitespace_before_as=parse_simple_whitespace( config, as_token.whitespace_before), whitespace_after_as=parse_simple_whitespace( config, as_token.whitespace_after), name=expr_node.value, ) else: (test, ) = children test_node = test.value asname = None return WithItem(item=test_node, asname=asname)
def convert_if_stmt_else(config: ParserConfig, children: Sequence[Any]) -> Any: else_tok, colon_tok, suite = children return Else( leading_lines=parse_empty_lines(config, else_tok.whitespace_before), whitespace_before_colon=parse_simple_whitespace( config, colon_tok.whitespace_before), body=suite, )
def convert_import_as_name(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (dotted_name, ) = children return ImportAlias(name=Name(dotted_name.string), asname=None) else: dotted_name, astoken, name = children return ImportAlias( name=Name(dotted_name.string), asname=AsName( whitespace_before_as=parse_simple_whitespace( config, astoken.whitespace_before), whitespace_after_as=parse_simple_whitespace( config, astoken.whitespace_after), name=Name(name.string), ), )
def convert_if_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if_tok, test, colon_tok, suite, *tail = children if len(tail) > 0: (orelse, ) = tail else: orelse = None return If( leading_lines=parse_empty_lines(config, if_tok.whitespace_before), whitespace_before_test=parse_simple_whitespace( config, if_tok.whitespace_after), test=test.value, whitespace_after_test=parse_simple_whitespace( config, colon_tok.whitespace_before), body=suite, orelse=orelse, )
def _construct_nameitems(config: ParserConfig, names: Sequence[Any]) -> List[NameItem]: nameitems: List[NameItem] = [] for name, maybe_comma in grouper(names, 2): if maybe_comma is None: nameitems.append(NameItem(Name(name.string))) else: nameitems.append( NameItem( Name(name.string), comma=Comma( whitespace_before=parse_simple_whitespace( config, maybe_comma.whitespace_before), whitespace_after=parse_simple_whitespace( config, maybe_comma.whitespace_after), ), )) return nameitems
def convert_for_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: ( for_token, expr, in_token, test, for_colon_token, for_suite, *else_block, ) = children if len(else_block) > 0: (else_token, else_colon_token, else_suite) = else_block orelse = Else( leading_lines=parse_empty_lines(config, else_token.whitespace_before), whitespace_before_colon=parse_simple_whitespace( config, else_colon_token.whitespace_before ), body=else_suite, ) else: orelse = None return WithLeadingWhitespace( For( whitespace_after_for=parse_simple_whitespace( config, for_token.whitespace_after ), target=expr.value, whitespace_before_in=parse_simple_whitespace( config, in_token.whitespace_before ), whitespace_after_in=parse_simple_whitespace( config, in_token.whitespace_after ), iter=test.value, whitespace_before_colon=parse_simple_whitespace( config, for_colon_token.whitespace_before ), body=for_suite, orelse=orelse, ), for_token.whitespace_before, )
def convert_import_name(config: ParserConfig, children: Sequence[Any]) -> Any: importtoken, names = children return WithLeadingWhitespace( Import( names=names.names, whitespace_after_import=parse_simple_whitespace( config, importtoken.whitespace_after), ), importtoken.whitespace_before, )
def convert_del_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: (del_name, exprlist) = children return WithLeadingWhitespace( Del( target=exprlist.value, whitespace_after_del=parse_simple_whitespace( config, del_name.whitespace_after), ), del_name.whitespace_before, )
def convert_global_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: (global_token, *names) = children return WithLeadingWhitespace( Global( names=tuple(_construct_nameitems(config, names)), whitespace_after_global=parse_simple_whitespace( config, names[0].whitespace_before), ), global_token.whitespace_before, )
def convert_simple_stmt_suite(config: ParserConfig, children: Sequence[Any]) -> Any: """ This function is similar to convert_simple_stmt_line, but yields a different type """ (partial,) = children return SimpleStatementSuite( partial.body, leading_whitespace=parse_simple_whitespace(config, partial.whitespace_before), trailing_whitespace=partial.trailing_whitespace, )
def _extract_async( config: ParserConfig, children: Sequence[Any] ) -> Tuple[List[EmptyLine], Optional[Asynchronous], Any]: if len(children) == 1: (stmt, ) = children whitespace_before = stmt.whitespace_before asyncnode = None else: asynctoken, stmt = children whitespace_before = asynctoken.whitespace_before asyncnode = Asynchronous(whitespace_after=parse_simple_whitespace( config, asynctoken.whitespace_after)) return (parse_empty_lines(config, whitespace_before), asyncnode, stmt.value)
def convert_return_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (keyword, ) = children return WithLeadingWhitespace( Return(whitespace_after_return=SimpleWhitespace("")), keyword.whitespace_before, ) else: (keyword, testlist) = children return WithLeadingWhitespace( Return( value=testlist.value, whitespace_after_return=parse_simple_whitespace( config, keyword.whitespace_after), ), keyword.whitespace_before, )
def convert_funcdef(config: ParserConfig, children: Sequence[Any]) -> Any: defnode, namenode, param_partial, *annotation, colon, suite = children # If the trailing paremeter doesn't have a comma, then it owns the trailing # whitespace before the rpar. Otherwise, the comma owns it (and will have # already parsed it). We don't check/update ParamStar because if it exists # then we are guaranteed have at least one kwonly_param. parameters = param_partial.params if parameters.star_kwarg is not None: if parameters.star_kwarg.comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes( star_kwarg=parameters.star_kwarg.with_changes( whitespace_after_param=param_partial.rpar.whitespace_before )) elif parameters.kwonly_params: if parameters.kwonly_params[-1].comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes(kwonly_params=( *parameters.kwonly_params[:-1], parameters.kwonly_params[-1].with_changes( whitespace_after_param=param_partial.rpar.whitespace_before ), )) elif isinstance(parameters.star_arg, Param): if parameters.star_arg.comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes( star_arg=parameters.star_arg.with_changes( whitespace_after_param=param_partial.rpar.whitespace_before )) elif parameters.default_params: if parameters.default_params[-1].comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes(default_params=( *parameters.default_params[:-1], parameters.default_params[-1].with_changes( whitespace_after_param=param_partial.rpar.whitespace_before ), )) elif parameters.params: if parameters.params[-1].comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes(params=( *parameters.params[:-1], parameters.params[-1].with_changes( whitespace_after_param=param_partial.rpar.whitespace_before ), )) return WithLeadingWhitespace( FunctionDef( whitespace_after_def=parse_simple_whitespace( config, defnode.whitespace_after), name=Name(namenode.string), whitespace_after_name=parse_simple_whitespace( config, namenode.whitespace_after), whitespace_before_params=param_partial.lpar.whitespace_after, params=parameters, returns=None if not annotation else annotation[0], whitespace_before_colon=parse_simple_whitespace( config, colon.whitespace_before), body=suite, ), defnode.whitespace_before, )