class Is(BaseCompOp, _BaseOneTokenOp): """ A comparision operator that can be used in a :class:`Comparison` expression. """ #: Any space that appears directly before this operator. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace(" ") #: Any space that appears directly after this operator. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace(" ") def _get_token(self) -> str: return "is"
class Dot(_BaseOneTokenOp): """ Used by :class:`Attribute` as a separator between subsequent :class:`Name` nodes. """ #: Any space that appears directly before this dot. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace("") #: Any space that appears directly after this dot. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace("") def _get_token(self) -> str: return "."
class Colon(_BaseOneTokenOp): """ Used by :class:`Slice` as a separator between subsequent expressions, and in :class:`Lambda` to separate arguments and body. """ #: Any space that appears directly before this colon. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace("") #: Any space that appears directly after this colon. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace("") def _get_token(self) -> str: return ":"
class Or(BaseBooleanOp): """ A boolean operator that can be used in a :class:`BooleanOperation` expression. """ #: Any space that appears directly before this operator. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace(" ") #: Any space that appears directly after this operator. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace(" ") def _get_token(self) -> str: return "or"
class MatrixMultiply(BaseBinaryOp, _BaseOneTokenOp): """ A binary operator that can be used in a :class:`BinaryOperation` expression. """ #: Any space that appears directly before this operator. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace(" ") #: Any space that appears directly after this operator. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace(" ") def _get_token(self) -> str: return "@"
class PowerAssign(BaseAugOp, _BaseOneTokenOp): """ An augmented assignment operator that can be used in a :class:`AugAssign` statement. """ #: Any space that appears directly before this operator. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace(" ") #: Any space that appears directly after this operator. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace(" ") def _get_token(self) -> str: return "**="
class AssignEqual(_BaseOneTokenOp): """ Used by :class:`AnnAssign` to denote a single equal character when doing an assignment on top of a type annotation. Also used by :class:`Param` and :class:`Arg` to denote assignment of a default value. """ #: Any space that appears directly before this equal sign. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace(" ") #: Any space that appears directly after this equal sign. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace(" ") def _get_token(self) -> str: return "="
class Semicolon(_BaseOneTokenOp): """ Used by any small statement (any subclass of :class:`BaseSmallStatement` such as :class:`Pass`) as a separator between subsequent nodes contained within a :class:`SimpleStatementLine` or :class:`SimpleStatementSuite`. """ #: Any space that appears directly before this semicolon. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace("") #: Any space that appears directly after this semicolon. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace("") def _get_token(self) -> str: return ";"
def convert_except_clause(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (except_token, ) = children whitespace_after_except = SimpleWhitespace("") test = None name = None elif len(children) == 2: (except_token, test_node) = children whitespace_after_except = parse_simple_whitespace( config, except_token.whitespace_after) test = test_node.value name = None else: (except_token, test_node, as_token, name_token) = children whitespace_after_except = parse_simple_whitespace( config, except_token.whitespace_after) test = test_node.value name = AsName( whitespace_before_as=parse_simple_whitespace( config, as_token.whitespace_before), whitespace_after_as=parse_simple_whitespace( config, as_token.whitespace_after), name=Name(name_token.string), ) return ExceptClausePartial( leading_lines=parse_empty_lines(config, except_token.whitespace_before), whitespace_after_except=whitespace_after_except, type=test, name=name, )
def convert_simple_stmt_partial(config: ParserConfig, children: Sequence[Any]) -> Any: *statements, trailing_whitespace = children last_stmt = len(statements) / 2 body = [] for i, (stmt_body, semi) in enumerate(grouper(statements, 2)): if semi is not None: if i == (last_stmt - 1): # Trailing semicolons only own the whitespace before. semi = Semicolon( whitespace_before=parse_simple_whitespace( config, semi.whitespace_before), whitespace_after=SimpleWhitespace(""), ) else: # Middle semicolons own the whitespace before and after. semi = Semicolon( whitespace_before=parse_simple_whitespace( config, semi.whitespace_before), whitespace_after=parse_simple_whitespace( config, semi.whitespace_after), ) else: semi = MaybeSentinel.DEFAULT body.append(stmt_body.value.with_changes(semicolon=semi)) return SimpleStatementPartial( body, whitespace_before=statements[0].whitespace_before, trailing_whitespace=trailing_whitespace, )
class IsNot(BaseCompOp, _BaseTwoTokenOp): """ A comparision operator that can be used in a :class:`Comparison` expression. This operator spans two tokens that must be separated by at least one space, so there is a third whitespace attribute to represent this. """ #: Any space that appears directly before this operator. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace(" ") #: Any space that appears between the ``is`` and ``not`` tokens. whitespace_between: BaseParenthesizableWhitespace = SimpleWhitespace(" ") #: Any space that appears directly after this operator. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace(" ") def _get_tokens(self) -> Tuple[str, str]: return ("is", "not")
class Comma(_BaseOneTokenOp): """ Syntactic trivia used as a separator between subsequent items in various parts of the grammar. Some use-cases are: * :class:`Import` or :class:`ImportFrom`. * :class:`FunctionDef` arguments. * :class:`Tuple`/:class:`List`/:class:`Set`/:class:`Dict` elements. """ #: Any space that appears directly before this comma. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace("") #: Any space that appears directly after this comma. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace("") def _get_token(self) -> str: return ","
class Not(BaseUnaryOp): """ A unary operator that can be used in a :class:`UnaryOperation` expression. """ #: Any space that appears directly after this operator. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace(" ") def _get_token(self) -> str: return "not"
def convert_import_from(config: ParserConfig, children: Sequence[Any]) -> Any: fromtoken, import_relative, importtoken, *importlist = children if len(importlist) == 1: (possible_star, ) = importlist if isinstance(possible_star, Token): # Its a "*" import, so we must construct this node. names = ImportStar() else: # Its an import as names partial, grab the names from that. names = possible_star.names lpar = None rpar = None else: # Its an import as names partial with parens lpartoken, namespartial, rpartoken = importlist lpar = LeftParen(whitespace_after=parse_parenthesizable_whitespace( config, lpartoken.whitespace_after)) names = namespartial.names rpar = RightParen(whitespace_before=parse_parenthesizable_whitespace( config, rpartoken.whitespace_before)) # If we have a relative-only import, then we need to relocate the space # after the final dot to be owned by the import token. if len(import_relative.relative) > 0 and import_relative.module is None: whitespace_before_import = import_relative.relative[ -1].whitespace_after relative = ( *import_relative.relative[:-1], import_relative.relative[-1].with_changes( whitespace_after=SimpleWhitespace("")), ) else: whitespace_before_import = parse_simple_whitespace( config, importtoken.whitespace_before) relative = import_relative.relative return WithLeadingWhitespace( ImportFrom( whitespace_after_from=parse_simple_whitespace( config, fromtoken.whitespace_after), relative=relative, module=import_relative.module, whitespace_before_import=whitespace_before_import, whitespace_after_import=parse_simple_whitespace( config, importtoken.whitespace_after), lpar=lpar, names=names, rpar=rpar, ), fromtoken.whitespace_before, )
class NotEqual(BaseCompOp): """ A comparison operator that can be used in a :class:`Comparison` expression. This node defines a static value for convenience, but in reality due to PEP 401 it can be one of two values, both of which should be a :class:`NotEqual` :class:`Comparison` operator. """ #: The actual text value of this operator. Can be either ``!=`` or ``<>``. value: str = "!=" #: Any space that appears directly before this operator. whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace(" ") #: Any space that appears directly after this operator. whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace(" ") def _validate(self) -> None: if self.value not in ["!=", "<>"]: raise CSTValidationError("Invalid value for NotEqual node.") def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "BaseCompOp": return self.__class__( whitespace_before=visit_required(self, "whitespace_before", self.whitespace_before, visitor), value=self.value, whitespace_after=visit_required(self, "whitespace_after", self.whitespace_after, visitor), ) def _codegen_impl(self, state: CodegenState) -> None: self.whitespace_before._codegen(state) state.add_token(self.value) self.whitespace_after._codegen(state)
def convert_return_stmt(config: ParserConfig, children: Sequence[Any]) -> Any: if len(children) == 1: (keyword, ) = children return WithLeadingWhitespace( Return(whitespace_after_return=SimpleWhitespace("")), keyword.whitespace_before, ) else: (keyword, testlist) = children return WithLeadingWhitespace( Return( value=testlist.value, whitespace_after_return=parse_simple_whitespace( config, keyword.whitespace_after), ), keyword.whitespace_before, )
def _convert_dict_element( config: ParserConfig, children_iter: typing.Iterator[typing.Any], last_child: typing.Any, ) -> typing.Union[DictElement, StarredDictElement]: first = next(children_iter) if isinstance(first, Token) and first.string == "**": expr = next(children_iter) element = StarredDictElement( expr.value, whitespace_before_value=parse_parenthesizable_whitespace( config, expr.whitespace_before), ) else: key = first colon_tok = next(children_iter) value = next(children_iter) element = DictElement( key.value, value.value, whitespace_before_colon=parse_parenthesizable_whitespace( config, colon_tok.whitespace_before), whitespace_after_colon=parse_parenthesizable_whitespace( config, colon_tok.whitespace_after), ) # Handle the trailing comma (if there is one) try: comma_token = next(children_iter) element = element.with_changes(comma=Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma_token.whitespace_before), # Only compute whitespace_after if we're not a trailing comma. # If we're a trailing comma, that whitespace should be consumed by the # RightBracket. whitespace_after=(parse_parenthesizable_whitespace( config, comma_token.whitespace_after ) if comma_token is not last_child else SimpleWhitespace("")), )) except StopIteration: pass return element
def _convert_sequencelike( config: ParserConfig, children: typing.Sequence[typing.Any], single_child_is_sequence: bool, sequence_type: typing.Union[typing.Type[Tuple], typing.Type[List], typing.Type[Set]], ) -> typing.Any: if not single_child_is_sequence and len(children) == 1: return children[0] # N.B. The parent node (e.g. atom) is responsible for computing and attaching # whitespace information on any parenthesis, square brackets, or curly braces elements = [] for wrapped_expr_or_starred_element, comma_token in grouper(children, 2): expr_or_starred_element = wrapped_expr_or_starred_element.value if comma_token is None: comma = MaybeSentinel.DEFAULT else: comma = Comma( whitespace_before=parse_parenthesizable_whitespace( config, comma_token.whitespace_before), # Only compute whitespace_after if we're not a trailing comma. # If we're a trailing comma, that whitespace should be consumed by the # TrailingWhitespace, parenthesis, etc. whitespace_after=(parse_parenthesizable_whitespace( config, comma_token.whitespace_after) if comma_token is not children[-1] else SimpleWhitespace("")), ) if isinstance(expr_or_starred_element, StarredElement): starred_element = expr_or_starred_element elements.append(starred_element.with_changes(comma=comma)) else: expr = expr_or_starred_element elements.append(Element(value=expr, comma=comma)) # lpar/rpar are the responsibility of our parent return WithLeadingWhitespace(sequence_type(elements, lpar=(), rpar=()), children[0].whitespace_before)
def parse_simple_whitespace(config: BaseWhitespaceParserConfig, state: State) -> SimpleWhitespace: # The match never fails because the pattern can match an empty string lines = config.lines # pyre-fixme[16]: Optional type has no attribute `group`. ws_line = SIMPLE_WHITESPACE_RE.match(lines[state.line - 1], state.column).group(0) ws_line_list = [ws_line] while "\\" in ws_line: # continuation character state.line += 1 state.column = 0 # pyre-fixme[16]: Optional type has no attribute `group`. ws_line = SIMPLE_WHITESPACE_RE.match(lines[state.line - 1], state.column).group(0) ws_line_list.append(ws_line) # TODO: we could special-case the common case where there's no continuation # character to avoid list construction and joining. # once we've finished collecting continuation characters state.column += len(ws_line) return SimpleWhitespace("".join(ws_line_list))
def convert_lambda(config: ParserConfig, children: typing.Sequence[typing.Any]) -> typing.Any: lambdatoken, *params, colontoken, test = children # Grab the whitespace around the colon. If there are no params, then # the colon owns the whitespace before and after it. If there are # any params, then the last param owns the whitespace before the colon. # We handle the parameter movement below. colon = Colon( whitespace_before=parse_parenthesizable_whitespace( config, colontoken.whitespace_before), whitespace_after=parse_parenthesizable_whitespace( config, colontoken.whitespace_after), ) # Unpack optional parameters if len(params) == 0: parameters = Parameters() whitespace_after_lambda = MaybeSentinel.DEFAULT else: (parameters, ) = params whitespace_after_lambda = parse_parenthesizable_whitespace( config, lambdatoken.whitespace_after) # Handle pre-colon whitespace if parameters.star_kwarg is not None: if parameters.star_kwarg.comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes( star_kwarg=parameters.star_kwarg.with_changes( whitespace_after_param=colon.whitespace_before)) elif parameters.kwonly_params: if parameters.kwonly_params[-1].comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes(kwonly_params=( *parameters.kwonly_params[:-1], parameters.kwonly_params[-1].with_changes( whitespace_after_param=colon.whitespace_before), )) elif isinstance(parameters.star_arg, Param): if parameters.star_arg.comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes( star_arg=parameters.star_arg.with_changes( whitespace_after_param=colon.whitespace_before)) elif parameters.default_params: if parameters.default_params[-1].comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes(default_params=( *parameters.default_params[:-1], parameters.default_params[-1].with_changes( whitespace_after_param=colon.whitespace_before), )) elif parameters.params: if parameters.params[-1].comma == MaybeSentinel.DEFAULT: parameters = parameters.with_changes(params=( *parameters.params[:-1], parameters.params[-1].with_changes( whitespace_after_param=colon.whitespace_before), )) # Colon doesn't own its own pre-whitespace now. colon = colon.with_changes(whitespace_before=SimpleWhitespace("")) # Return a lambda return WithLeadingWhitespace( Lambda( whitespace_after_lambda=whitespace_after_lambda, params=parameters, body=test.value, colon=colon, ), lambdatoken.whitespace_before, )