class LIFETIME:
lifetime: str
RULE = grammars.Rule(
ast=None,
name="LIFETIME",
exp=grammars.Pattern(r"\s*'\s*" + IDENT._IDENTIFIER),
params=None,
kwparams=None,
)
@classmethod
def from_ast(cls, ast: str) -> LIFETIME:
return cls(ast.strip()[1:].strip())
class PUNCT:
punct: str
# from https://github.com/rust-lang/rust/blob/1.46.0/src/librustc_lexer/src/lib.rs#L72-L126
# minus (){}[] (which mess with TokenTree
RULE = grammars.Rule(
ast=None,
name="PUNCT",
exp=grammars.Pattern(r"\s*[;,.@#~?:$=!<>\-&+*/^%]"),
params=None,
kwparams=None,
)
@classmethod
def from_ast(cls, ast: str) -> PUNCT:
return cls(ast.strip())
class IDENT:
ident: str
# from https://doc.rust-lang.org/reference/identifiers.html
_IDENTIFIER_OR_KEYWORD = "([a-zA-Z][a-zA-Z0-9_]*|_[a-zA-Z0-9_]+)"
_RAW_IDENTIFIER = ("r#" + _IDENTIFIER_OR_KEYWORD
) # TODO: Except crate, self, super, Self
_NON_KEYWORD_IDENTIFIER = (_IDENTIFIER_OR_KEYWORD
) # TODO: Except a strict or reserved keyword
_IDENTIFIER = rf"({_NON_KEYWORD_IDENTIFIER}|{_RAW_IDENTIFIER})"
RULE = grammars.Rule(
ast=None,
name="IDENT",
exp=grammars.Pattern(r"\s*" + _IDENTIFIER),
params=None,
kwparams=None,
)
@classmethod
def from_ast(cls, ast: str) -> IDENT:
return cls(ast.strip())
class LITERAL:
literal: str
# from https://github.com/rust-lang/rust/blob/1.46.0/src/librustc_lexer/src/lib.rs#L133-L150
_PATTERNS = [
r"0b([01_]+\.?[01_]*|[01_]*\.[01_]+)([fui][0-9]+)?", # int/float (bin)
r"0x([0-9a-f_]+\.?[0-9a-f_]*|[0-9a-f_]*\.[0-9a-f_]+)([fui][0-9]+)?", # int/float (hex)
r"([0-9][0-9_]*\.?[0-9_]*|([0-9][0-9_]*)?\.[0-9_]+)([fui][0-9]+)?", # int/float (dec)
r"b?'(\\\\|[^\\])'", # char and byte
r'b?"([^\\]|\\\\)*"', # str and bytestr
# TODO: rawstr and rawbytestr
]
RULE = grammars.Rule(
ast=None,
name="LITERAL",
exp=grammars.Pattern(rf"\s*({'|'.join(_PATTERNS)})"),
params=None,
kwparams=None,
)
@classmethod
def from_ast(cls, ast: str) -> PUNCT:
return cls(ast.strip())
def negative(self, ast):
neg = model.NegativeLookahead(ast)
any = model.Pattern('.')
return model.Sequence(AST(sequence=[neg, any]))
def any(self, ast):
return model.Pattern('\w+|\S+')
def newranges(self, ast):
pattern = ''.join(ast)
re.compile(pattern)
return model.Pattern(pattern)
def regexp(self, ast):
pattern = ''.join(ast)
re.compile(pattern)
return model.Pattern(pattern)
def pattern(self, ast, *args):
return grammars.Pattern(ast)
