class LIFETIME: lifetime: str RULE = grammars.Rule( ast=None, name="LIFETIME", exp=grammars.Pattern(r"\s*'\s*" + IDENT._IDENTIFIER), params=None, kwparams=None, ) @classmethod def from_ast(cls, ast: str) -> LIFETIME: return cls(ast.strip()[1:].strip())
class PUNCT: punct: str # from https://github.com/rust-lang/rust/blob/1.46.0/src/librustc_lexer/src/lib.rs#L72-L126 # minus (){}[] (which mess with TokenTree RULE = grammars.Rule( ast=None, name="PUNCT", exp=grammars.Pattern(r"\s*[;,.@#~?:$=!<>\-&+*/^%]"), params=None, kwparams=None, ) @classmethod def from_ast(cls, ast: str) -> PUNCT: return cls(ast.strip())
class IDENT: ident: str # from https://doc.rust-lang.org/reference/identifiers.html _IDENTIFIER_OR_KEYWORD = "([a-zA-Z][a-zA-Z0-9_]*|_[a-zA-Z0-9_]+)" _RAW_IDENTIFIER = ("r#" + _IDENTIFIER_OR_KEYWORD ) # TODO: Except crate, self, super, Self _NON_KEYWORD_IDENTIFIER = (_IDENTIFIER_OR_KEYWORD ) # TODO: Except a strict or reserved keyword _IDENTIFIER = rf"({_NON_KEYWORD_IDENTIFIER}|{_RAW_IDENTIFIER})" RULE = grammars.Rule( ast=None, name="IDENT", exp=grammars.Pattern(r"\s*" + _IDENTIFIER), params=None, kwparams=None, ) @classmethod def from_ast(cls, ast: str) -> IDENT: return cls(ast.strip())
class LITERAL: literal: str # from https://github.com/rust-lang/rust/blob/1.46.0/src/librustc_lexer/src/lib.rs#L133-L150 _PATTERNS = [ r"0b([01_]+\.?[01_]*|[01_]*\.[01_]+)([fui][0-9]+)?", # int/float (bin) r"0x([0-9a-f_]+\.?[0-9a-f_]*|[0-9a-f_]*\.[0-9a-f_]+)([fui][0-9]+)?", # int/float (hex) r"([0-9][0-9_]*\.?[0-9_]*|([0-9][0-9_]*)?\.[0-9_]+)([fui][0-9]+)?", # int/float (dec) r"b?'(\\\\|[^\\])'", # char and byte r'b?"([^\\]|\\\\)*"', # str and bytestr # TODO: rawstr and rawbytestr ] RULE = grammars.Rule( ast=None, name="LITERAL", exp=grammars.Pattern(rf"\s*({'|'.join(_PATTERNS)})"), params=None, kwparams=None, ) @classmethod def from_ast(cls, ast: str) -> PUNCT: return cls(ast.strip())
def negative(self, ast): neg = model.NegativeLookahead(ast) any = model.Pattern('.') return model.Sequence(AST(sequence=[neg, any]))
def any(self, ast): return model.Pattern('\w+|\S+')
def newranges(self, ast): pattern = ''.join(ast) re.compile(pattern) return model.Pattern(pattern)
def regexp(self, ast): pattern = ''.join(ast) re.compile(pattern) return model.Pattern(pattern)
def pattern(self, ast, *args): return grammars.Pattern(ast)