class TupleTokens(XSpec):
r_lparen = LexTok(r'\(', LP)
r_rparen = LexTok(r'\)', RP)
r_num = LexTok(r'[0-9]+', Num)
r_blank = LexTok(r' +', Blank, discard=True)
root = [r_lparen, r_rparen, r_num, r_blank]
class HClassTokens(XSpec):
"""
Character class/set tokens.
"""
t_escape = LexSeq(SeqTok(r'\\', Escape, discard=True), SeqTok(r'.', Char))
t_minus = LexTok(r'\-', Minus)
t_char = LexTok(r'.', Char)
root = [t_escape, t_minus, t_char]
class KeywordTokens(XSpec):
t_if = LexSeq(SeqTok(r'if', type=Keyword), SeqTok(r'\s+', type=Blank))
t_blank = LexTok(r' +', type=Blank)
t_lparen = LexTok(r'\(', type=LP)
t_rparen = LexTok(r'\)', type=RP)
t_colon = LexTok(r'\:', type=Colon)
# Match identifier only if it is not an if.
t_identifier = LexTok(r'[a-zA-Z0-9]+', type=Identifier)
root = [t_if, t_blank, t_lparen, t_rparen, t_colon, t_identifier]
class StringTokens(XSpec):
t_dquote = LexSeq(SeqTok(r'\"', DoubleQuote), SeqTok(r'[^\"]+', String),
SeqTok(r'\"', DoubleQuote))
t_blank = LexTok(r' +', type=Blank)
root = [t_dquote, t_blank]
class CalcTokens(XSpec):
t_plus = LexTok(r'\+', Plus)
t_minus = LexTok(r'\-', Minus)
t_lparen = LexTok(r'\(', LP)
t_rparen = LexTok(r'\)', RP)
t_mul = LexTok(r'\*', Mul)
t_div = LexTok(r'\/', Div)
t_num = LexTok(r'[0-9]+', Num, float)
t_blank = LexTok(r' +', Blank)
root = [t_num, t_blank, t_plus, t_minus, t_lparen, t_rparen, t_mul, t_div]
class CalcTokens(XSpec):
# Used to extract the tokens.
t_plus = LexTok(r'\+', Plus)
t_minus = LexTok(r'\-', Minus)
t_lparen = LexTok(r'\(', LP)
t_rparen = LexTok(r'\)', RP)
t_mul = LexTok(r'\*', Mul)
t_div = LexTok(r'\/', Div)
t_num = LexTok(r'[0-9]+', Num, float)
t_blank = LexTok(r' +', Blank, discard=True)
root = [t_plus, t_minus, t_lparen, t_num, t_blank, t_rparen, t_mul, t_div]
class ExprTokens(XSpec):
t_one = LexTok(r'1', One)
t_two = LexTok(r'2', Two)
t_three = LexTok(r'3', Three)
t_four = LexTok(r'4', Four)
t_five = LexTok(r'5', Five)
t_blank = LexTok(r' +', Blank, discard=True)
root = [t_one, t_two, t_three, t_four, t_five, t_blank]
class CalcTokens(XSpec):
# Token extractors. When it matches the regex's it instantiate
# a Token class with the specified type.
t_plus = LexTok(r'\+', Plus)
t_minus = LexTok(r'\-', Minus)
t_lparen = LexTok(r'\(', LP)
t_rparen = LexTok(r'\)', RP)
t_mul = LexTok(r'\*', Mul)
t_div = LexTok(r'\/', Div)
# Automatically convert the token value to a float.
t_num = LexTok(r'[0-9]+', Num, float)
# White spaces are discarded.
t_blank = LexTok(r' +', Blank, discard=True)
root = [t_plus, t_minus, t_lparen, t_num, t_blank, t_rparen, t_mul, t_div]
class NumsTokens(XSpec):
t_blank = LexTok(r' +', Blank)
t_num = LexTok(r'[0-9]{3,6}', Num)
root = [t_num, t_blank]
class LetterTokens(XSpec):
t_blank = LexTok(r' +', Blank)
t_letter = LexTok(r'[a-zA-Z]', Letter)
root = [t_letter, t_blank]
class CharTokens(XSpec):
t_char = LexTok(r'.', Char)
root = [t_char]
class WordTokens(XSpec):
t_word = LexTok(r'[a-zA-Z]+', Word)
t_blank = LexTok(r' +', type=Blank, discard=True)
root = [t_word, t_blank]
class RegexTokens(XSpec):
t_escape = LexSeq(SeqTok(r'\\', Escape, discard=True), SeqTok(r'.', Char))
t_word = LexSeq(SeqTok(r'\\', Escape), SeqTok(r'w', WordSymbol))
t_nword = LexSeq(SeqTok(r'\\', Escape), SeqTok(r'W', NotWordSymbol))
t_metab = LexSeq(SeqTok(r'\\', Escape), SeqTok(r'b', MetabSymbol))
t_metaB = LexSeq(SeqTok(r'\\', Escape), SeqTok(r'B', MetaBSymbol))
t_plus = LexTok(r'\+', Plus)
t_dot = LexTok(r'\.', Dot)
t_pipe = LexTok(r'\|', Pipe)
t_lparen = LexTok(r'\(', LP)
t_rparen = LexTok(r'\)', RP)
t_gref = LexSeq(SeqTok(r'\\', Escape), SeqTok(r'[0-9]', Num))
# t_lbracket = LexTok(r'\[', LB)
# t_rbracket = LexTok(r'\]', RB)
t_include = LexSeq(SeqTok(r'\[', LB), SeqTok(r'(\\\[|[^\[])+', String),
SeqTok(r'\]', RB))
t_exclude = LexSeq(SeqTok(r'\[', LB), SeqTok(r'\^', Caret),
SeqTok(r'(\\\[|[^\[])+', String), SeqTok(r'\]', RB))
t_lbrace = LexTok(r'\{', LBR)
t_rbrace = LexTok(r'\}', RBR)
t_comma = LexTok(r'\,', Comma)
t_question = LexTok(r'\?', Question)
t_caret = LexTok(r'\^', Caret)
t_dollar = LexTok(r'\$', Dollar)
t_colon = LexTok(r'\:', Colon)
t_mul = LexTok(r'\*', Mul)
# t_minus = LexTok(r'\-', Minus)
# t_hash = LexTok(r'\#', Hash)
t_equal = LexTok(r'\=', Equal)
t_exclam = LexTok(r'\!', Exclam)
t_lesser = LexTok(r'\<', Lesser)
t_greater = LexTok(r'\>', Greater)
t_comment = LexSeq(SeqTok(r'\(', LP), SeqTok(r'\?', Question),
SeqTok(r'\#', Hash), SeqTok(r'(\\\)|[^)])+', Comment),
SeqTok(r'\)', RP))
t_pngroup = LexSeq(SeqTok(r'\(', LP), SeqTok(r'\?', Question),
SeqTok(r'P', GroupSymbol), SeqTok(r'\<', Lesser),
SeqTok(r'[a-zA-Z0-9]+', GroupName),
SeqTok(r'\>', Greater))
t_ngref = LexSeq(SeqTok(r'\(', LP), SeqTok(r'\?', Question),
SeqTok(r'P', GroupSymbol), SeqTok(r'\=', Equal),
SeqTok(r'[a-zA-Z0-9]+', GroupName), SeqTok(r'\)', RP))
t_char = LexTok(r'.', Char)
root = [
t_gref, t_colon, t_ngref, t_comment, t_word, t_nword, t_metab, t_metaB,
t_escape, t_pngroup, t_plus, t_dot, t_lparen, t_rparen, t_mul,
t_exclude, t_include, t_lbrace, t_rbrace, t_comma, t_question,
t_dollar, t_caret, t_pipe, t_equal, t_lesser, t_greater, t_exclam,
t_char
]