def testLexerPairs(self):
def MakeLookup(p):
return dict((pat, tok) for _, pat, tok in p)
lookup = MakeLookup(ID_SPEC.LexerPairs(Kind.BoolUnary))
print(lookup)
self.assertEqual(Id.BoolUnary_e, lookup['-e'])
self.assertEqual(Id.BoolUnary_z, lookup['-z'])
lookup2 = MakeLookup(ID_SPEC.LexerPairs(Kind.BoolBinary))
self.assertEqual(Id.BoolBinary_eq, lookup2['-eq'])
def testOilTokenDef(self):
# Used for
#tok_def = OilTokenDef()
# NOTE: These overlap with Kind.Op.
# We need ID_SPEC.ExprOperators(), which has Op, Arith, and Expr kinds.
# We don't have:
# LexerPairs(Kind.Op)
# LexerPairs(Kind.Expr)
# Because we really need a lookup for a MODE.
# Problem: _UNQUOTED is used in both DBracket and ShCommand mode.
arith = ID_SPEC.LexerPairs(Kind.Arith)
print(arith)
# Preprocessing before Outer
LEXER_DEF[lex_mode_e.Backtick] = [
C(r'`', Id.Backtick_Right),
# A backslash, and then one of the SAME FOUR escaped chars in the DQ mode.
R(r'\\[$`"\\]', Id.Backtick_Quoted),
R(r'[^`\\\0]+', Id.Backtick_Other), # contiguous run of literals
R(r'[^\0]', Id.Backtick_Other), # anything else
]
# DBRACKET: can be like Outer, except:
# - Don't really need redirects either... Redir_Less could be Op_Less
# - Id.Op_DLeftParen can't be nested inside.
LEXER_DEF[lex_mode_e.DBracket] = [
C(']]', Id.Lit_DRightBracket),
C('!', Id.KW_Bang),
] + ID_SPEC.LexerPairs(Kind.BoolUnary) + \
ID_SPEC.LexerPairs(Kind.BoolBinary) + \
_UNQUOTED + _EXTGLOB_BEGIN
# Inside an extended glob, most characters are literals, including spaces and
# punctuation. We also accept \, $var, ${var}, "", etc. They can also be
# nested, so _EXTGLOB_BEGIN appears here.
#
# Example: echo @(<> <>|&&|'foo'|$bar)
LEXER_DEF[lex_mode_e.ExtGlob] = \
_BACKSLASH + _LEFT_SUBS + _LEFT_UNQUOTED + _VARS + _EXTGLOB_BEGIN + [
R(r'[^\\$`"\'|)@*+!?\0]+', Id.Lit_Chars),
C('|', Id.Op_Pipe),
C(')', Id.Op_RParen), # maybe be translated to Id.ExtGlob_RParen
R(r'[^\0]', Id.Lit_Other), # everything else is literal
]
# Preprocessing before Outer
LEXER_DEF[lex_mode_e.Backtick] = [
C(r'`', Id.Backtick_Right),
# A backslash, and then one of the SAME FOUR escaped chars in the DQ mode.
R(r'\\[$`"\\]', Id.Backtick_Quoted),
R(r'[^`\\\0]+', Id.Backtick_Other), # contiguous run of literals
R(r'[^\0]', Id.Backtick_Other), # anything else
]
# DBRACKET: can be like Outer, except:
# - Don't really need redirects either... Redir_Less could be Op_Less
# - Id.Op_DLeftParen can't be nested inside.
LEXER_DEF[lex_mode_e.DBracket] = [
C(']]', Id.Lit_DRightBracket),
C('!', Id.KW_Bang),
] + ID_SPEC.LexerPairs(Kind.BoolUnary) + \
ID_SPEC.LexerPairs(Kind.BoolBinary) + \
_UNQUOTED + _EXTGLOB_BEGIN
# Inside an extended glob, most characters are literals, including spaces and
# punctuation. We also accept \, $var, ${var}, "", etc. They can also be
# nested, so _EXTGLOB_BEGIN appears here.
#
# Example: echo @(<> <>|&&|'foo'|$bar)
LEXER_DEF[lex_mode_e.ExtGlob] = \
_BACKSLASH + _LEFT_SUBS + _LEFT_UNQUOTED + _VARS + _EXTGLOB_BEGIN + [
R(r'[^\\$`"\'|)@*+!?\0]+', Id.Lit_Chars),
C('|', Id.Op_Pipe),
C(')', Id.Op_RParen), # maybe be translated to Id.ExtGlob_RParen
R(r'[^\0]', Id.Lit_Other), # everything else is literal
]
def testLexer(self):
# NOTE: Kind.Expr for Oil doesn't have LexerPairs
pairs = ID_SPEC.LexerPairs(Kind.Arith)
for p in pairs:
#print(p)
pass