def testMode_Printf(self):
CASES = [
r'hello %s\n',
r'%% percent %%\377',
]
for case in CASES:
print()
print('--- %s ---' % case)
print()
lexer = _InitLexer(case)
while True:
t = lexer.Read(lex_mode_e.PrintfOuter)
print(t)
if t.id == Id.Eof_Real:
break
# Now test the Printf_Percent mode
CASES = [r'-3.3f', r'03d']
for case in CASES:
print()
print('--- %s ---' % case)
print()
lexer = _InitLexer(case)
while True:
t = lexer.Read(lex_mode_e.PrintfPercent)
print(t)
if t.id == Id.Eof_Real:
break
def testRead(self):
lexer = _InitLexer(CMD)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Lit_Chars, 'ls'), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.WS_Space, None), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Lit_Chars, '/'), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Op_Newline, None), t)
# Line two
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Lit_Chars, 'ls'), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.WS_Space, None), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Lit_Chars, '/home/'), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Op_Newline, None), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Eof_Real, ''), t)
# Another EOF gives EOF
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Eof_Real, ''), t)
def testLookAhead(self):
# I think this is the usage pattern we care about. Peek and Next() past
# the function; then Peek() the next token. Then Lookahead in that state.
lexer = _InitLexer('fun()')
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(token(Id.Lit_Chars, 'fun'), t)
#self.assertEqual(Id.Op_LParen, lexer.LookAhead())
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(token(Id.Op_LParen, '('), t)
self.assertTokensEqual(token(Id.Op_RParen, ')'),
lexer.LookAhead(lex_mode_e.ShCommand))
lexer = _InitLexer('fun ()')
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(token(Id.Lit_Chars, 'fun'), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(token(Id.WS_Space, ' '), t)
self.assertTokensEqual(token(Id.Op_LParen, '('),
lexer.LookAhead(lex_mode_e.ShCommand))
def testMode_DollarSq(self):
lexer = _InitLexer(r'foo bar\n \x00 \000 \u0065')
t = lexer.Read(lex_mode_e.SQ_C)
print(t)
self.assertTokensEqual(Tok(Id.Char_Literals, 'foo bar'), t)
t = lexer.Read(lex_mode_e.SQ_C)
print(t)
self.assertTokensEqual(Tok(Id.Char_OneChar, r'\n'), t)
def testMode_ExtGlob(self):
lexer = _InitLexer('@(foo|bar)')
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.ExtGlob_At, '@('), t)
t = lexer.Read(lex_mode_e.ExtGlob)
self.assertTokensEqual(Tok(Id.Lit_Chars, 'foo'), t)
t = lexer.Read(lex_mode_e.ExtGlob)
self.assertTokensEqual(Tok(Id.Op_Pipe, None), t)
t = lexer.Read(lex_mode_e.ExtGlob)
self.assertTokensEqual(Tok(Id.Lit_Chars, 'bar'), t)
t = lexer.Read(lex_mode_e.ExtGlob)
self.assertTokensEqual(Tok(Id.Op_RParen, None), t)
# Individual cases
lexer = _InitLexer('@(')
t = lexer.Read(lex_mode_e.ExtGlob)
self.assertTokensEqual(Tok(Id.ExtGlob_At, '@('), t)
lexer = _InitLexer('*(')
t = lexer.Read(lex_mode_e.ExtGlob)
self.assertTokensEqual(Tok(Id.ExtGlob_Star, '*('), t)
lexer = _InitLexer('?(')
t = lexer.Read(lex_mode_e.ExtGlob)
self.assertTokensEqual(Tok(Id.ExtGlob_QMark, '?('), t)
lexer = _InitLexer('$')
t = lexer.Read(lex_mode_e.ExtGlob)
self.assertTokensEqual(Tok(Id.Lit_Other, '$'), t)
def testMode_BashRegex(self):
lexer = _InitLexer('(foo|bar)')
t = lexer.Read(lex_mode_e.BashRegex)
self.assertTokensEqual(Tok(Id.Lit_Other, '('), t)
t = lexer.Read(lex_mode_e.BashRegex)
self.assertTokensEqual(Tok(Id.Lit_Chars, 'foo'), t)
t = lexer.Read(lex_mode_e.BashRegex)
self.assertTokensEqual(Tok(Id.Lit_Other, '|'), t)
def testPushHint(self):
# Extglob use case
lexer = _InitLexer('@()')
lexer.PushHint(Id.Op_RParen, Id.Right_ExtGlob)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.ExtGlob_At, '@('), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Right_ExtGlob, None), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Eof_Real, ''), t)
def testMode_VSub_ArgUnquoted(self):
# Another EOF gives EOF
lexer = _InitLexer("'hi'")
t = lexer.Read(lex_mode_e.VSub_ArgUnquoted)
#self.assertTokensEqual(Tok(Id.Eof_Real, ''), t)
#t = l.Read(lex_mode_e.VSub_ArgUnquoted)
print(t)
def testEmitCompDummy(self):
lexer = _InitLexer('echo ')
lexer.EmitCompDummy()
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Lit_Chars, 'echo'), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.WS_Space, None), t)
# Right before EOF
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Lit_CompDummy, ''), t)
t = lexer.Read(lex_mode_e.ShCommand)
self.assertTokensEqual(Tok(Id.Eof_Real, ''), t)
def testMode_Expr(self):
CASES = [
r'@[ ]',
]
for case in CASES:
print()
print('--- %s ---' % case)
print()
lexer = _InitLexer(case)
while True:
t = lexer.Read(lex_mode_e.Expr)
print(t)
if t.id == Id.Eof_Real:
break
def testMode_Backtick(self):
CASES = [
r'echo \" \\ hi`',
r'`',
r'',
]
for case in CASES:
print()
print('--- %s ---' % case)
print()
lexer = _InitLexer(case)
while True:
t = lexer.Read(lex_mode_e.Backtick)
print(t)
if t.id == Id.Eof_Real:
break
