def testMode_Printf(self): CASES = [ r'hello %s\n', r'%% percent %%\377', ] for case in CASES: print() print('--- %s ---' % case) print() lexer = _InitLexer(case) while True: t = lexer.Read(lex_mode_e.PrintfOuter) print(t) if t.id == Id.Eof_Real: break # Now test the Printf_Percent mode CASES = [r'-3.3f', r'03d'] for case in CASES: print() print('--- %s ---' % case) print() lexer = _InitLexer(case) while True: t = lexer.Read(lex_mode_e.PrintfPercent) print(t) if t.id == Id.Eof_Real: break
def testRead(self): lexer = _InitLexer(CMD) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Lit_Chars, 'ls'), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.WS_Space, None), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Lit_Chars, '/'), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Op_Newline, None), t) # Line two t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Lit_Chars, 'ls'), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.WS_Space, None), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Lit_Chars, '/home/'), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Op_Newline, None), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Eof_Real, ''), t) # Another EOF gives EOF t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Eof_Real, ''), t)
def testLookAhead(self): # I think this is the usage pattern we care about. Peek and Next() past # the function; then Peek() the next token. Then Lookahead in that state. lexer = _InitLexer('fun()') t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(token(Id.Lit_Chars, 'fun'), t) #self.assertEqual(Id.Op_LParen, lexer.LookAhead()) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(token(Id.Op_LParen, '('), t) self.assertTokensEqual(token(Id.Op_RParen, ')'), lexer.LookAhead(lex_mode_e.ShCommand)) lexer = _InitLexer('fun ()') t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(token(Id.Lit_Chars, 'fun'), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(token(Id.WS_Space, ' '), t) self.assertTokensEqual(token(Id.Op_LParen, '('), lexer.LookAhead(lex_mode_e.ShCommand))
def testMode_DollarSq(self): lexer = _InitLexer(r'foo bar\n \x00 \000 \u0065') t = lexer.Read(lex_mode_e.SQ_C) print(t) self.assertTokensEqual(Tok(Id.Char_Literals, 'foo bar'), t) t = lexer.Read(lex_mode_e.SQ_C) print(t) self.assertTokensEqual(Tok(Id.Char_OneChar, r'\n'), t)
def testMode_ExtGlob(self): lexer = _InitLexer('@(foo|bar)') t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.ExtGlob_At, '@('), t) t = lexer.Read(lex_mode_e.ExtGlob) self.assertTokensEqual(Tok(Id.Lit_Chars, 'foo'), t) t = lexer.Read(lex_mode_e.ExtGlob) self.assertTokensEqual(Tok(Id.Op_Pipe, None), t) t = lexer.Read(lex_mode_e.ExtGlob) self.assertTokensEqual(Tok(Id.Lit_Chars, 'bar'), t) t = lexer.Read(lex_mode_e.ExtGlob) self.assertTokensEqual(Tok(Id.Op_RParen, None), t) # Individual cases lexer = _InitLexer('@(') t = lexer.Read(lex_mode_e.ExtGlob) self.assertTokensEqual(Tok(Id.ExtGlob_At, '@('), t) lexer = _InitLexer('*(') t = lexer.Read(lex_mode_e.ExtGlob) self.assertTokensEqual(Tok(Id.ExtGlob_Star, '*('), t) lexer = _InitLexer('?(') t = lexer.Read(lex_mode_e.ExtGlob) self.assertTokensEqual(Tok(Id.ExtGlob_QMark, '?('), t) lexer = _InitLexer('$') t = lexer.Read(lex_mode_e.ExtGlob) self.assertTokensEqual(Tok(Id.Lit_Other, '$'), t)
def testMode_BashRegex(self): lexer = _InitLexer('(foo|bar)') t = lexer.Read(lex_mode_e.BashRegex) self.assertTokensEqual(Tok(Id.Lit_Other, '('), t) t = lexer.Read(lex_mode_e.BashRegex) self.assertTokensEqual(Tok(Id.Lit_Chars, 'foo'), t) t = lexer.Read(lex_mode_e.BashRegex) self.assertTokensEqual(Tok(Id.Lit_Other, '|'), t)
def testPushHint(self): # Extglob use case lexer = _InitLexer('@()') lexer.PushHint(Id.Op_RParen, Id.Right_ExtGlob) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.ExtGlob_At, '@('), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Right_ExtGlob, None), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Eof_Real, ''), t)
def testMode_VSub_ArgUnquoted(self): # Another EOF gives EOF lexer = _InitLexer("'hi'") t = lexer.Read(lex_mode_e.VSub_ArgUnquoted) #self.assertTokensEqual(Tok(Id.Eof_Real, ''), t) #t = l.Read(lex_mode_e.VSub_ArgUnquoted) print(t)
def testEmitCompDummy(self): lexer = _InitLexer('echo ') lexer.EmitCompDummy() t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Lit_Chars, 'echo'), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.WS_Space, None), t) # Right before EOF t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Lit_CompDummy, ''), t) t = lexer.Read(lex_mode_e.ShCommand) self.assertTokensEqual(Tok(Id.Eof_Real, ''), t)
def testMode_Expr(self): CASES = [ r'@[ ]', ] for case in CASES: print() print('--- %s ---' % case) print() lexer = _InitLexer(case) while True: t = lexer.Read(lex_mode_e.Expr) print(t) if t.id == Id.Eof_Real: break
def testMode_Backtick(self): CASES = [ r'echo \" \\ hi`', r'`', r'', ] for case in CASES: print() print('--- %s ---' % case) print() lexer = _InitLexer(case) while True: t = lexer.Read(lex_mode_e.Backtick) print(t) if t.id == Id.Eof_Real: break