def _ReadArithWord(self): """Helper function for ReadArithWord.""" #assert self.token_type != Id.Undefined_Tok self._Peek() #print('_ReadArithWord', self.cur_token) if self.token_kind == Kind.Unknown: self.AddErrorContext("Unknown token in arith context: %s", self.cur_token, token=self.cur_token) return None, False elif self.token_kind == Kind.Eof: # Just return EOF token w = ast.TokenWord(self.cur_token) return w, False #self.AddErrorContext("Unexpected EOF in arith context: %s", # self.cur_token, token=self.cur_token) #return None, False elif self.token_kind == Kind.Ignored: # Space should be ignored. TODO: change this to SPACE_SPACE and # SPACE_NEWLINE? or SPACE_TOK. self._Next(lex_mode_e.ARITH) return None, True # Tell wrapper to try again elif self.token_kind in (Kind.Arith, Kind.Right): # Id.Right_ArithSub IS just a normal token, handled by ArithParser self._Next(lex_mode_e.ARITH) w = ast.TokenWord(self.cur_token) return w, False elif self.token_kind in (Kind.Lit, Kind.Left): w = self._ReadCompoundWord(lex_mode=lex_mode_e.ARITH) if not w: return None, True return w, False elif self.token_kind == Kind.VSub: part = ast.SimpleVarSub(self.cur_token) self._Next(lex_mode_e.ARITH) w = ast.CompoundWord([part]) return w, False else: self._BadToken("Unexpected token parsing arith sub: %s", self.cur_token) return None, False raise AssertionError("Shouldn't get here")
def _ReadArithWord(self): """Helper function for ReadArithWord.""" self._Peek() if self.token_kind == Kind.Unknown: p_die('Unexpected token in arithmetic context', token=self.cur_token) elif self.token_kind == Kind.Eof: # Just return EOF token w = ast.TokenWord(self.cur_token) return w, False elif self.token_kind == Kind.Ignored: # Space should be ignored. TODO: change this to SPACE_SPACE and # SPACE_NEWLINE? or SPACE_TOK. self._Next(lex_mode_e.ARITH) return None, True # Tell wrapper to try again elif self.token_kind in (Kind.Arith, Kind.Right): # Id.Right_ArithSub IS just a normal token, handled by ArithParser self._Next(lex_mode_e.ARITH) w = ast.TokenWord(self.cur_token) return w, False elif self.token_kind in (Kind.Lit, Kind.Left): w = self._ReadCompoundWord(lex_mode=lex_mode_e.ARITH) return w, False elif self.token_kind == Kind.VSub: part = ast.SimpleVarSub(self.cur_token) self._Next(lex_mode_e.ARITH) w = ast.CompoundWord([part]) return w, False else: assert False, ("Unexpected token parsing arith sub: %s" % self.cur_token) raise AssertionError("Shouldn't get here")
def _assertReadWordWithArena(test, word_str): print('\n---', word_str) arena, w_parser = _InitWordParserWithArena(word_str) w = w_parser.ReadWord(lex_mode_e.OUTER) assert w is not None ast_lib.PrettyPrint(w) # Next word must be Eof_Real w2 = w_parser.ReadWord(lex_mode_e.OUTER) test.assertTrue( test_lib.TokenWordsEqual(ast.TokenWord(ast.token(Id.Eof_Real, '')), w2), w2) return arena, w
def testMultiLine(self): w_parser = InitWordParser("""\ ls foo # Multiple newlines and comments should be ignored ls bar """) print('--MULTI') w = w_parser.ReadWord(lex_mode_e.OUTER) parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'ls'))] test_lib.AssertAsdlEqual(self, ast.CompoundWord(parts), w) w = w_parser.ReadWord(lex_mode_e.OUTER) parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'foo'))] test_lib.AssertAsdlEqual(self, ast.CompoundWord(parts), w) w = w_parser.ReadWord(lex_mode_e.OUTER) t = ast.token(Id.Op_Newline, '\n') test_lib.AssertAsdlEqual(self, ast.TokenWord(t), w) w = w_parser.ReadWord(lex_mode_e.OUTER) parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'ls'))] test_lib.AssertAsdlEqual(self, ast.CompoundWord(parts), w) w = w_parser.ReadWord(lex_mode_e.OUTER) parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'bar'))] test_lib.AssertAsdlEqual(self, ast.CompoundWord(parts), w) w = w_parser.ReadWord(lex_mode_e.OUTER) t = ast.token(Id.Op_Newline, '\n') test_lib.AssertAsdlEqual(self, ast.TokenWord(t), w) w = w_parser.ReadWord(lex_mode_e.OUTER) t = ast.token(Id.Eof_Real, '') test_lib.AssertAsdlEqual(self, ast.TokenWord(t), w)
def _assertReadWordWithArena(test, word_str): print('\n---', word_str) arena, w_parser = _InitWordParserWithArena(word_str) w = w_parser.ReadWord(lex_mode_e.OUTER) if w: ast_lib.PrettyPrint(w) else: err = w_parser.Error() test.fail("Couldn't parse %r: %s" % (word_str, err)) # Next word must be Eof_Real w2 = w_parser.ReadWord(lex_mode_e.OUTER) test.assertTrue( test_lib.TokenWordsEqual(ast.TokenWord(ast.token(Id.Eof_Real, '')), w2), w2) return arena, w
def _ReadWord(self, lex_mode): """Helper function for Read(). Returns: 2-tuple (word, need_more) word: Word, or None if there was an error, or need_more is set need_more: True if the caller should call us again """ #print('_Read', lex_mode, self.cur_token) self._Peek() if self.token_kind == Kind.Eof: # No advance return ast.TokenWord(self.cur_token), False # Allow Arith for ) at end of for loop? elif self.token_kind in (Kind.Op, Kind.Redir, Kind.Arith): self._Next(lex_mode) if self.token_type == Id.Op_Newline: if self.cursor_was_newline: #print('SKIP(nl)', self.cur_token) return None, True return ast.TokenWord(self.cur_token), False elif self.token_kind == Kind.Right: #print('WordParser.Read: Kind.Right', self.cur_token) if self.token_type not in (Id.Right_Subshell, Id.Right_FuncDef, Id.Right_CasePat, Id.Right_ArrayLiteral): raise AssertionError(self.cur_token) self._Next(lex_mode) return ast.TokenWord(self.cur_token), False elif self.token_kind in (Kind.Ignored, Kind.WS): self._Next(lex_mode) return None, True # tell Read() to try again elif self.token_kind in (Kind.VSub, Kind.Lit, Kind.Left, Kind.KW, Kind.Assign, Kind.ControlFlow, Kind.BoolUnary, Kind.BoolBinary, Kind.ExtGlob): # We're beginning a word. If we see Id.Lit_Pound, change to # lex_mode_e.COMMENT and read until end of line. (TODO: How to add # comments to AST?) # TODO: Can we do the same thing for Tilde here? Enter a state where we # look for / too. if self.token_type == Id.Lit_Pound: self._Next(lex_mode_e.COMMENT) self._Peek() # NOTE: The # could be the last character in the file. It can't be # Eof_{RParen,Backtick} because #) and #` are comments. assert self.token_type in (Id.Ignored_Comment, Id.Eof_Real), \ self.cur_token # The next iteration will go into Kind.Ignored and set lex state to # lex_mode_e.OUTER/etc. return None, True # tell Read() to try again after comment else: w = self._ReadCompoundWord(lex_mode=lex_mode) if not w: self.AddErrorContext('Error reading command word', token=self.cur_token) return None, False return w, False else: raise AssertionError('Unhandled: %s (%s)' % (self.cur_token, self.token_kind)) raise AssertionError("Shouldn't get here")