def _ReadArithWord(self): # type: () -> Tuple[word_t, bool] """Helper function for ReadWord.""" self._Peek() if self.token_kind == Kind.Unknown: p_die('Unexpected token in arithmetic context', token=self.cur_token) elif self.token_kind == Kind.Eof: # Just return EOF token w = word.Token(self.cur_token) # type: word_t return w, False elif self.token_kind == Kind.Ignored: # Space should be ignored. TODO: change this to SPACE_SPACE and # SPACE_NEWLINE? or SPACE_TOK. self._Next(lex_mode_e.Arith) return None, True # Tell wrapper to try again elif self.token_kind in (Kind.Arith, Kind.Right): # Id.Right_DollarDParen IS just a normal token, handled by ArithParser self._Next(lex_mode_e.Arith) w = word.Token(self.cur_token) return w, False elif self.token_kind in (Kind.Lit, Kind.Left, Kind.VSub): w = self._ReadCompoundWord(lex_mode=lex_mode_e.Arith) return w, False else: assert False, ("Unexpected token parsing arith sub: %s" % self.cur_token) raise AssertionError("Shouldn't get here")
def ParseCommandExpr(self): # type: () -> expr_t enode, last_token = self.parse_ctx.ParseOilExpr(self.lexer, grammar_nt.command_expr) if last_token.id == Id.Op_RBrace: last_token.id = Id.Lit_RBrace self.buffered_word = word.Token(last_token) return enode
def _assertReadWordWithArena(test, w_parser): w = w_parser.ReadWord(lex_mode_e.ShCommand) assert w is not None w.PrettyPrint() # Next word must be Eof_Real w2 = w_parser.ReadWord(lex_mode_e.ShCommand) test.assertTrue( test_lib.TokenWordsEqual(word.Token(token(Id.Eof_Real, '')), w2), w2) return w
def ParseBareDecl(self): # type: () -> expr_t """ Parse the RHS of x = {name: val} """ self._Next(lex_mode_e.Expr) self._Peek() enode, last_token = self.parse_ctx.ParseOilExpr(self.lexer, grammar_nt.command_expr) if last_token.id == Id.Op_RBrace: last_token.id = Id.Lit_RBrace self.buffered_word = word.Token(last_token) self._Next(lex_mode_e.ShCommand) return enode
def testMultiLine(self): w_parser = test_lib.InitWordParser("""\ ls foo # Multiple newlines and comments should be ignored ls bar """) print('--MULTI') w = w_parser.ReadWord(lex_mode_e.ShCommand) parts = [word_part.Literal(token(Id.Lit_Chars, 'ls'))] test_lib.AssertAsdlEqual(self, word.Compound(parts), w) w = w_parser.ReadWord(lex_mode_e.ShCommand) parts = [word_part.Literal(token(Id.Lit_Chars, 'foo'))] test_lib.AssertAsdlEqual(self, word.Compound(parts), w) w = w_parser.ReadWord(lex_mode_e.ShCommand) t = token(Id.Op_Newline, '\n') test_lib.AssertAsdlEqual(self, word.Token(t), w) w = w_parser.ReadWord(lex_mode_e.ShCommand) parts = [word_part.Literal(token(Id.Lit_Chars, 'ls'))] test_lib.AssertAsdlEqual(self, word.Compound(parts), w) w = w_parser.ReadWord(lex_mode_e.ShCommand) parts = [word_part.Literal(token(Id.Lit_Chars, 'bar'))] test_lib.AssertAsdlEqual(self, word.Compound(parts), w) w = w_parser.ReadWord(lex_mode_e.ShCommand) t = token(Id.Op_Newline, '\n') test_lib.AssertAsdlEqual(self, word.Token(t), w) w = w_parser.ReadWord(lex_mode_e.ShCommand) t = token(Id.Eof_Real, '') test_lib.AssertAsdlEqual(self, word.Token(t), w)
def ParseProc(self, node): # type: (command__Proc) -> None # proc name-with-hyphens() must be accepted self._Next(lex_mode_e.ShCommand) self._Peek() # example: 'proc f[' gets you Lit_ArrayLhsOpen if self.token_type != Id.Lit_Chars: p_die('Invalid proc name %r', self.cur_token.val, token=self.cur_token) node.name = self.cur_token last_token = self.parse_ctx.ParseProc(self.lexer, node) if last_token.id == Id.Op_LBrace: # Translate to what CommandParser wants last_token.id = Id.Lit_LBrace self.buffered_word = word.Token(last_token) self._Next(lex_mode_e.ShCommand) # required
def ParsePlaceMutation(self, kw_token): # type: (token) -> command_t """ setvar a[i] = 1 setvar i += 1 setvar i++ """ self._Next(lex_mode_e.Expr) enode, last_token = self.parse_ctx.ParsePlaceMutation(kw_token, self.lexer) # Hack to move } from what the Expr lexer modes gives to what CommandParser # wants if last_token.id == Id.Op_RBrace: last_token.id = Id.Lit_RBrace # Let the CommandParser see the Op_Semi or Op_Newline. self.buffered_word = word.Token(last_token) self._Next(lex_mode_e.ShCommand) # always back to this return enode
def ParseVarDecl(self, kw_token): # type: (token) -> command_t """ oil_var_decl: name_type_list '=' testlist end_stmt Note that assignments must end with \n ; } or EOF. Unlike shell assignments, we disallow: var x = 42 | wc -l var x = 42 && echo hi """ self._Next(lex_mode_e.Expr) enode, last_token = self.parse_ctx.ParseVarDecl(kw_token, self.lexer) # Hack to move } from what the Expr lexer modes gives to what CommandParser # wants if last_token.id == Id.Op_RBrace: last_token.id = Id.Lit_RBrace # Let the CommandParser see the Op_Semi or Op_Newline. self.buffered_word = word.Token(last_token) self._Next(lex_mode_e.ShCommand) # always back to this return enode
def ParseFunc(self, node): # type: (command__Func) -> None last_token = self.parse_ctx.ParseFunc(self.lexer, node) if last_token.id == Id.Op_LBrace: # Translate to what CommandParser wants last_token.id = Id.Lit_LBrace self.buffered_word = word.Token(last_token)
def _ReadWord(self, lex_mode): # type: (lex_mode_t) -> Tuple[word_t, bool] """Helper function for Read(). Returns: 2-tuple (word, need_more) word: Word, or None if there was an error, or need_more is set need_more: True if the caller should call us again """ self._Peek() if self.token_kind == Kind.Eof: # No advance return word.Token(self.cur_token), False # Allow Arith for ) at end of for loop? elif self.token_kind in (Kind.Op, Kind.Redir, Kind.Arith): self._Next(lex_mode) if self.token_type == Id.Op_Newline: if self.cursor_was_newline: return None, True return word.Token(self.cur_token), False elif self.token_kind == Kind.Right: if self.token_type not in ( Id.Right_Subshell, Id.Right_ShFunction, Id.Right_CasePat, Id.Right_ShArrayLiteral): raise AssertionError(self.cur_token) self._Next(lex_mode) return word.Token(self.cur_token), False elif self.token_kind in (Kind.Ignored, Kind.WS): self._Next(lex_mode) return None, True # tell Read() to try again elif self.token_kind in ( Kind.VSub, Kind.Lit, Kind.History, Kind.Left, Kind.KW, Kind.ControlFlow, Kind.BoolUnary, Kind.BoolBinary, Kind.ExtGlob): # We're beginning a word. If we see Id.Lit_Pound, change to # lex_mode_e.Comment and read until end of line. if self.token_type == Id.Lit_Pound: self._Next(lex_mode_e.Comment) self._Peek() # NOTE: The # could be the last character in the file. It can't be # Eof_{RParen,Backtick} because #) and #` are comments. assert self.token_type in (Id.Ignored_Comment, Id.Eof_Real), \ self.cur_token # The next iteration will go into Kind.Ignored and set lex state to # lex_mode_e.ShCommand/etc. return None, True # tell Read() to try again after comment else: w = self._ReadCompoundWord(lex_mode=lex_mode) return w, False else: raise AssertionError( 'Unhandled: %s (%s)' % (self.cur_token, self.token_kind)) raise AssertionError("Shouldn't get here")