def _ReadArithWord(self): # type: () -> Tuple[word_t, bool] """Helper function for ReadWord.""" self._Peek() if self.token_kind == Kind.Unknown: p_die('Unexpected token in arithmetic context', token=self.cur_token) elif self.token_kind == Kind.Eof: # Just return EOF token w = osh_word.TokenWord(self.cur_token) # type: word_t return w, False elif self.token_kind == Kind.Ignored: # Space should be ignored. TODO: change this to SPACE_SPACE and # SPACE_NEWLINE? or SPACE_TOK. self._Next(lex_mode_e.Arith) return None, True # Tell wrapper to try again elif self.token_kind in (Kind.Arith, Kind.Right): # Id.Right_DollarDParen IS just a normal token, handled by ArithParser self._Next(lex_mode_e.Arith) w = osh_word.TokenWord(self.cur_token) return w, False elif self.token_kind in (Kind.Lit, Kind.Left, Kind.VSub): w = self._ReadCompoundWord(lex_mode=lex_mode_e.Arith) return w, False else: assert False, ("Unexpected token parsing arith sub: %s" % self.cur_token) raise AssertionError("Shouldn't get here")
def ParseSetVar(self, kw_token): # type: (token) -> command_t """ setvar a[i] = 1 setvar i += 1 setvar i++ """ self._Next(lex_mode_e.Expr) enode, last_token = self.parse_ctx.ParseOilAssign( self.lexer, grammar_nt.oil_setvar) # Let the CommandParser see the Op_Semi or Op_Newline. self.buffered_word = osh_word.TokenWord(last_token) self._Next(lex_mode_e.ShCommand) # always back to this return enode
def testMultiLine(self): w_parser = _InitWordParser("""\ ls foo # Multiple newlines and comments should be ignored ls bar """) print('--MULTI') w = w_parser.ReadWord(lex_mode_e.Outer) parts = [word_part.LiteralPart(token(Id.Lit_Chars, 'ls'))] test_lib.AssertAsdlEqual(self, osh_word.CompoundWord(parts), w) w = w_parser.ReadWord(lex_mode_e.Outer) parts = [word_part.LiteralPart(token(Id.Lit_Chars, 'foo'))] test_lib.AssertAsdlEqual(self, osh_word.CompoundWord(parts), w) w = w_parser.ReadWord(lex_mode_e.Outer) t = token(Id.Op_Newline, '\n') test_lib.AssertAsdlEqual(self, osh_word.TokenWord(t), w) w = w_parser.ReadWord(lex_mode_e.Outer) parts = [word_part.LiteralPart(token(Id.Lit_Chars, 'ls'))] test_lib.AssertAsdlEqual(self, osh_word.CompoundWord(parts), w) w = w_parser.ReadWord(lex_mode_e.Outer) parts = [word_part.LiteralPart(token(Id.Lit_Chars, 'bar'))] test_lib.AssertAsdlEqual(self, osh_word.CompoundWord(parts), w) w = w_parser.ReadWord(lex_mode_e.Outer) t = token(Id.Op_Newline, '\n') test_lib.AssertAsdlEqual(self, osh_word.TokenWord(t), w) w = w_parser.ReadWord(lex_mode_e.Outer) t = token(Id.Eof_Real, '') test_lib.AssertAsdlEqual(self, osh_word.TokenWord(t), w)
def _assertReadWordWithArena(test, word_str): print('\n---', word_str) arena = test_lib.MakeArena('word_parse_test.py') w_parser = _InitWordParser(word_str, arena=arena) w = w_parser.ReadWord(lex_mode_e.Outer) assert w is not None w.PrettyPrint() # Next word must be Eof_Real w2 = w_parser.ReadWord(lex_mode_e.Outer) test.assertTrue( test_lib.TokenWordsEqual(osh_word.TokenWord(token(Id.Eof_Real, '')), w2), w2) return arena, w
def ParseVar(self, kw_token): # type: (token) -> command_t """ oil_var: 'var' <'oil_var' in grammar.pgen2> Note that assignments must end with a newline or a semicolon. Unlike shell assignments, we disallow: var x = 42 | wc -l var x = 42 && echo hi """ self._Next(lex_mode_e.Expr) enode, last_token = self.parse_ctx.ParseOilAssign( self.lexer, grammar_nt.oil_var) # Let the CommandParser see the Op_Semi or Op_Newline. self.buffered_word = osh_word.TokenWord(last_token) self._Next(lex_mode_e.ShCommand) # always back to this return enode
def _ReadWord(self, lex_mode): # type: (lex_mode_t) -> Tuple[word_t, bool] """Helper function for Read(). Returns: 2-tuple (word, need_more) word: Word, or None if there was an error, or need_more is set need_more: True if the caller should call us again """ self._Peek() if self.token_kind == Kind.Eof: # No advance return osh_word.TokenWord(self.cur_token), False # Allow Arith for ) at end of for loop? elif self.token_kind in (Kind.Op, Kind.Redir, Kind.Arith): self._Next(lex_mode) if self.token_type == Id.Op_Newline: if self.cursor_was_newline: return None, True return osh_word.TokenWord(self.cur_token), False elif self.token_kind == Kind.Right: if self.token_type not in (Id.Right_Subshell, Id.Right_FuncDef, Id.Right_CasePat, Id.Right_ArrayLiteral): raise AssertionError(self.cur_token) self._Next(lex_mode) return osh_word.TokenWord(self.cur_token), False elif self.token_kind in (Kind.Ignored, Kind.WS): self._Next(lex_mode) return None, True # tell Read() to try again elif self.token_kind in (Kind.VSub, Kind.Lit, Kind.History, Kind.Left, Kind.KW, Kind.Assign, Kind.ControlFlow, Kind.BoolUnary, Kind.BoolBinary, Kind.ExtGlob): # We're beginning a word. If we see Id.Lit_Pound, change to # lex_mode_e.Comment and read until end of line. if self.token_type == Id.Lit_Pound: self._Next(lex_mode_e.Comment) self._Peek() # NOTE: The # could be the last character in the file. It can't be # Eof_{RParen,Backtick} because #) and #` are comments. assert self.token_type in (Id.Ignored_Comment, Id.Eof_Real), \ self.cur_token # The next iteration will go into Kind.Ignored and set lex state to # lex_mode_e.Outer/etc. return None, True # tell Read() to try again after comment else: w = self._ReadCompoundWord(lex_mode=lex_mode) return w, False else: raise AssertionError('Unhandled: %s (%s)' % (self.cur_token, self.token_kind)) raise AssertionError("Shouldn't get here")