def _ReadArithWord(self):
# type: () -> Tuple[word_t, bool]
"""Helper function for ReadWord."""
self._Peek()
if self.token_kind == Kind.Unknown:
p_die('Unexpected token in arithmetic context',
token=self.cur_token)
elif self.token_kind == Kind.Eof:
# Just return EOF token
w = osh_word.TokenWord(self.cur_token) # type: word_t
return w, False
elif self.token_kind == Kind.Ignored:
# Space should be ignored. TODO: change this to SPACE_SPACE and
# SPACE_NEWLINE? or SPACE_TOK.
self._Next(lex_mode_e.Arith)
return None, True # Tell wrapper to try again
elif self.token_kind in (Kind.Arith, Kind.Right):
# Id.Right_DollarDParen IS just a normal token, handled by ArithParser
self._Next(lex_mode_e.Arith)
w = osh_word.TokenWord(self.cur_token)
return w, False
elif self.token_kind in (Kind.Lit, Kind.Left, Kind.VSub):
w = self._ReadCompoundWord(lex_mode=lex_mode_e.Arith)
return w, False
else:
assert False, ("Unexpected token parsing arith sub: %s" %
self.cur_token)
raise AssertionError("Shouldn't get here")
def ParseSetVar(self, kw_token):
# type: (token) -> command_t
"""
setvar a[i] = 1
setvar i += 1
setvar i++
"""
self._Next(lex_mode_e.Expr)
enode, last_token = self.parse_ctx.ParseOilAssign(
self.lexer, grammar_nt.oil_setvar)
# Let the CommandParser see the Op_Semi or Op_Newline.
self.buffered_word = osh_word.TokenWord(last_token)
self._Next(lex_mode_e.ShCommand) # always back to this
return enode
def testMultiLine(self):
w_parser = _InitWordParser("""\
ls foo
# Multiple newlines and comments should be ignored
ls bar
""")
print('--MULTI')
w = w_parser.ReadWord(lex_mode_e.Outer)
parts = [word_part.LiteralPart(token(Id.Lit_Chars, 'ls'))]
test_lib.AssertAsdlEqual(self, osh_word.CompoundWord(parts), w)
w = w_parser.ReadWord(lex_mode_e.Outer)
parts = [word_part.LiteralPart(token(Id.Lit_Chars, 'foo'))]
test_lib.AssertAsdlEqual(self, osh_word.CompoundWord(parts), w)
w = w_parser.ReadWord(lex_mode_e.Outer)
t = token(Id.Op_Newline, '\n')
test_lib.AssertAsdlEqual(self, osh_word.TokenWord(t), w)
w = w_parser.ReadWord(lex_mode_e.Outer)
parts = [word_part.LiteralPart(token(Id.Lit_Chars, 'ls'))]
test_lib.AssertAsdlEqual(self, osh_word.CompoundWord(parts), w)
w = w_parser.ReadWord(lex_mode_e.Outer)
parts = [word_part.LiteralPart(token(Id.Lit_Chars, 'bar'))]
test_lib.AssertAsdlEqual(self, osh_word.CompoundWord(parts), w)
w = w_parser.ReadWord(lex_mode_e.Outer)
t = token(Id.Op_Newline, '\n')
test_lib.AssertAsdlEqual(self, osh_word.TokenWord(t), w)
w = w_parser.ReadWord(lex_mode_e.Outer)
t = token(Id.Eof_Real, '')
test_lib.AssertAsdlEqual(self, osh_word.TokenWord(t), w)
def _assertReadWordWithArena(test, word_str):
print('\n---', word_str)
arena = test_lib.MakeArena('word_parse_test.py')
w_parser = _InitWordParser(word_str, arena=arena)
w = w_parser.ReadWord(lex_mode_e.Outer)
assert w is not None
w.PrettyPrint()
# Next word must be Eof_Real
w2 = w_parser.ReadWord(lex_mode_e.Outer)
test.assertTrue(
test_lib.TokenWordsEqual(osh_word.TokenWord(token(Id.Eof_Real, '')),
w2), w2)
return arena, w
def ParseVar(self, kw_token):
# type: (token) -> command_t
"""
oil_var: 'var' <'oil_var' in grammar.pgen2>
Note that assignments must end with a newline or a semicolon. Unlike shell
assignments, we disallow:
var x = 42 | wc -l
var x = 42 && echo hi
"""
self._Next(lex_mode_e.Expr)
enode, last_token = self.parse_ctx.ParseOilAssign(
self.lexer, grammar_nt.oil_var)
# Let the CommandParser see the Op_Semi or Op_Newline.
self.buffered_word = osh_word.TokenWord(last_token)
self._Next(lex_mode_e.ShCommand) # always back to this
return enode
def _ReadWord(self, lex_mode):
# type: (lex_mode_t) -> Tuple[word_t, bool]
"""Helper function for Read().
Returns:
2-tuple (word, need_more)
word: Word, or None if there was an error, or need_more is set
need_more: True if the caller should call us again
"""
self._Peek()
if self.token_kind == Kind.Eof:
# No advance
return osh_word.TokenWord(self.cur_token), False
# Allow Arith for ) at end of for loop?
elif self.token_kind in (Kind.Op, Kind.Redir, Kind.Arith):
self._Next(lex_mode)
if self.token_type == Id.Op_Newline:
if self.cursor_was_newline:
return None, True
return osh_word.TokenWord(self.cur_token), False
elif self.token_kind == Kind.Right:
if self.token_type not in (Id.Right_Subshell, Id.Right_FuncDef,
Id.Right_CasePat,
Id.Right_ArrayLiteral):
raise AssertionError(self.cur_token)
self._Next(lex_mode)
return osh_word.TokenWord(self.cur_token), False
elif self.token_kind in (Kind.Ignored, Kind.WS):
self._Next(lex_mode)
return None, True # tell Read() to try again
elif self.token_kind in (Kind.VSub, Kind.Lit, Kind.History, Kind.Left,
Kind.KW, Kind.Assign, Kind.ControlFlow,
Kind.BoolUnary, Kind.BoolBinary,
Kind.ExtGlob):
# We're beginning a word. If we see Id.Lit_Pound, change to
# lex_mode_e.Comment and read until end of line.
if self.token_type == Id.Lit_Pound:
self._Next(lex_mode_e.Comment)
self._Peek()
# NOTE: The # could be the last character in the file. It can't be
# Eof_{RParen,Backtick} because #) and #` are comments.
assert self.token_type in (Id.Ignored_Comment, Id.Eof_Real), \
self.cur_token
# The next iteration will go into Kind.Ignored and set lex state to
# lex_mode_e.Outer/etc.
return None, True # tell Read() to try again after comment
else:
w = self._ReadCompoundWord(lex_mode=lex_mode)
return w, False
else:
raise AssertionError('Unhandled: %s (%s)' %
(self.cur_token, self.token_kind))
raise AssertionError("Shouldn't get here")
