def _ReadArithWord(self):
# type: () -> Tuple[word_t, bool]
"""Helper function for ReadWord."""
self._Peek()
if self.token_kind == Kind.Unknown:
p_die('Unexpected token in arithmetic context', token=self.cur_token)
elif self.token_kind == Kind.Eof:
# Just return EOF token
w = word.Token(self.cur_token) # type: word_t
return w, False
elif self.token_kind == Kind.Ignored:
# Space should be ignored. TODO: change this to SPACE_SPACE and
# SPACE_NEWLINE? or SPACE_TOK.
self._Next(lex_mode_e.Arith)
return None, True # Tell wrapper to try again
elif self.token_kind in (Kind.Arith, Kind.Right):
# Id.Right_DollarDParen IS just a normal token, handled by ArithParser
self._Next(lex_mode_e.Arith)
w = word.Token(self.cur_token)
return w, False
elif self.token_kind in (Kind.Lit, Kind.Left, Kind.VSub):
w = self._ReadCompoundWord(lex_mode=lex_mode_e.Arith)
return w, False
else:
assert False, ("Unexpected token parsing arith sub: %s" % self.cur_token)
raise AssertionError("Shouldn't get here")
def ParseCommandExpr(self):
# type: () -> expr_t
enode, last_token = self.parse_ctx.ParseOilExpr(self.lexer,
grammar_nt.command_expr)
if last_token.id == Id.Op_RBrace:
last_token.id = Id.Lit_RBrace
self.buffered_word = word.Token(last_token)
return enode
def _assertReadWordWithArena(test, w_parser):
w = w_parser.ReadWord(lex_mode_e.ShCommand)
assert w is not None
w.PrettyPrint()
# Next word must be Eof_Real
w2 = w_parser.ReadWord(lex_mode_e.ShCommand)
test.assertTrue(
test_lib.TokenWordsEqual(word.Token(token(Id.Eof_Real, '')), w2), w2)
return w
def ParseBareDecl(self):
# type: () -> expr_t
"""
Parse the RHS of x = {name: val}
"""
self._Next(lex_mode_e.Expr)
self._Peek()
enode, last_token = self.parse_ctx.ParseOilExpr(self.lexer,
grammar_nt.command_expr)
if last_token.id == Id.Op_RBrace:
last_token.id = Id.Lit_RBrace
self.buffered_word = word.Token(last_token)
self._Next(lex_mode_e.ShCommand)
return enode
def testMultiLine(self):
w_parser = test_lib.InitWordParser("""\
ls foo
# Multiple newlines and comments should be ignored
ls bar
""")
print('--MULTI')
w = w_parser.ReadWord(lex_mode_e.ShCommand)
parts = [word_part.Literal(token(Id.Lit_Chars, 'ls'))]
test_lib.AssertAsdlEqual(self, word.Compound(parts), w)
w = w_parser.ReadWord(lex_mode_e.ShCommand)
parts = [word_part.Literal(token(Id.Lit_Chars, 'foo'))]
test_lib.AssertAsdlEqual(self, word.Compound(parts), w)
w = w_parser.ReadWord(lex_mode_e.ShCommand)
t = token(Id.Op_Newline, '\n')
test_lib.AssertAsdlEqual(self, word.Token(t), w)
w = w_parser.ReadWord(lex_mode_e.ShCommand)
parts = [word_part.Literal(token(Id.Lit_Chars, 'ls'))]
test_lib.AssertAsdlEqual(self, word.Compound(parts), w)
w = w_parser.ReadWord(lex_mode_e.ShCommand)
parts = [word_part.Literal(token(Id.Lit_Chars, 'bar'))]
test_lib.AssertAsdlEqual(self, word.Compound(parts), w)
w = w_parser.ReadWord(lex_mode_e.ShCommand)
t = token(Id.Op_Newline, '\n')
test_lib.AssertAsdlEqual(self, word.Token(t), w)
w = w_parser.ReadWord(lex_mode_e.ShCommand)
t = token(Id.Eof_Real, '')
test_lib.AssertAsdlEqual(self, word.Token(t), w)
def ParseProc(self, node):
# type: (command__Proc) -> None
# proc name-with-hyphens() must be accepted
self._Next(lex_mode_e.ShCommand)
self._Peek()
# example: 'proc f[' gets you Lit_ArrayLhsOpen
if self.token_type != Id.Lit_Chars:
p_die('Invalid proc name %r', self.cur_token.val, token=self.cur_token)
node.name = self.cur_token
last_token = self.parse_ctx.ParseProc(self.lexer, node)
if last_token.id == Id.Op_LBrace: # Translate to what CommandParser wants
last_token.id = Id.Lit_LBrace
self.buffered_word = word.Token(last_token)
self._Next(lex_mode_e.ShCommand) # required
def ParsePlaceMutation(self, kw_token):
# type: (token) -> command_t
"""
setvar a[i] = 1
setvar i += 1
setvar i++
"""
self._Next(lex_mode_e.Expr)
enode, last_token = self.parse_ctx.ParsePlaceMutation(kw_token, self.lexer)
# Hack to move } from what the Expr lexer modes gives to what CommandParser
# wants
if last_token.id == Id.Op_RBrace:
last_token.id = Id.Lit_RBrace
# Let the CommandParser see the Op_Semi or Op_Newline.
self.buffered_word = word.Token(last_token)
self._Next(lex_mode_e.ShCommand) # always back to this
return enode
def ParseVarDecl(self, kw_token):
# type: (token) -> command_t
"""
oil_var_decl: name_type_list '=' testlist end_stmt
Note that assignments must end with \n ; } or EOF. Unlike shell
assignments, we disallow:
var x = 42 | wc -l
var x = 42 && echo hi
"""
self._Next(lex_mode_e.Expr)
enode, last_token = self.parse_ctx.ParseVarDecl(kw_token, self.lexer)
# Hack to move } from what the Expr lexer modes gives to what CommandParser
# wants
if last_token.id == Id.Op_RBrace:
last_token.id = Id.Lit_RBrace
# Let the CommandParser see the Op_Semi or Op_Newline.
self.buffered_word = word.Token(last_token)
self._Next(lex_mode_e.ShCommand) # always back to this
return enode
def ParseFunc(self, node):
# type: (command__Func) -> None
last_token = self.parse_ctx.ParseFunc(self.lexer, node)
if last_token.id == Id.Op_LBrace: # Translate to what CommandParser wants
last_token.id = Id.Lit_LBrace
self.buffered_word = word.Token(last_token)
def _ReadWord(self, lex_mode):
# type: (lex_mode_t) -> Tuple[word_t, bool]
"""Helper function for Read().
Returns:
2-tuple (word, need_more)
word: Word, or None if there was an error, or need_more is set
need_more: True if the caller should call us again
"""
self._Peek()
if self.token_kind == Kind.Eof:
# No advance
return word.Token(self.cur_token), False
# Allow Arith for ) at end of for loop?
elif self.token_kind in (Kind.Op, Kind.Redir, Kind.Arith):
self._Next(lex_mode)
if self.token_type == Id.Op_Newline:
if self.cursor_was_newline:
return None, True
return word.Token(self.cur_token), False
elif self.token_kind == Kind.Right:
if self.token_type not in (
Id.Right_Subshell, Id.Right_ShFunction, Id.Right_CasePat,
Id.Right_ShArrayLiteral):
raise AssertionError(self.cur_token)
self._Next(lex_mode)
return word.Token(self.cur_token), False
elif self.token_kind in (Kind.Ignored, Kind.WS):
self._Next(lex_mode)
return None, True # tell Read() to try again
elif self.token_kind in (
Kind.VSub, Kind.Lit, Kind.History, Kind.Left, Kind.KW,
Kind.ControlFlow, Kind.BoolUnary, Kind.BoolBinary, Kind.ExtGlob):
# We're beginning a word. If we see Id.Lit_Pound, change to
# lex_mode_e.Comment and read until end of line.
if self.token_type == Id.Lit_Pound:
self._Next(lex_mode_e.Comment)
self._Peek()
# NOTE: The # could be the last character in the file. It can't be
# Eof_{RParen,Backtick} because #) and #` are comments.
assert self.token_type in (Id.Ignored_Comment, Id.Eof_Real), \
self.cur_token
# The next iteration will go into Kind.Ignored and set lex state to
# lex_mode_e.ShCommand/etc.
return None, True # tell Read() to try again after comment
else:
w = self._ReadCompoundWord(lex_mode=lex_mode)
return w, False
else:
raise AssertionError(
'Unhandled: %s (%s)' % (self.cur_token, self.token_kind))
raise AssertionError("Shouldn't get here")