def ReadWord(self, lex_mode):
"""Read the next Word.
Returns:
Word, or None if there was an error
"""
# Implementation note: This is an stateful/iterative function that calls
# the stateless "_ReadWord" function.
while True:
if lex_mode == lex_mode_e.ARITH:
# TODO: Can this be unified?
w, need_more = self._ReadArithWord()
elif lex_mode in (lex_mode_e.OUTER, lex_mode_e.DBRACKET,
lex_mode_e.BASH_REGEX):
w, need_more = self._ReadWord(lex_mode)
else:
raise AssertionError('Invalid lex state %s' % lex_mode)
if not need_more:
break
if not w: # Assumes AddErrorContext was already called
return None
self.cursor = w
# TODO: Do consolidation of newlines in the lexer?
# Note that there can be an infinite (Id.Ignored_Comment Id.Op_Newline
# Id.Ignored_Comment Id.Op_Newline) sequence, so we have to keep track of
# the last non-ignored token.
self.cursor_was_newline = (word.CommandId(
self.cursor) == Id.Op_Newline)
return self.cursor
def _ReadArrayLiteralPart(self):
self._Next(lex_mode_e.OUTER) # advance past (
self._Peek()
if self.cur_token.id != Id.Op_LParen:
self.AddErrorContext('Expected ( after =', token=self.cur_token)
return None
# MUST use a new word parser (with same lexer).
w_parser = WordParser(self.lexer, self.line_reader)
words = []
while True:
w = w_parser.ReadWord(lex_mode_e.OUTER)
if not w:
self.error_stack.extend(w_parser.Error())
return None
if w.tag == word_e.TokenWord:
word_id = word.CommandId(w)
if word_id == Id.Right_ArrayLiteral:
break
# Unlike command parsing, array parsing allows embedded \n.
elif word_id == Id.Op_Newline:
continue
else:
self.AddErrorContext(
'Unexpected word in array literal: %s', w, word=w)
return None
words.append(w)
words2 = braces.BraceDetectAll(words)
words3 = word.TildeDetectAll(words2)
return ast.ArrayLiteralPart(words3)
def _Peek(self):
"""Helper method.
Returns True for success and False on error. Error examples: bad command
sub word, or unterminated quoted string, etc.
"""
if self.next_lex_mode != lex_mode_e.NONE:
w = self.w_parser.ReadWord(self.next_lex_mode)
if w is None:
error_stack = self.w_parser.Error()
self.error_stack.extend(error_stack)
return False
# Here docs only happen in command mode, so other kinds of newlines don't
# count.
if w.tag == word_e.TokenWord and w.token.id == Id.Op_Newline:
if not self._MaybeReadHereDocs():
return False
self.cur_word = w
self.c_kind = word.CommandKind(self.cur_word)
self.c_id = word.CommandId(self.cur_word)
self.next_lex_mode = lex_mode_e.NONE
#print('_Peek', self.cur_word)
return True
def _ReadArrayLiteralPart(self):
self._Next(lex_mode_e.OUTER) # advance past (
self._Peek()
if self.cur_token.id != Id.Op_LParen:
p_die('Expected ( after =, got %r',
self.cur_token.val,
token=self.cur_token)
# MUST use a new word parser (with same lexer).
w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader)
words = []
while True:
w = w_parser.ReadWord(lex_mode_e.OUTER)
assert w is not None
if w.tag == word_e.TokenWord:
word_id = word.CommandId(w)
if word_id == Id.Right_ArrayLiteral:
break
# Unlike command parsing, array parsing allows embedded \n.
elif word_id == Id.Op_Newline:
continue
else:
# TokenWord
p_die('Unexpected token in array literal: %r',
w.token.val,
word=w)
words.append(w)
words2 = braces.BraceDetectAll(words)
words3 = word.TildeDetectAll(words2)
return ast.ArrayLiteralPart(words3)
def InteractiveLoop(opts, ex, c_parser, w_parser, line_reader):
# Is this correct? Are there any non-ANSI terminals? I guess you can pass
# -i but redirect stdout.
if opts.ast_output == '-':
ast_f = fmt.DetectConsoleOutput(sys.stdout)
elif opts.ast_output == '-':
f = open(opts.ast_output,
'w') # implicitly closed when the process ends
ast_f = fmt.DetectConsoleOutput(f)
else:
ast_f = None
while True:
try:
w = c_parser.Peek()
except KeyboardInterrupt:
print('Ctrl-C')
break
if w is None:
raise RuntimeError('Failed parse: %s' % c_parser.Error())
c_id = word.CommandId(w)
if c_id == Id.Op_Newline:
print('nothing to execute')
elif c_id == Id.Eof_Real:
print('EOF')
break
else:
node = c_parser.ParseCommandLine()
# TODO: Need an error for an empty command, which we ignore? GetLine
# could do that in the first position?
# ParseSimpleCommand fails with '\n' token?
if not node:
# TODO: PrintError here
raise RuntimeError('failed parse: %s' % c_parser.Error())
if ast_f:
ast.PrettyPrint(node)
status = ex.Execute(node)
if opts.print_status:
print('STATUS', repr(status))
# Reset prompt and clear memory. TODO: If there are any function
# definitions ANYWHERE in the node, you should not clear the underlying
# memory. We still need to execute those strings!
line_reader.Reset()
# Reset internal newline state.
# NOTE: It would actually be correct to reinitialize all objects (except
# Env) on every iteration. But we know that the w_parser is the only thing
# that needs to be reset, for now.
w_parser.Reset()
c_parser.Reset()
def InteractiveLoop(opts, ex, c_parser, w_parser, line_reader):
if opts.show_ast:
ast_f = fmt.DetectConsoleOutput(sys.stdout)
else:
ast_f = None
status = 0
while True:
try:
w = c_parser.Peek()
except KeyboardInterrupt:
print('Ctrl-C')
break
if w is None:
raise RuntimeError('Failed parse: %s' % c_parser.Error())
c_id = word.CommandId(w)
if c_id == Id.Op_Newline:
print('nothing to execute')
elif c_id == Id.Eof_Real:
print('EOF')
break
else:
node = c_parser.ParseCommandLine()
# TODO: Need an error for an empty command, which we ignore? GetLine
# could do that in the first position?
# ParseSimpleCommand fails with '\n' token?
if not node:
# TODO: PrintError here
raise RuntimeError('failed parse: %s' % c_parser.Error())
if ast_f:
ast_lib.PrettyPrint(node)
status, is_control_flow = ex.ExecuteAndCatch(node)
if is_control_flow: # exit or return
break
if opts.print_status:
print('STATUS', repr(status))
# Reset prompt to PS1.
line_reader.Reset()
# Reset internal newline state.
# NOTE: It would actually be correct to reinitialize all objects (except
# Env) on every iteration. But we know that the w_parser is the only thing
# that needs to be reset, for now.
w_parser.Reset()
c_parser.Reset()
return status
def _ReadArrayLiteralPart(self):
array_part = ast.ArrayLiteralPart()
self._Next(LexMode.OUTER) # advance past (
self._Peek()
assert self.cur_token.id == Id.Op_LParen, self.cur_token
# MUST use a new word parser (with same lexer).
w_parser = WordParser(self.lexer, self.line_reader)
while True:
w = w_parser.ReadWord(LexMode.OUTER)
if word.CommandId(w) == Id.Right_ArrayLiteral:
break
array_part.words.append(w)
return array_part
def testReadArith(self):
CASES = [
'1 + 2',
'a + b',
'$a * $b',
'${a} * ${b}',
'$(echo 1) * $(echo 2)',
'`echo 1` + 2',
'$((1 + 2)) * $((3 + 4))',
"'single quoted'", # Allowed by oil but not bash
'"${a}" + "${b}"', # Ditto
'$# + $$',
# This doesn't work but does in bash -- should be 15
#'$(( $(echo 1)$(echo 2) + 3 ))',
'$(( x[0] < 5 ))',
'$(( ++i ))',
'$(( i++ ))',
'$(( x -= 1))',
'$(( x |= 1))',
'$(( x[0] = 1 ))',
'$(( 1 | 0 ))',
'$((0x$size))',
]
for expr in CASES:
print('---')
print(expr)
print()
w_parser = InitWordParser(expr)
w_parser._Next(LexMode.ARITH) # Can we remove this requirement?
while True:
w = w_parser.ReadWord(LexMode.ARITH)
if not w:
err = w_parser.Error()
print('ERROR', err)
self.fail(err)
break
ast.PrettyPrint(w)
if word.CommandId(w) in (Id.Eof_Real, Id.Unknown_Tok):
break
def testRead(self):
CASES = [
'ls "foo"',
'$(( 1 + 2 ))',
'$(echo $(( 1 )) )', # OLD BUG: arith sub within command sub
'echo ${#array[@]} b', # Had a bug here
'echo $(( ${#array[@]} ))', # Bug here
# Had a bug: unary minus
#'${mounted_disk_regex:0:-1}',
'echo ${@%suffix}', # had a bug here
'${@}',
'echo ${var,,}',
'echo ${var,,?}',
# Line continuation tests
'${\\\nfoo}', # VS_1
'${foo\\\n}', # VS_2
'${foo#\\\nyo}', # VS_ARG_UNQ
'"${foo#\\\nyo}"', # VS_ARG_DQ
]
for expr in CASES:
print('---')
print(expr)
print()
w_parser = InitWordParser(expr)
while True:
w = w_parser.ReadWord(LexMode.OUTER)
if w is None:
e = w_parser.Error()
print('Error in word parser: %s' % e)
self.fail(e)
ast.PrettyPrint(w)
if word.CommandId(w) == Id.Eof_Real:
break
def _Peek(self):
"""Helper method.
Returns True for success and False on error. Error examples: bad command
sub word, or unterminated quoted string, etc.
"""
if self.next_lex_mode != LexMode.NONE:
w = self.w_parser.ReadWord(self.next_lex_mode)
if w is None:
error_stack = self.w_parser.Error()
self.error_stack.extend(error_stack)
return False
self.cur_word = w
self.c_kind = word.CommandKind(self.cur_word)
self.c_id = word.CommandId(self.cur_word)
self.next_lex_mode = LexMode.NONE
#print('_Peek', self.cur_word)
return True
def _ReadArrayLiteralPart(self):
self._Next(LexMode.OUTER) # advance past (
self._Peek()
assert self.cur_token.id == Id.Op_LParen, self.cur_token
# MUST use a new word parser (with same lexer).
w_parser = WordParser(self.lexer, self.line_reader)
words = []
while True:
w = w_parser.ReadWord(LexMode.OUTER)
if word.CommandId(w) == Id.Right_ArrayLiteral:
break
words.append(w)
words2 = braces.BraceDetectAll(words)
words3 = word.TildeDetectAll(words2)
return ast.ArrayLiteralPart(words3)
