def ReadWord(self, lex_mode): """Read the next Word. Returns: Word, or None if there was an error """ # Implementation note: This is an stateful/iterative function that calls # the stateless "_ReadWord" function. while True: if lex_mode == lex_mode_e.ARITH: # TODO: Can this be unified? w, need_more = self._ReadArithWord() elif lex_mode in (lex_mode_e.OUTER, lex_mode_e.DBRACKET, lex_mode_e.BASH_REGEX): w, need_more = self._ReadWord(lex_mode) else: raise AssertionError('Invalid lex state %s' % lex_mode) if not need_more: break if not w: # Assumes AddErrorContext was already called return None self.cursor = w # TODO: Do consolidation of newlines in the lexer? # Note that there can be an infinite (Id.Ignored_Comment Id.Op_Newline # Id.Ignored_Comment Id.Op_Newline) sequence, so we have to keep track of # the last non-ignored token. self.cursor_was_newline = (word.CommandId( self.cursor) == Id.Op_Newline) return self.cursor
def _ReadArrayLiteralPart(self): self._Next(lex_mode_e.OUTER) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: self.AddErrorContext('Expected ( after =', token=self.cur_token) return None # MUST use a new word parser (with same lexer). w_parser = WordParser(self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.OUTER) if not w: self.error_stack.extend(w_parser.Error()) return None if w.tag == word_e.TokenWord: word_id = word.CommandId(w) if word_id == Id.Right_ArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: self.AddErrorContext( 'Unexpected word in array literal: %s', w, word=w) return None words.append(w) words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) return ast.ArrayLiteralPart(words3)
def _Peek(self): """Helper method. Returns True for success and False on error. Error examples: bad command sub word, or unterminated quoted string, etc. """ if self.next_lex_mode != lex_mode_e.NONE: w = self.w_parser.ReadWord(self.next_lex_mode) if w is None: error_stack = self.w_parser.Error() self.error_stack.extend(error_stack) return False # Here docs only happen in command mode, so other kinds of newlines don't # count. if w.tag == word_e.TokenWord and w.token.id == Id.Op_Newline: if not self._MaybeReadHereDocs(): return False self.cur_word = w self.c_kind = word.CommandKind(self.cur_word) self.c_id = word.CommandId(self.cur_word) self.next_lex_mode = lex_mode_e.NONE #print('_Peek', self.cur_word) return True
def _ReadArrayLiteralPart(self): self._Next(lex_mode_e.OUTER) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =, got %r', self.cur_token.val, token=self.cur_token) # MUST use a new word parser (with same lexer). w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.OUTER) assert w is not None if w.tag == word_e.TokenWord: word_id = word.CommandId(w) if word_id == Id.Right_ArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: # TokenWord p_die('Unexpected token in array literal: %r', w.token.val, word=w) words.append(w) words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) return ast.ArrayLiteralPart(words3)
def InteractiveLoop(opts, ex, c_parser, w_parser, line_reader): # Is this correct? Are there any non-ANSI terminals? I guess you can pass # -i but redirect stdout. if opts.ast_output == '-': ast_f = fmt.DetectConsoleOutput(sys.stdout) elif opts.ast_output == '-': f = open(opts.ast_output, 'w') # implicitly closed when the process ends ast_f = fmt.DetectConsoleOutput(f) else: ast_f = None while True: try: w = c_parser.Peek() except KeyboardInterrupt: print('Ctrl-C') break if w is None: raise RuntimeError('Failed parse: %s' % c_parser.Error()) c_id = word.CommandId(w) if c_id == Id.Op_Newline: print('nothing to execute') elif c_id == Id.Eof_Real: print('EOF') break else: node = c_parser.ParseCommandLine() # TODO: Need an error for an empty command, which we ignore? GetLine # could do that in the first position? # ParseSimpleCommand fails with '\n' token? if not node: # TODO: PrintError here raise RuntimeError('failed parse: %s' % c_parser.Error()) if ast_f: ast.PrettyPrint(node) status = ex.Execute(node) if opts.print_status: print('STATUS', repr(status)) # Reset prompt and clear memory. TODO: If there are any function # definitions ANYWHERE in the node, you should not clear the underlying # memory. We still need to execute those strings! line_reader.Reset() # Reset internal newline state. # NOTE: It would actually be correct to reinitialize all objects (except # Env) on every iteration. But we know that the w_parser is the only thing # that needs to be reset, for now. w_parser.Reset() c_parser.Reset()
def InteractiveLoop(opts, ex, c_parser, w_parser, line_reader): if opts.show_ast: ast_f = fmt.DetectConsoleOutput(sys.stdout) else: ast_f = None status = 0 while True: try: w = c_parser.Peek() except KeyboardInterrupt: print('Ctrl-C') break if w is None: raise RuntimeError('Failed parse: %s' % c_parser.Error()) c_id = word.CommandId(w) if c_id == Id.Op_Newline: print('nothing to execute') elif c_id == Id.Eof_Real: print('EOF') break else: node = c_parser.ParseCommandLine() # TODO: Need an error for an empty command, which we ignore? GetLine # could do that in the first position? # ParseSimpleCommand fails with '\n' token? if not node: # TODO: PrintError here raise RuntimeError('failed parse: %s' % c_parser.Error()) if ast_f: ast_lib.PrettyPrint(node) status, is_control_flow = ex.ExecuteAndCatch(node) if is_control_flow: # exit or return break if opts.print_status: print('STATUS', repr(status)) # Reset prompt to PS1. line_reader.Reset() # Reset internal newline state. # NOTE: It would actually be correct to reinitialize all objects (except # Env) on every iteration. But we know that the w_parser is the only thing # that needs to be reset, for now. w_parser.Reset() c_parser.Reset() return status
def _ReadArrayLiteralPart(self): array_part = ast.ArrayLiteralPart() self._Next(LexMode.OUTER) # advance past ( self._Peek() assert self.cur_token.id == Id.Op_LParen, self.cur_token # MUST use a new word parser (with same lexer). w_parser = WordParser(self.lexer, self.line_reader) while True: w = w_parser.ReadWord(LexMode.OUTER) if word.CommandId(w) == Id.Right_ArrayLiteral: break array_part.words.append(w) return array_part
def testReadArith(self): CASES = [ '1 + 2', 'a + b', '$a * $b', '${a} * ${b}', '$(echo 1) * $(echo 2)', '`echo 1` + 2', '$((1 + 2)) * $((3 + 4))', "'single quoted'", # Allowed by oil but not bash '"${a}" + "${b}"', # Ditto '$# + $$', # This doesn't work but does in bash -- should be 15 #'$(( $(echo 1)$(echo 2) + 3 ))', '$(( x[0] < 5 ))', '$(( ++i ))', '$(( i++ ))', '$(( x -= 1))', '$(( x |= 1))', '$(( x[0] = 1 ))', '$(( 1 | 0 ))', '$((0x$size))', ] for expr in CASES: print('---') print(expr) print() w_parser = InitWordParser(expr) w_parser._Next(LexMode.ARITH) # Can we remove this requirement? while True: w = w_parser.ReadWord(LexMode.ARITH) if not w: err = w_parser.Error() print('ERROR', err) self.fail(err) break ast.PrettyPrint(w) if word.CommandId(w) in (Id.Eof_Real, Id.Unknown_Tok): break
def testRead(self): CASES = [ 'ls "foo"', '$(( 1 + 2 ))', '$(echo $(( 1 )) )', # OLD BUG: arith sub within command sub 'echo ${#array[@]} b', # Had a bug here 'echo $(( ${#array[@]} ))', # Bug here # Had a bug: unary minus #'${mounted_disk_regex:0:-1}', 'echo ${@%suffix}', # had a bug here '${@}', 'echo ${var,,}', 'echo ${var,,?}', # Line continuation tests '${\\\nfoo}', # VS_1 '${foo\\\n}', # VS_2 '${foo#\\\nyo}', # VS_ARG_UNQ '"${foo#\\\nyo}"', # VS_ARG_DQ ] for expr in CASES: print('---') print(expr) print() w_parser = InitWordParser(expr) while True: w = w_parser.ReadWord(LexMode.OUTER) if w is None: e = w_parser.Error() print('Error in word parser: %s' % e) self.fail(e) ast.PrettyPrint(w) if word.CommandId(w) == Id.Eof_Real: break
def _Peek(self): """Helper method. Returns True for success and False on error. Error examples: bad command sub word, or unterminated quoted string, etc. """ if self.next_lex_mode != LexMode.NONE: w = self.w_parser.ReadWord(self.next_lex_mode) if w is None: error_stack = self.w_parser.Error() self.error_stack.extend(error_stack) return False self.cur_word = w self.c_kind = word.CommandKind(self.cur_word) self.c_id = word.CommandId(self.cur_word) self.next_lex_mode = LexMode.NONE #print('_Peek', self.cur_word) return True
def _ReadArrayLiteralPart(self): self._Next(LexMode.OUTER) # advance past ( self._Peek() assert self.cur_token.id == Id.Op_LParen, self.cur_token # MUST use a new word parser (with same lexer). w_parser = WordParser(self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(LexMode.OUTER) if word.CommandId(w) == Id.Right_ArrayLiteral: break words.append(w) words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) return ast.ArrayLiteralPart(words3)