def _ReadArrayLiteralPart(self): self._Next(lex_mode_e.Outer) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =, got %r', self.cur_token.val, token=self.cur_token) # MUST use a new word parser (with same lexer). w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.Outer) assert w is not None if w.tag == word_e.TokenWord: word_id = word.CommandId(w) if word_id == Id.Right_ArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: # TokenWord p_die('Unexpected token in array literal: %r', w.token.val, word=w) words.append(w) words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) return word_part.ArrayLiteralPart(words3)
def ReadWord(self, lex_mode): """Read the next Word. Returns: Word, or None if there was an error """ # Implementation note: This is an stateful/iterative function that calls # the stateless "_ReadWord" function. while True: if lex_mode == lex_mode_e.Arith: # TODO: Can this be unified? w, need_more = self._ReadArithWord() elif lex_mode in (lex_mode_e.Outer, lex_mode_e.DBracket, lex_mode_e.BashRegex): w, need_more = self._ReadWord(lex_mode) else: raise AssertionError('Invalid lex state %s' % lex_mode) if not need_more: break assert w is not None, w self.cursor = w # TODO: Do consolidation of newlines in the lexer? # Note that there can be an infinite (Id.Ignored_Comment Id.Op_Newline # Id.Ignored_Comment Id.Op_Newline) sequence, so we have to keep track of # the last non-ignored token. self.cursor_was_newline = (word.CommandId( self.cursor) == Id.Op_Newline) return self.cursor
def Interactive(opts, ex, c_parser, arena): status = 0 while True: # Reset internal newline state. NOTE: It would actually be correct to # reinitialize all objects (except Env) on every iteration. c_parser.Reset() c_parser.ResetInputObjects() try: w = c_parser.Peek() # may raise HistoryError or ParseError c_id = word.CommandId(w) if c_id == Id.Op_Newline: # print PS1 again, not PS2 continue # next command elif c_id == Id.Eof_Real: # InteractiveLineReader prints ^D break # end node = c_parser.ParseLogicalLine( ) # ditto, HistoryError or ParseError except util.HistoryError as e: # e.g. expansion failed # Where this happens: # for i in 1 2 3; do # !invalid # done print(e.UserErrorString()) continue except util.ParseError as e: ui.PrettyPrintError(e, arena) # NOTE: This should set the status interactively! Bash does this. status = 2 continue if node is None: # EOF # NOTE: We don't care if there are pending here docs in the interative case. break is_control_flow, is_fatal = ex.ExecuteAndCatch(node) status = ex.LastStatus() if is_control_flow: # e.g. 'exit' in the middle of a script break if is_fatal: # e.g. divide by zero continue # TODO: Replace this with a shell hook? with 'trap', or it could be just # like command_not_found. The hook can be 'echo $?' or something more # complicated, i.e. with timetamps. if opts.print_status: print('STATUS', repr(status)) if ex.MaybeRunExitTrap(): return ex.LastStatus() else: return status # could be a parse error
def _Eat(self, c_id): """Consume a word of a type. If it doesn't match, return False. Args: c_id: either EKeyword.* or a token type like Id.Right_Subshell. TODO: Rationalize / type check this. """ self._Peek() # TODO: Printing something like KW_Do is not friendly. We can map # backwards using the _KEYWORDS list in osh/lex.py. if self.c_id != c_id: p_die('Expected word type %s, got %s', c_id, word.CommandId(self.cur_word), word=self.cur_word) self._Next()
def testReadArith(self): CASES = [ '1 + 2', 'a + b', '$a * $b', '${a} * ${b}', '$(echo 1) * $(echo 2)', '`echo 1` + 2', '$((1 + 2)) * $((3 + 4))', "'single quoted'", # Allowed by oil but not bash '"${a}" + "${b}"', # Ditto '$# + $$', # This doesn't work but does in bash -- should be 15 #'$(( $(echo 1)$(echo 2) + 3 ))', '$(( x[0] < 5 ))', '$(( ++i ))', '$(( i++ ))', '$(( x -= 1))', '$(( x |= 1))', '$(( x[0] = 1 ))', '$(( 1 | 0 ))', '$((0x$size))', ] for expr in CASES: print('---') print(expr) print() w_parser = _InitWordParser(expr) w_parser._Next(lex_mode_e.Arith) # Can we remove this requirement? while True: w = w_parser.ReadWord(lex_mode_e.Arith) assert w is not None w.PrettyPrint() if word.CommandId(w) in (Id.Eof_Real, Id.Unknown_Tok): break
def testRead(self): CASES = [ 'ls "foo"', '$(( 1 + 2 ))', '$(echo $(( 1 )) )', # OLD BUG: arith sub within command sub 'echo ${#array[@]} b', # Had a bug here 'echo $(( ${#array[@]} ))', # Bug here # Had a bug: unary minus #'${mounted_disk_regex:0:-1}', 'echo ${@%suffix}', # had a bug here '${@}', 'echo ${var,,}', 'echo ${var,,?}', # Line continuation tests '${\\\nfoo}', # VS_1 '${foo\\\n}', # VS_2 '${foo#\\\nyo}', # VS_ARG_UNQ '"${foo#\\\nyo}"', # VS_ARG_DQ ] for expr in CASES: print('---') print(expr) print() w_parser = _InitWordParser(expr) while True: w = w_parser.ReadWord(lex_mode_e.Outer) assert w is not None w.PrettyPrint() if word.CommandId(w) == Id.Eof_Real: break
def _Peek(self): """Helper method. Returns True for success and False on error. Error examples: bad command sub word, or unterminated quoted string, etc. """ if self.next_lex_mode != lex_mode_e.Undefined: w = self.w_parser.ReadWord(self.next_lex_mode) assert w is not None # Here docs only happen in command mode, so other kinds of newlines don't # count. if w.tag == word_e.TokenWord and w.token.id == Id.Op_Newline: for h in self.pending_here_docs: _ParseHereDocBody(self.parse_ctx, h, self.line_reader, self.arena) del self.pending_here_docs[:] # No .clear() until Python 3.3. self.cur_word = w self.c_kind = word.CommandKind(self.cur_word) self.c_id = word.CommandId(self.cur_word) self.next_lex_mode = lex_mode_e.Undefined
def Interactive(opts, ex, c_parser, display, arena): # type: (Any, Any, CommandParser, Any, Arena) -> Any status = 0 done = False while not done: # - This loop has a an odd structure because we want to do cleanup after # every 'break'. (The ones without 'done = True' were 'continue') # - display.EraseLines() needs to be called BEFORE displaying anything, so # it appears in all branches. while True: # ONLY EXECUTES ONCE try: w = c_parser.Peek() # may raise HistoryError or ParseError c_id = word.CommandId(w) if c_id == Id.Op_Newline: # print PS1 again, not PS2 display.EraseLines() break # next command elif c_id == Id.Eof_Real: # InteractiveLineReader prints ^D display.EraseLines() done = True break # quit shell node = c_parser.ParseLogicalLine( ) # ditto, HistoryError or ParseError except util.HistoryError as e: # e.g. expansion failed # Where this happens: # for i in 1 2 3; do # !invalid # done display.EraseLines() print(e.UserErrorString()) break except util.ParseError as e: display.EraseLines() ui.PrettyPrintError(e, arena) # NOTE: This should set the status interactively! Bash does this. status = 2 break except KeyboardInterrupt: # thrown by InteractiveLineReader._GetLine() # Here we must print a newline BEFORE EraseLines() print('^C') display.EraseLines() # http://www.tldp.org/LDP/abs/html/exitcodes.html # bash gives 130, dash gives 0, zsh gives 1. # Unless we SET ex.last_status, scripts see it, so don't bother now. break if node is None: # EOF display.EraseLines() # NOTE: We don't care about pending here docs in the interative case. done = True break display.EraseLines() # Clear candidates right before executing is_control_flow, is_fatal = ex.ExecuteAndCatch(node) status = ex.LastStatus() if is_control_flow: # e.g. 'exit' in the middle of a script done = True break if is_fatal: # e.g. divide by zero break break # QUIT LOOP after one iteration. # Cleanup after every command (or failed command). # Reset internal newline state. c_parser.Reset() c_parser.ResetInputObjects() display.Reset() # clears dupes and number of lines last displayed # TODO: Replace this with a shell hook? with 'trap', or it could be just # like command_not_found. The hook can be 'echo $?' or something more # complicated, i.e. with timetamps. if opts.print_status: print('STATUS', repr(status)) if ex.MaybeRunExitTrap(): return ex.LastStatus() else: return status # could be a parse error
def _ReadArrayLiteralPart(self): # type: () -> word_part_t """ a=(1 2 3) TODO: See osh/cmd_parse.py:164 for Id.Lit_ArrayLhsOpen, for a[x++]=1 We want: A=(['x']=1 ["x"]=2 [$x$y]=3) Maybe allow this as a literal string? Because I think I've seen it before? Or maybe force people to patch to learn the rule. A=([x]=4) Starts with Lit_Other '[', and then it has Lit_ArrayLhsClose Maybe enforce that ALL have keys or NONE of have keys. """ self._Next(lex_mode_e.ShCommand) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =, got %r', self.cur_token.val, token=self.cur_token) paren_spid = self.cur_token.span_id # MUST use a new word parser (with same lexer). w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if isinstance(w, word__TokenWord): word_id = word.CommandId(w) if word_id == Id.Right_ArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: # TokenWord p_die('Unexpected token in array literal: %r', w.token.val, word=w) assert isinstance(w, word__CompoundWord) # for MyPy words.append(w) if not words: # a=() is empty indexed array node = word_part.ArrayLiteralPart( words) # type: ignore # invariant List? node.spids.append(paren_spid) return node # If the first one is a key/value pair, then the rest are assumed to be. pair = word.DetectAssocPair(words[0]) if pair: pairs = [pair[0], pair[1]] # flat representation n = len(words) for i in xrange(1, n): w = words[i] pair = word.DetectAssocPair(w) if not pair: p_die("Expected associative array pair", word=w) pairs.append(pair[0]) # flat representation pairs.append(pair[1]) node = word_part.AssocArrayLiteral( pairs) # type: ignore # invariant List? node.spids.append(paren_spid) return node words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) node = word_part.ArrayLiteralPart(words3) node.spids.append(paren_spid) return node