def _MakeSimpleCommand(preparsed_list, suffix_words, redirects): """Create an command.SimpleCommand node.""" # FOO=(1 2 3) ls is not allowed. for _, _, _, w in preparsed_list: if word.HasArrayPart(w): p_die("Environment bindings can't contain array literals", word=w) # echo FOO=(1 2 3) is not allowed (but we should NOT fail on echo FOO[x]=1). for w in suffix_words: if word.HasArrayPart(w): p_die("Commands can't contain array literals", word=w) # NOTE: # In bash, {~bob,~jane}/src works, even though ~ isn't the leading # character of the initial word. # However, this means we must do tilde detection AFTER brace EXPANSION, not # just after brace DETECTION like we're doing here. # The BracedWordTree instances have to be expanded into CompoundWord # instances for the tilde detection to work. words2 = braces.BraceDetectAll(suffix_words) words3 = word.TildeDetectAll(words2) node = command.SimpleCommand() node.words = words3 node.redirects = redirects _AppendMoreEnv(preparsed_list, node.more_env) return node
def _ReadArrayLiteralPart(self): self._Next(lex_mode_e.Outer) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =, got %r', self.cur_token.val, token=self.cur_token) # MUST use a new word parser (with same lexer). w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.Outer) assert w is not None if w.tag == word_e.TokenWord: word_id = word.CommandId(w) if word_id == Id.Right_ArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: # TokenWord p_die('Unexpected token in array literal: %r', w.token.val, word=w) words.append(w) words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) return word_part.ArrayLiteralPart(words3)
def _ParseForEachLoop(self): node = command.ForEach() node.do_arg_iter = False ok, iter_name, quoted = word.StaticEval(self.cur_word) if not ok or quoted: p_die("Loop variable name should be a constant", word=self.cur_word) if not match.IsValidVarName(iter_name): p_die("Invalid loop variable name", word=self.cur_word) node.iter_name = iter_name self._Next() # skip past name self._NewlineOk() in_spid = const.NO_INTEGER semi_spid = const.NO_INTEGER self._Peek() if self.c_id == Id.KW_In: self._Next() # skip in in_spid = word.LeftMostSpanForWord(self.cur_word) + 1 iter_words, semi_spid = self.ParseForWords() assert iter_words is not None words2 = braces.BraceDetectAll(iter_words) words3 = word.TildeDetectAll(words2) node.iter_words = words3 elif self.c_id == Id.Op_Semi: node.do_arg_iter = True # implicit for loop self._Next() elif self.c_id == Id.KW_Do: node.do_arg_iter = True # implicit for loop # do not advance else: # for foo BAD p_die('Unexpected word after for loop variable', word=self.cur_word) node.spids.extend((in_spid, semi_spid)) body_node = self.ParseDoGroup() assert body_node is not None node.body = body_node return node
def Matches(self, comp): """ Args: comp: Callback args from readline. Readline uses set_completer_delims to tokenize the string. Returns a list of matches relative to readline's completion_delims. We have to post-process the output of various completers. """ arena = self.parse_ctx.arena # Used by inner functions # Pass the original line "out of band" to the completion callback. line_until_tab = comp.line[:comp.end] self.comp_ui_state.line_until_tab = line_until_tab self.parse_ctx.trail.Clear() line_reader = reader.StringLineReader(line_until_tab, self.parse_ctx.arena) c_parser = self.parse_ctx.MakeOshParser(line_reader, emit_comp_dummy=True) # We want the output from parse_ctx, so we don't use the return value. try: c_parser.ParseLogicalLine() except util.ParseError as e: # e.g. 'ls | ' will not parse. Now inspect the parser state! pass debug_f = self.debug_f trail = self.parse_ctx.trail if 1: trail.PrintDebugString(debug_f) # # First try completing the shell language itself. # # NOTE: We get Eof_Real in the command state, but not in the middle of a # BracedVarSub. This is due to the difference between the CommandParser # and WordParser. tokens = trail.tokens last = -1 if tokens[-1].id == Id.Eof_Real: last -= 1 # ignore it try: t1 = tokens[last] except IndexError: t1 = None try: t2 = tokens[last-1] except IndexError: t2 = None debug_f.log('line: %r', comp.line) debug_f.log('rl_slice from byte %d to %d: %r', comp.begin, comp.end, comp.line[comp.begin:comp.end]) debug_f.log('t1 %s', t1) debug_f.log('t2 %s', t2) # Each of the 'yield' statements below returns a fully-completed line, to # appease the readline library. The root cause of this dance: If there's # one candidate, readline is responsible for redrawing the input line. OSH # only displays candidates and never redraws the input line. def _TokenStart(tok): span = arena.GetLineSpan(tok.span_id) return span.col if t2: # We always have t1? # echo $ if IsDollar(t2) and IsDummy(t1): self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $ for name in self.mem.VarNames(): yield line_until_tab + name # no need to quote var names return # echo ${ if t2.id == Id.Left_DollarBrace and IsDummy(t1): self.comp_ui_state.display_pos = _TokenStart(t2) + 2 # 2 for ${ for name in self.mem.VarNames(): yield line_until_tab + name # no need to quote var names return # echo $P if t2.id == Id.VSub_DollarName and IsDummy(t1): # Example: ${undef:-$P # readline splits at ':' so we have to prepend '-$' to every completed # variable name. self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $ to_complete = t2.val[1:] n = len(to_complete) for name in self.mem.VarNames(): if name.startswith(to_complete): yield line_until_tab + name[n:] # no need to quote var names return # echo ${P if t2.id == Id.VSub_Name and IsDummy(t1): self.comp_ui_state.display_pos = _TokenStart(t2) # no offset to_complete = t2.val n = len(to_complete) for name in self.mem.VarNames(): if name.startswith(to_complete): yield line_until_tab + name[n:] # no need to quote var names return # echo $(( VAR if t2.id == Id.Lit_ArithVarLike and IsDummy(t1): self.comp_ui_state.display_pos = _TokenStart(t2) # no offset to_complete = t2.val n = len(to_complete) for name in self.mem.VarNames(): if name.startswith(to_complete): yield line_until_tab + name[n:] # no need to quote var names return if trail.words: # echo ~<TAB> # echo ~a<TAB> $(home dirs) # This must be done at a word level, and TildeDetectAll() does NOT help # here, because they don't have trailing slashes yet! We can't do it on # tokens, because otherwise f~a will complete. Looking at word_part is # EXACTLY what we want. parts = trail.words[-1].parts if (len(parts) == 2 and parts[0].tag == word_part_e.LiteralPart and parts[1].tag == word_part_e.LiteralPart and parts[0].token.id == Id.Lit_TildeLike and parts[1].token.id == Id.Lit_CompDummy): t2 = parts[0].token # +1 for ~ self.comp_ui_state.display_pos = _TokenStart(parts[0].token) + 1 to_complete = t2.val[1:] n = len(to_complete) for u in pwd.getpwall(): # catch errors? name = u.pw_name if name.startswith(to_complete): yield line_until_tab + ShellQuoteB(name[n:]) + '/' return # echo hi > f<TAB> (complete redirect arg) if trail.redirects: r = trail.redirects[-1] # Only complete 'echo >', but not 'echo >&' or 'cat <<' if (r.tag == redir_e.Redir and REDIR_ARG_TYPES[r.op.id] == redir_arg_type_e.Path): if WordEndsWithCompDummy(r.arg_word): debug_f.log('Completing redirect arg') try: val = self.word_ev.EvalWordToString(r.arg_word) except util.FatalRuntimeError as e: debug_f.log('Error evaluating redirect word: %s', e) return if val.tag != value_e.Str: debug_f.log("Didn't get a string from redir arg") return span_id = word.LeftMostSpanForWord(r.arg_word) span = arena.GetLineSpan(span_id) self.comp_ui_state.display_pos = span.col comp.Update(to_complete=val.s) # FileSystemAction uses only this n = len(val.s) action = FileSystemAction(add_slash=True) for name in action.Matches(comp): yield line_until_tab + ShellQuoteB(name[n:]) return # # We're not completing the shell language. Delegate to user-defined # completion for external tools. # # Set below, and set on retries. base_opts = None user_spec = None # Used on retries. partial_argv = [] num_partial = -1 first = None if trail.words: # Now check if we're completing a word! if WordEndsWithCompDummy(trail.words[-1]): debug_f.log('Completing words') # # It didn't look like we need to complete var names, tilde, redirects, # etc. Now try partial_argv, which may involve invoking PLUGINS. # needed to complete paths with ~ words2 = word.TildeDetectAll(trail.words) if 0: debug_f.log('After tilde detection') for w in words2: print(w, file=debug_f) if 0: debug_f.log('words2:') for w2 in words2: debug_f.log(' %s', w2) for w in words2: try: # TODO: # - Should we call EvalWordSequence? But turn globbing off? It # can do splitting and such. # - We could have a variant to eval TildeSubPart to ~ ? val = self.word_ev.EvalWordToString(w) except util.FatalRuntimeError: # Why would it fail? continue if val.tag == value_e.Str: partial_argv.append(val.s) else: pass debug_f.log('partial_argv: %s', partial_argv) num_partial = len(partial_argv) first = partial_argv[0] alias_first = None debug_f.log('alias_words: %s', trail.alias_words) if trail.alias_words: w = trail.alias_words[0] try: val = self.word_ev.EvalWordToString(w) except util.FatalRuntimeError: pass alias_first = val.s debug_f.log('alias_first: %s', alias_first) if num_partial == 0: # should never happen because of Lit_CompDummy raise AssertionError elif num_partial == 1: base_opts, user_spec = self.comp_lookup.GetFirstSpec() # Display/replace since the beginning of the first word. Note: this # is non-zero in the case of # echo $(gr and # echo `gr span_id = word.LeftMostSpanForWord(trail.words[0]) span = arena.GetLineSpan(span_id) self.comp_ui_state.display_pos = span.col self.debug_f.log('** DISPLAY_POS = %d', self.comp_ui_state.display_pos) else: base_opts, user_spec = self.comp_lookup.GetSpecForName(first) if not user_spec and alias_first: base_opts, user_spec = self.comp_lookup.GetSpecForName(alias_first) if user_spec: # Pass the aliased command to the user-defined function, and use # it for retries. first = alias_first if not user_spec: base_opts, user_spec = self.comp_lookup.GetFallback() # Display since the beginning span_id = word.LeftMostSpanForWord(trail.words[-1]) span = arena.GetLineSpan(span_id) self.comp_ui_state.display_pos = span.col self.debug_f.log('words[-1]: %r', trail.words[-1]) self.debug_f.log('display_pos %d', self.comp_ui_state.display_pos) # Update the API for user-defined functions. index = len(partial_argv) - 1 # COMP_CWORD is -1 when it's empty prev = '' if index == 0 else partial_argv[index-1] comp.Update(first=first, to_complete=partial_argv[-1], prev=prev, index=index, partial_argv=partial_argv) # This happens in the case of [[ and ((, or a syntax error like 'echo < >'. if not user_spec: debug_f.log("Didn't find anything to complete") return # Reset it back to what was registered. User-defined functions can mutate # it. dynamic_opts = {} self.compopt_state.dynamic_opts = dynamic_opts self.compopt_state.currently_completing = True try: done = False while not done: try: for candidate in self._PostProcess( base_opts, dynamic_opts, user_spec, comp): yield candidate except _RetryCompletion as e: debug_f.log('Got 124, trying again ...') # Get another user_spec. The ShellFuncAction may have 'sourced' code # and run 'complete' to mutate comp_lookup, and we want to get that # new entry. if num_partial == 0: raise AssertionError elif num_partial == 1: base_opts, user_spec = self.comp_lookup.GetFirstSpec() else: # (already processed alias_first) base_opts, user_spec = self.comp_lookup.GetSpecForName(first) if not user_spec: base_opts, user_spec = self.comp_lookup.GetFallback() else: done = True # exhausted candidates without getting a retry finally: self.compopt_state.currently_completing = False
def Matches(self, comp): """ Args: comp: Callback args from readline. Readline uses set_completer_delims to tokenize the string. Returns a list of matches relative to readline's completion_delims. We have to post-process the output of various completers. """ arena = self.parse_ctx.arena # Used by inner functions self.parse_ctx.trail.Clear() line_reader = reader.StringLineReader(comp.line, self.parse_ctx.arena) c_parser = self.parse_ctx.MakeOshParser(line_reader, emit_comp_dummy=True) # We want the output from parse_ctx, so we don't use the return value. try: c_parser.ParseLogicalLine() except util.ParseError as e: # e.g. 'ls | ' will not parse. Now inspect the parser state! pass debug_f = self.debug_f trail = self.parse_ctx.trail if 1: trail.PrintDebugString(debug_f) # NOTE: We get Eof_Real in the command state, but not in the middle of a # BracedVarSub. This is due to the difference between the CommandParser # and WordParser. tokens = trail.tokens last = -1 if tokens[-1].id == Id.Eof_Real: last -= 1 # ignore it try: t1 = tokens[last] except IndexError: t1 = None try: t2 = tokens[last - 1] except IndexError: t2 = None debug_f.log('line: %r', comp.line) debug_f.log('rl_slice from byte %d to %d: %r', comp.begin, comp.end, comp.line[comp.begin:comp.end]) debug_f.log('t1 %s', t1) debug_f.log('t2 %s', t2) def _MakePrefix(tok, offset=0): span = arena.GetLineSpan(tok.span_id) return comp.line[comp.begin:span.col + offset] #return comp.line[0 : span.col+offset] if t2: # We always have t1? if IsDollar(t2) and IsDummy(t1): prefix = _MakePrefix(t2, offset=1) for name in self.mem.VarNames(): yield prefix + name return # echo ${ if t2.id == Id.Left_VarSub and IsDummy(t1): prefix = _MakePrefix(t2, offset=2) # 2 for ${ for name in self.mem.VarNames(): yield prefix + name return # echo $P if t2.id == Id.VSub_DollarName and IsDummy(t1): # Example: ${undef:-$P # readline splits at ':' so we have to prepend '-$' to every completed # variable name. prefix = _MakePrefix(t2, offset=1) # 1 for $ to_complete = t2.val[1:] for name in self.mem.VarNames(): if name.startswith(to_complete): yield prefix + name return # echo ${P if t2.id == Id.VSub_Name and IsDummy(t1): prefix = _MakePrefix(t2) # no offset to_complete = t2.val for name in self.mem.VarNames(): if name.startswith(to_complete): yield prefix + name return if t2.id == Id.Lit_ArithVarLike and IsDummy(t1): prefix = _MakePrefix(t2) # no offset to_complete = t2.val for name in self.mem.VarNames(): if name.startswith(to_complete): yield prefix + name return # NOTE: Instead of looking at the column positions on line spans, we could # look for IsDummy() on the rightmost LiteralPart(token) of words. def LastColForWord(w): span_id = word.RightMostSpanForWord(w) span = arena.GetLineSpan(span_id) debug_f.log('span %s', span) debug_f.log('span col %d length %d', span.col, span.length) return span.col + span.length if trail.words: # First check if we're completing a path that begins with ~. # # Complete tilde like 'echo ~' and 'echo ~a'. This must be done at a word # level, and TildeDetectAll() does NOT help here, because they don't have # trailing slashes yet! We can't do it on tokens, because otherwise f~a # will complete. Looking at word_part is EXACTLY what we want. parts = trail.words[-1].parts if (len(parts) == 2 and parts[0].tag == word_part_e.LiteralPart and parts[1].tag == word_part_e.LiteralPart and parts[0].token.id == Id.Lit_TildeLike and parts[1].token.id == Id.Lit_CompDummy): t2 = parts[0].token # NOTE: We're assuming readline does its job, and not bothering to # compute the prefix. What are the incorrect corner cases? prefix = '~' to_complete = t2.val[1:] for u in pwd.getpwall(): name = u.pw_name if name.startswith(to_complete): yield prefix + name + '/' return # Check if we should complete a redirect if trail.redirects: r = trail.redirects[-1] # Only complete 'echo >', but not 'echo >&' or 'cat <<' if (r.tag == redir_e.Redir and REDIR_ARG_TYPES[r.op.id] == redir_arg_type_e.Path): last_col = LastColForWord(r.arg_word) if last_col == comp.end: debug_f.log('Completing redirect arg') try: val = self.word_ev.EvalWordToString(r.arg_word) except util.FatalRuntimeError as e: debug_f.log('Error evaluating redirect word: %s', e) return if val.tag != value_e.Str: debug_f.log("Didn't get a string from redir arg") return comp.Update( to_complete=val.s) # FileSystemAction uses only this action = FileSystemAction(add_slash=True) for name in action.Matches(comp): # TODO: form prefix from r.arg_word yield name return base_opts = None user_spec = None # Set below if trail.words: # Now check if we're completing a word! last_col = LastColForWord(trail.words[-1]) debug_f.log('last_col for word: %d', last_col) if last_col == comp.end: # We're not completing the last word! debug_f.log('Completing words') # # It didn't look like we need to complete var names, tilde, redirects, # etc. Now try partial_argv, which may involve invoking PLUGINS. # needed to complete paths with ~ words2 = word.TildeDetectAll(trail.words) if 0: debug_f.log('After tilde detection') for w in words2: print(w, file=debug_f) partial_argv = [] for w in words2: try: # TODO: # - Should we call EvalWordSequence? But turn globbing off? It # can do splitting and such. # - We could have a variant to eval TildeSubPart to ~ ? val = self.word_ev.EvalWordToString(w) except util.FatalRuntimeError: # Why would it fail? continue if val.tag == value_e.Str: partial_argv.append(val.s) else: pass debug_f.log('partial_argv: %s', partial_argv) n = len(partial_argv) # TODO: Form prefix for RootCompleter to add to user_spec candidates if n == 0: # We should never get this because of Lit_CompDummy. raise AssertionError elif n == 1: # First base_opts, user_spec = self.comp_lookup.GetFirstSpec() else: base_opts, user_spec = self.comp_lookup.GetSpecForName( partial_argv[0]) # Update the API for user-defined functions. index = len( partial_argv) - 1 # COMP_CWORD is -1 when it's empty prev = '' if index == 0 else partial_argv[index - 1] comp.Update(first=partial_argv[0], to_complete=partial_argv[-1], prev=prev, index=index, partial_argv=partial_argv) # This happens in the case of [[ and ((, or a syntax error like 'echo < >'. if not user_spec: debug_f.log("Didn't find anything to complete") return # Reset it back to what was registered. User-defined functions can mutate # it. dynamic_opts = {} self.comp_state.dynamic_opts = dynamic_opts self.comp_state.currently_completing = True try: done = False while not done: try: for entry in self._PostProcess(base_opts, dynamic_opts, user_spec, comp): yield entry except _RetryCompletion as e: debug_f.log('Got 124, trying again ...') n = len(partial_argv) # Get another user_spec. The ShellFuncAction may have 'sourced' code # and run 'complete' to mutate comp_lookup, and we want to get that # new entry. if n == 0: raise AssertionError elif n == 1: # First base_opts, user_spec = self.comp_lookup.GetFirstSpec() else: base_opts, user_spec = self.comp_lookup.GetSpecForName( partial_argv[0]) else: done = True # exhausted candidates without getting a retry finally: self.comp_state.currently_completing = False
def _ReadArrayLiteralPart(self): # type: () -> word_part_t """ a=(1 2 3) TODO: See osh/cmd_parse.py:164 for Id.Lit_ArrayLhsOpen, for a[x++]=1 We want: A=(['x']=1 ["x"]=2 [$x$y]=3) Maybe allow this as a literal string? Because I think I've seen it before? Or maybe force people to patch to learn the rule. A=([x]=4) Starts with Lit_Other '[', and then it has Lit_ArrayLhsClose Maybe enforce that ALL have keys or NONE of have keys. """ self._Next(lex_mode_e.ShCommand) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =, got %r', self.cur_token.val, token=self.cur_token) paren_spid = self.cur_token.span_id # MUST use a new word parser (with same lexer). w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if isinstance(w, word__TokenWord): word_id = word.CommandId(w) if word_id == Id.Right_ArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: # TokenWord p_die('Unexpected token in array literal: %r', w.token.val, word=w) assert isinstance(w, word__CompoundWord) # for MyPy words.append(w) if not words: # a=() is empty indexed array node = word_part.ArrayLiteralPart( words) # type: ignore # invariant List? node.spids.append(paren_spid) return node # If the first one is a key/value pair, then the rest are assumed to be. pair = word.DetectAssocPair(words[0]) if pair: pairs = [pair[0], pair[1]] # flat representation n = len(words) for i in xrange(1, n): w = words[i] pair = word.DetectAssocPair(w) if not pair: p_die("Expected associative array pair", word=w) pairs.append(pair[0]) # flat representation pairs.append(pair[1]) node = word_part.AssocArrayLiteral( pairs) # type: ignore # invariant List? node.spids.append(paren_spid) return node words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) node = word_part.ArrayLiteralPart(words3) node.spids.append(paren_spid) return node