def record_source(self, word_ob, text, target): pos = word_.LeftMostSpanForWord(word_ob) logger.info("Recording source: %r -> %r", text, target) logger.debug(" position: %d, word object: %r", pos, word_ob) self.word_obs[text] = word_ob self.sources[text].append(pos) self.resolved_source[text] = target
def _ValToIntOrError(self, val, blame_word=None, span_id=runtime.NO_SPID): # type: (value_t, word_t, int) -> int if span_id == runtime.NO_SPID and blame_word: span_id = word_.LeftMostSpanForWord(blame_word) #log('_ValToIntOrError span=%s blame=%s', span_id, blame_word) try: if val.tag == value_e.Undef: # 'nounset' already handled before got here # Happens upon a[undefined]=42, which unfortunately turns into a[0]=42. #log('blame_word %s arena %s', blame_word, self.arena) e_die('Undefined value in arithmetic context', span_id=span_id) if val.tag == value_e.Int: return val.i if val.tag == value_e.Str: return _StringToInteger(val.s, span_id=span_id) # calls e_die except error.FatalRuntime as e: if self.exec_opts.strict_arith: raise else: span_id = word_.SpanIdFromError(e) self.errfmt.PrettyPrintError(e, prefix='warning: ') return 0 # Arrays and associative arrays always fail -- not controlled by strict_arith. # In bash, (( a )) is like (( a[0] )), but I don't want that. # And returning '0' gives different results. e_die("Expected a value convertible to integer, got %s", val, span_id=span_id)
def _assertSpanForWord(test, word_str): arena = test_lib.MakeArena('word_parse_test.py') w_parser = test_lib.InitWordParser(word_str, arena=arena) w = _assertReadWordWithArena(test, w_parser) span_id = word_.LeftMostSpanForWord(w) print(word_str) print(span_id) if span_id != runtime.NO_SPID: span = arena.GetLineSpan(span_id) print(span)
def SpanForArithExpr(node): # type: (arith_expr_t) -> int UP_node = node with tagswitch(node) as case: if case(arith_expr_e.VarRef): token = cast(Token, UP_node) return token.span_id elif case(arith_expr_e.Word): w = cast(compound_word, UP_node) return word_.LeftMostSpanForWord(w) return runtime.NO_SPID
def SpanForArithExpr(node): # type: (arith_expr_t) -> int UP_node = node with tagswitch(node) as case: if case(arith_expr_e.VarRef): # TODO: VarRef should store a token instead of a string! return runtime.NO_SPID elif case(arith_expr_e.ArithWord): node = cast(arith_expr__ArithWord, UP_node) return word_.LeftMostSpanForWord(node.w) return runtime.NO_SPID
def _ValToArithOrError(self, val, blame_word=None, span_id=runtime.NO_SPID): if span_id == runtime.NO_SPID and blame_word: span_id = word_.LeftMostSpanForWord(blame_word) #log('_ValToArithOrError span=%s blame=%s', span_id, blame_word) try: i = self._ValToArith(val, span_id) except util.FatalRuntimeError as e: if self.exec_opts.strict_arith: raise else: i = 0 span_id = word_.SpanIdFromError(e) self.errfmt.PrettyPrintError(e, prefix='warning: ') return i
def _StringToIntegerOrError(self, s, blame_word=None, span_id=runtime.NO_SPID): """Used by both [[ $x -gt 3 ]] and (( $x )).""" if span_id == runtime.NO_SPID and blame_word: span_id = word_.LeftMostSpanForWord(blame_word) try: i = _StringToInteger(s, span_id=span_id) except util.FatalRuntimeError as e: if self.exec_opts.strict_arith: raise else: self.errfmt.PrettyPrintError(e, prefix='warning: ') i = 0 return i
def record_word(self, word_ob, text): pos = word_.LeftMostSpanForWord(word_ob) self.word_obs[text] = word_ob if (consts.LookupSpecialBuiltin(text) == consts.NO_INDEX and consts.LookupAssignBuiltin(text) == consts.NO_INDEX and consts.LookupNormalBuiltin(text) == consts.NO_INDEX): logger.info("Recording command: %r", text) logger.debug(" position: %d, word object: %r", pos, word_ob) self.commands[text].append(pos) else: # TODO: no immediate use since I'm no longer patching builtins # but there may still be utility in recording builtins a script # depends on. This would support spotting function/alias # clashes and such. self.builtins[text].append(pos)
def _StringToIntegerOrError(self, s, blame_word=None): # type: (str, Optional[word_t]) -> int """Used by both [[ $x -gt 3 ]] and (( $x )).""" if blame_word: span_id = word_.LeftMostSpanForWord(blame_word) else: span_id = runtime.NO_SPID try: i = self._StringToInteger(s, span_id=span_id) except error.Strict as e: if self.always_strict or self.exec_opts.strict_arith(): raise else: i = 0 return i
def _VarRefOrWord(self, anode): # type: (arith_expr_t) -> Tuple[Optional[str], int] """ Returns (var_name, span_id) if the arith node can be interpreted that way """ UP_anode = anode with tagswitch(anode) as case: if case(arith_expr_e.VarRef): tok = cast(Token, UP_anode) return (tok.val, tok.span_id) elif case(arith_expr_e.Word): w = cast(compound_word, UP_anode) var_name = self.EvalWordToString(w) span_id = word_.LeftMostSpanForWord(w) return (var_name, span_id) no_str = None # type: str return (no_str, runtime.NO_SPID)
def DoCommand(self, node, local_symbols, at_top_level=False): if node.tag == command_e.CommandList: # TODO: How to distinguish between echo hi; echo bye; and on separate # lines for child in node.children: self.DoCommand(child, local_symbols, at_top_level=at_top_level) elif node.tag == command_e.Simple: # How to preserve spaces between words? Do you want to do it? # Well you need to test this: # # echo foo \ # bar # TODO: Need to print until the left most part of the phrase? the phrase # is a word, binding, redirect. #self.cursor.PrintUntil() if node.more_env: (left_spid, ) = node.more_env[0].spids self.cursor.PrintUntil(left_spid) self.f.write('env ') # We only need to transform the right side, not left side. for pair in node.more_env: self.DoWordInCommand(pair.val, local_symbols) # More translations: # - . to source # - eval to sh-eval if node.words: first_word = node.words[0] ok, val, quoted = word_.StaticEval(first_word) word0_spid = word_.LeftMostSpanForWord(first_word) if ok and not quoted: if val == '[': last_word = node.words[-1] # Check if last word is ] ok, val, quoted = word_.StaticEval(last_word) if ok and not quoted and val == ']': # Replace [ with 'test' self.cursor.PrintUntil(word0_spid) self.cursor.SkipUntil(word0_spid + 1) self.f.write('test') for w in node.words[1:-1]: self.DoWordInCommand(w, local_symbols) # Now omit ] last_spid = word_.LeftMostSpanForWord(last_word) self.cursor.PrintUntil(last_spid - 1) # Get the space before self.cursor.SkipUntil(last_spid + 1) # ] takes one spid return else: raise RuntimeError('Got [ without ]') elif val == '.': self.cursor.PrintUntil(word0_spid) self.cursor.SkipUntil(word0_spid + 1) self.f.write('source') return for w in node.words: self.DoWordInCommand(w, local_symbols) # NOTE: This will change to "phrase"? Word or redirect. for r in node.redirects: self.DoRedirect(r, local_symbols) # TODO: Print the terminator. Could be \n or ; # Need to print env like PYTHONPATH = 'foo' && ls # Need to print redirects: # < > are the same. << is here string, and >> is assignment. # append is >+ # TODO: static_eval of simple command # - [ -> "test". Eliminate trailing ]. # - . -> source, etc. elif node.tag == command_e.ShAssignment: self.DoShAssignment(node, at_top_level, local_symbols) elif node.tag == command_e.Pipeline: # Obscure: |& turns into |- or |+ for stderr. # TODO: # if ! true; then -> if not true { # if ! echo | grep; then -> if not { echo | grep } { # } # not is like do {}, but it negates the return value I guess. for child in node.children: self.DoCommand(child, local_symbols) elif node.tag == command_e.AndOr: for child in node.children: self.DoCommand(child, local_symbols) elif node.tag == command_e.Sentence: # 'ls &' to 'fork ls' # Keep ; the same. self.DoCommand(node.child, local_symbols) # This has to be different in the function case. elif node.tag == command_e.BraceGroup: # { echo hi; } -> do { echo hi } # For now it might be OK to keep 'do { echo hi; } #left_spid, right_spid = node.spids (left_spid, ) = node.spids self.cursor.PrintUntil(left_spid) self.cursor.SkipUntil(left_spid + 1) self.f.write('do {') for child in node.children: self.DoCommand(child, local_symbols) elif node.tag == command_e.Subshell: # (echo hi) -> shell echo hi # (echo hi; echo bye) -> shell {echo hi; echo bye} (left_spid, right_spid) = node.spids self.cursor.PrintUntil(left_spid) self.cursor.SkipUntil(left_spid + 1) self.f.write('shell {') self.DoCommand(node.child, local_symbols) #self._DebugSpid(right_spid) #self._DebugSpid(right_spid + 1) #print('RIGHT SPID', right_spid) self.cursor.PrintUntil(right_spid) self.cursor.SkipUntil(right_spid + 1) self.f.write('}') elif node.tag == command_e.DParen: # (( a == 0 )) is sh-expr ' a == 0 ' # # NOTE: (( n++ )) is auto-translated to sh-expr 'n++', but could be set # n++. left_spid, right_spid = node.spids self.cursor.PrintUntil(left_spid) self.cursor.SkipUntil(left_spid + 1) self.f.write("sh-expr '") self.cursor.PrintUntil(right_spid - 1) # before )) self.cursor.SkipUntil(right_spid + 1) # after )) -- each one is a token self.f.write("'") elif node.tag == command_e.DBracket: # [[ 1 -eq 2 ]] to (1 == 2) self.DoBoolExpr(node.expr) elif node.tag == command_e.ShFunction: # TODO: skip name #self.f.write('proc %s' % node.name) # New symbol table for every function. new_local_symbols = {} # Should be the left most span, including 'function' self.cursor.PrintUntil(node.spids[0]) self.f.write('proc ') self.f.write(node.name) self.cursor.SkipUntil(node.spids[2]) if node.body.tag == command_e.BraceGroup: # Don't add "do" like a standalone brace group. Just use {}. for child in node.body.children: self.DoCommand(child, new_local_symbols) else: pass # Add {}. # proc foo { # shell {echo hi; echo bye} # } #self.DoCommand(node.body) elif node.tag == command_e.BraceGroup: for child in node.children: self.DoCommand(child, local_symbols) elif node.tag == command_e.DoGroup: do_spid, done_spid = node.spids self.cursor.PrintUntil(do_spid) self.cursor.SkipUntil(do_spid + 1) self.f.write('{') for child in node.children: self.DoCommand(child, local_symbols) self.cursor.PrintUntil(done_spid) self.cursor.SkipUntil(done_spid + 1) self.f.write('}') elif node.tag == command_e.ForEach: # Need to preserve spaces between words, because there can be line # wrapping. # for x in a b c \ # d e f; do _, in_spid, semi_spid = node.spids if in_spid == runtime.NO_SPID: #self.cursor.PrintUntil() # 'for x' and then space self.f.write('for %s in @Argv ' % node.iter_name) self.cursor.SkipUntil(node.body.spids[0]) else: self.cursor.PrintUntil(in_spid + 1) # 'for x in' and then space self.f.write('[') for w in node.iter_words: self.DoWordInCommand(w, local_symbols) self.f.write(']') #print("SKIPPING SEMI %d" % semi_spid, file=sys.stderr) if semi_spid != runtime.NO_SPID: self.cursor.PrintUntil(semi_spid) self.cursor.SkipUntil(semi_spid + 1) self.DoCommand(node.body, local_symbols) elif node.tag == command_e.ForExpr: # Change (( )) to ( ), and then _FixDoGroup pass elif node.tag == command_e.WhileUntil: # Skip 'until', and replace it with 'while not' if node.keyword.id == Id.KW_Until: kw_spid = node.keyword.span_id self.cursor.PrintUntil(kw_spid) self.f.write('while not') self.cursor.SkipUntil(kw_spid + 1) if node.cond.tag_() == condition_e.Shell: commands = node.cond.commands # Skip the semi-colon in the condition, which is ususally a Sentence if len(commands) == 1 and commands[0].tag_( ) == command_e.Sentence: self.DoCommand(commands[0].child, local_symbols) semi_spid = commands[0].terminator.span_id self.cursor.SkipUntil(semi_spid + 1) self.DoCommand(node.body, local_symbols) elif node.tag == command_e.If: else_spid, fi_spid = node.spids # if foo; then -> if foo { # elif foo; then -> } elif foo { for i, arm in enumerate(node.arms): elif_spid, then_spid = arm.spids if i != 0: # 'if' not 'elif' on the first arm self.cursor.PrintUntil(elif_spid) self.f.write('} ') cond = arm.cond if cond.tag_() == condition_e.Shell: if len( cond.commands ) == 1 and cond.commands[0].tag == command_e.Sentence: sentence = cond.commands[0] self.DoCommand(sentence, local_symbols) # Remove semi-colon semi_spid = sentence.terminator.span_id self.cursor.PrintUntil(semi_spid) self.cursor.SkipUntil(semi_spid + 1) else: for child in cond.commands: self.DoCommand(child, local_symbols) self.cursor.PrintUntil(then_spid) self.cursor.SkipUntil(then_spid + 1) self.f.write('{') for child in arm.action: self.DoCommand(child, local_symbols) # else -> } else { if node.else_action: self.cursor.PrintUntil(else_spid) self.f.write('} ') self.cursor.PrintUntil(else_spid + 1) self.f.write(' {') for child in node.else_action: self.DoCommand(child, local_symbols) # fi -> } self.cursor.PrintUntil(fi_spid) self.cursor.SkipUntil(fi_spid + 1) self.f.write('}') elif node.tag == command_e.Case: case_spid, in_spid, esac_spid = node.spids self.cursor.PrintUntil(case_spid) self.cursor.SkipUntil(case_spid + 1) self.f.write('match') # Reformat "$1" to $1 self.DoWordInCommand(node.to_match, local_symbols) self.cursor.PrintUntil(in_spid) self.cursor.SkipUntil(in_spid + 1) self.f.write('{') # matchstr $var { # each arm needs the ) and the ;; node to skip over? for arm in node.arms: left_spid, rparen_spid, dsemi_spid, last_spid = arm.spids #print(left_spid, rparen_spid, dsemi_spid) self.cursor.PrintUntil(left_spid) # Hm maybe keep | because it's semi-deprecated? You acn use # reload|force-relaod { # } # e/reload|force-reload/ { # } # / 'reload' or 'force-reload' / { # } # # Yeah it's the more abbreviated syntax. # change | to 'or' for pat in arm.pat_list: pass self.f.write('with ') # Remove the ) self.cursor.PrintUntil(rparen_spid) self.cursor.SkipUntil(rparen_spid + 1) for child in arm.action: self.DoCommand(child, local_symbols) if dsemi_spid != runtime.NO_SPID: # Remove ;; self.cursor.PrintUntil(dsemi_spid) self.cursor.SkipUntil(dsemi_spid + 1) elif last_spid != runtime.NO_SPID: self.cursor.PrintUntil(last_spid) else: raise AssertionError( "Expected with dsemi_spid or last_spid in case arm") self.cursor.PrintUntil(esac_spid) self.cursor.SkipUntil(esac_spid + 1) self.f.write('}') # strmatch $var { elif node.tag == command_e.NoOp: pass elif node.tag == command_e.ControlFlow: # No change for break / return / continue pass elif node.tag == command_e.TimeBlock: self.DoCommand(node.pipeline, local_symbols) else: #log('Command not handled: %s', node) raise AssertionError(node.__class__.__name__)
def DoShAssignment(self, node, at_top_level, local_symbols): """ local_symbols: - Add every 'local' declaration to it - problem: what if you have local in an "if" ? - we could treat it like nested scope and see what happens? Do any programs have a problem with it? case/if/for/while/BraceGroup all define scopes or what? You don't want inconsistency of variables that could be defined at any point. - or maybe you only need it within "if / case" ? Well I guess for/while can break out of the loop and cause problems. A break is an "if". - for subsequent """ # Change RHS to expression language. Bare words not allowed. foo -> 'foo' has_rhs = False # TODO: This is on a per-variable basis. # local foo -> var foo = '' # readonly foo -> setconst foo # export foo -> export foo # TODO: # - This depends on self.mode. # - And we also need the enclosing ShFunction node to analyze. # - or we need a symbol table for the current function. Forget about # # Oil keywords: # - global : scope qualifier # - var, const : mutability # - export : state mutation # - setconst -- make a variable mutable. or maybe freeze var? # # NOTE: Bash also has "unset". Does anyone use it? # You can use "delete" like Python I guess. It's not the opposite of # set. # NOTE: # - We CAN tell if a variable has been defined locally. # - We CANNOT tell if it's been defined globally, because different files # share the same global namespace, and we can't statically figure out what # files are in the program. defined_locally = False # is it a local variable in this function? # can't tell if global # DISABLED after doing dynamic assignments. We could reconstruct these # from SimpleCommand? Look at argv[0] and then do static parsing to # assign_pair? if 0: # Assume that 'local' it's a declaration. In osh, it's an error if # locals are redefined. In bash, it's OK to do 'local f=1; local f=2'. # Could have a flag if enough people do this. if at_top_level: raise RuntimeError('local at top level is invalid') if defined_locally: raise RuntimeError("Can't redefine local") keyword_spid = node.spids[0] self.cursor.PrintUntil(keyword_spid) self.cursor.SkipUntil(keyword_spid + 1) self.f.write('var') if local_symbols is not None: for pair in node.pairs: # NOTE: Not handling local a[b]=c if pair.lhs.tag == sh_lhs_expr_e.Name: #print("REGISTERED %s" % pair.lhs.name) local_symbols[pair.lhs.name] = True elif 1: self.cursor.PrintUntil(node.spids[0]) # For now, just detect whether the FIRST assignment on the line has been # declared locally. We might want to split every line into separate # statements. if local_symbols is not None: lhs0 = node.pairs[0].lhs if lhs0.tag == sh_lhs_expr_e.Name and lhs0.name in local_symbols: defined_locally = True #print("CHECKING NAME", lhs0.name, defined_locally, local_symbols) has_array = any(pair.lhs.tag == sh_lhs_expr_e.UnparsedIndex for pair in node.pairs) # need semantic analysis. # Would be nice to assume that it's a local though. if has_array: self.f.write('compat ') # 'compat array-assign' syntax elif at_top_level: self.f.write('setglobal ') elif defined_locally: self.f.write('set ') #self.f.write('[local mutated]') else: # We're in a function, but it's not defined locally, so we must be # mutatting a global. self.f.write('setglobal ') elif 0: # Explicit const. Assume it can't be redefined. # Verb. # # Top level; # readonly FOO=bar -> const FOO = 'bar' # readonly FOO -> freeze FOO # function level: # readonly FOO=bar -> const global FOO ::= 'bar' # readonly FOO -> freeze FOO keyword_spid = node.spids[0] if at_top_level: self.cursor.PrintUntil(keyword_spid) self.cursor.SkipUntil(keyword_spid + 1) self.f.write('const') elif defined_locally: # TODO: Actually we might want 'freeze here. In bash, you can make a # variable readonly after its defined. raise RuntimeError("Constant redefined locally") else: # Same as global level self.cursor.PrintUntil(keyword_spid) self.cursor.SkipUntil(keyword_spid + 1) self.f.write('const') elif 0: # declare -rx foo spam=eggs # export foo # setconst foo # # spam = eggs # export spam # Have to parse the flags self.f.write('TODO ') # foo=bar spam=eggs -> foo = 'bar', spam = 'eggs' n = len(node.pairs) for i, pair in enumerate(node.pairs): if pair.lhs.tag == sh_lhs_expr_e.Name: left_spid = pair.spids[0] self.cursor.PrintUntil(left_spid) # Assume skipping over one Lit_VarLike token self.cursor.SkipUntil(left_spid + 1) # Replace name. I guess it's Lit_Chars. self.f.write(pair.lhs.name) self.f.write(' = ') # TODO: This should be translated from Empty. if pair.rhs.tag == word_e.Empty: self.f.write("''") # local i -> var i = '' else: self.DoWordAsExpr(pair.rhs, local_symbols) elif pair.lhs.tag == sh_lhs_expr_e.UnparsedIndex: # NOTES: # - parse_ctx.one_pass_parse should be on, so the span invariant # is accurate # - Then do the following translation: # a[x+1]="foo $bar" -> # compat array-assign a 'x+1' "$foo $bar" # This avoids dealing with nested arenas. # # TODO: This isn't great when there are multiple assignments. # a[x++]=1 b[y++]=2 # # 'compat' could apply to the WHOLE statement, with multiple # assignments. self.f.write("array-assign %s '%s' " % (pair.lhs.name, pair.lhs.index)) if pair.rhs.tag == word_e.Empty: self.f.write("''") # local i -> var i = '' else: rhs_spid = word_.LeftMostSpanForWord(pair.rhs) self.cursor.SkipUntil(rhs_spid) self.DoWordAsExpr(pair.rhs, local_symbols) else: raise AssertionError(pair.lhs.__class__.__name__) if i != n - 1: self.f.write(',')
def DoRedirect(self, node, local_symbols): #print(node, file=sys.stderr) op_spid = node.op.span_id op_id = node.op.id self.cursor.PrintUntil(op_spid) # TODO: # - Do < and <& the same way. # - How to handle here docs and here docs? # - >> becomes >+ or >-, or maybe >>> #if node.tag == redir_e.Redir: if False: if node.fd == runtime.NO_SPID: if op_id == Id.Redir_Great: self.f.write('>') # Allow us to replace the operator self.cursor.SkipUntil(op_spid + 1) elif op_id == Id.Redir_GreatAnd: self.f.write('> !') # Replace >& 2 with > !2 spid = word_.LeftMostSpanForWord(node.arg_word) self.cursor.SkipUntil(spid) #self.DoWordInCommand(node.arg_word) else: # NOTE: Spacing like !2>err.txt vs !2 > err.txt can be done in the # formatter. self.f.write('!%d ' % node.fd) if op_id == Id.Redir_Great: self.f.write('>') self.cursor.SkipUntil(op_spid + 1) elif op_id == Id.Redir_GreatAnd: self.f.write('> !') # Replace 1>& 2 with !1 > !2 spid = word_.LeftMostSpanForWord(node.arg_word) self.cursor.SkipUntil(spid) self.DoWordInCommand(node.arg_word, local_symbols) #elif node.tag == redir_e.HereDoc: elif False: ok, delimiter, delim_quoted = word_.StaticEval(node.here_begin) if not ok: p_die('Invalid here doc delimiter', word=node.here_begin) # Turn everything into <<. We just change the quotes self.f.write('<<') #here_begin_spid2 = word_.RightMostSpanForWord(node.here_begin) if delim_quoted: self.f.write(" '''") else: self.f.write(' """') delim_end_spid = word_.RightMostSpanForWord(node.here_begin) self.cursor.SkipUntil(delim_end_spid + 1) #self.cursor.SkipUntil(here_begin_spid + 1) # Now print the lines. TODO: Have a flag to indent these to the level of # the owning command, e.g. # cat <<EOF # EOF # Or since most here docs are the top level, you could just have a hack # for a fixed indent? TODO: Look at real use cases. for part in node.stdin_parts: self.DoWordPart(part, local_symbols) self.cursor.SkipUntil(node.here_end_span_id + 1) if delim_quoted: self.f.write("'''\n") else: self.f.write('"""\n') # Need #self.cursor.SkipUntil(here_end_spid2) else: raise AssertionError(node.__class__.__name__) # <<< 'here word' # << 'here word' # # 2> out.txt # !2 > out.txt # cat 1<< EOF # hello $name # EOF # cat !1 << """ # hello $name # """ # # cat << 'EOF' # no expansion # EOF # cat <<- 'EOF' # no expansion and indented # # cat << ''' # no expansion # ''' # cat << ''' # no expansion and indented # ''' # Warn about multiple here docs on a line. # As an obscure feature, allow # cat << \'ONE' << \"TWO" # 123 # ONE # 234 # TWO # The _ is an indicator that it's not a string to be piped in. pass
def Matches(self, comp): # type: (Api) -> Iterator[Union[Iterator, Iterator[str]]] """ Args: comp: Callback args from readline. Readline uses set_completer_delims to tokenize the string. Returns a list of matches relative to readline's completion_delims. We have to post-process the output of various completers. """ arena = self.parse_ctx.arena # Used by inner functions # Pass the original line "out of band" to the completion callback. line_until_tab = comp.line[:comp.end] self.comp_ui_state.line_until_tab = line_until_tab self.parse_ctx.trail.Clear() line_reader = reader.StringLineReader(line_until_tab, self.parse_ctx.arena) c_parser = self.parse_ctx.MakeOshParser(line_reader, emit_comp_dummy=True) # We want the output from parse_ctx, so we don't use the return value. try: c_parser.ParseLogicalLine() except error.Parse as e: # e.g. 'ls | ' will not parse. Now inspect the parser state! pass debug_f = self.debug_f trail = self.parse_ctx.trail if 1: trail.PrintDebugString(debug_f) # # First try completing the shell language itself. # # NOTE: We get Eof_Real in the command state, but not in the middle of a # BracedVarSub. This is due to the difference between the CommandParser # and WordParser. tokens = trail.tokens last = -1 if tokens[-1].id == Id.Eof_Real: last -= 1 # ignore it try: t1 = tokens[last] except IndexError: t1 = None try: t2 = tokens[last - 1] except IndexError: t2 = None debug_f.log('line: %r', comp.line) debug_f.log('rl_slice from byte %d to %d: %r', comp.begin, comp.end, comp.line[comp.begin:comp.end]) debug_f.log('t1 %s', t1) debug_f.log('t2 %s', t2) # Each of the 'yield' statements below returns a fully-completed line, to # appease the readline library. The root cause of this dance: If there's # one candidate, readline is responsible for redrawing the input line. OSH # only displays candidates and never redraws the input line. def _TokenStart(tok): # type: (Token) -> int span = arena.GetLineSpan(tok.span_id) return span.col if t2: # We always have t1? # echo $ if IsDollar(t2) and IsDummy(t1): self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $ for name in self.mem.VarNames(): yield line_until_tab + name # no need to quote var names return # echo ${ if t2.id == Id.Left_DollarBrace and IsDummy(t1): self.comp_ui_state.display_pos = _TokenStart( t2) + 2 # 2 for ${ for name in self.mem.VarNames(): yield line_until_tab + name # no need to quote var names return # echo $P if t2.id == Id.VSub_DollarName and IsDummy(t1): # Example: ${undef:-$P # readline splits at ':' so we have to prepend '-$' to every completed # variable name. self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $ to_complete = t2.val[1:] n = len(to_complete) for name in self.mem.VarNames(): if name.startswith(to_complete): yield line_until_tab + name[ n:] # no need to quote var names return # echo ${P if t2.id == Id.VSub_Name and IsDummy(t1): self.comp_ui_state.display_pos = _TokenStart(t2) # no offset to_complete = t2.val n = len(to_complete) for name in self.mem.VarNames(): if name.startswith(to_complete): yield line_until_tab + name[ n:] # no need to quote var names return # echo $(( VAR if t2.id == Id.Lit_ArithVarLike and IsDummy(t1): self.comp_ui_state.display_pos = _TokenStart(t2) # no offset to_complete = t2.val n = len(to_complete) for name in self.mem.VarNames(): if name.startswith(to_complete): yield line_until_tab + name[ n:] # no need to quote var names return if trail.words: # echo ~<TAB> # echo ~a<TAB> $(home dirs) # This must be done at a word level, and TildeDetectAll() does NOT help # here, because they don't have trailing slashes yet! We can't do it on # tokens, because otherwise f~a will complete. Looking at word_part is # EXACTLY what we want. parts = trail.words[-1].parts if (len(parts) == 2 and parts[0].tag_() == word_part_e.Literal and parts[1].tag_() == word_part_e.Literal and parts[0].id == Id.Lit_TildeLike and parts[1].id == Id.Lit_CompDummy): t2 = parts[0] # +1 for ~ self.comp_ui_state.display_pos = _TokenStart(parts[0]) + 1 to_complete = t2.val[1:] n = len(to_complete) for u in pwd.getpwall(): # catch errors? name = u.pw_name if name.startswith(to_complete): yield line_until_tab + ShellQuoteB(name[n:]) + '/' return # echo hi > f<TAB> (complete redirect arg) if trail.redirects: r = trail.redirects[-1] # Only complete 'echo >', but not 'echo >&' or 'cat <<' # TODO: Don't complete <<< 'h' if (r.arg.tag_() == redir_param_e.Word and consts.RedirArgType(r.op.id) == redir_arg_type_e.Path): arg_word = r.arg if WordEndsWithCompDummy(arg_word): debug_f.log('Completing redirect arg') try: val = self.word_ev.EvalWordToString(r.arg) except error.FatalRuntime as e: debug_f.log('Error evaluating redirect word: %s', e) return if val.tag_() != value_e.Str: debug_f.log("Didn't get a string from redir arg") return span_id = word_.LeftMostSpanForWord(arg_word) span = arena.GetLineSpan(span_id) self.comp_ui_state.display_pos = span.col comp.Update( to_complete=val.s) # FileSystemAction uses only this n = len(val.s) action = FileSystemAction(add_slash=True) for name in action.Matches(comp): yield line_until_tab + ShellQuoteB(name[n:]) return # # We're not completing the shell language. Delegate to user-defined # completion for external tools. # # Set below, and set on retries. base_opts = None user_spec = None # Used on retries. partial_argv = [] num_partial = -1 first = None if trail.words: # Now check if we're completing a word! if WordEndsWithCompDummy(trail.words[-1]): debug_f.log('Completing words') # # It didn't look like we need to complete var names, tilde, redirects, # etc. Now try partial_argv, which may involve invoking PLUGINS. # needed to complete paths with ~ words2 = word_.TildeDetectAll(trail.words) if 0: debug_f.log('After tilde detection') for w in words2: print(w, file=debug_f) if 0: debug_f.log('words2:') for w2 in words2: debug_f.log(' %s', w2) for w in words2: try: # TODO: # - Should we call EvalWordSequence? But turn globbing off? It # can do splitting and such. # - We could have a variant to eval TildeSub to ~ ? val = self.word_ev.EvalWordToString(w) except error.FatalRuntime: # Why would it fail? continue if val.tag_() == value_e.Str: partial_argv.append(val.s) else: pass debug_f.log('partial_argv: %s', partial_argv) num_partial = len(partial_argv) first = partial_argv[0] alias_first = None debug_f.log('alias_words: %s', trail.alias_words) if trail.alias_words: w = trail.alias_words[0] try: val = self.word_ev.EvalWordToString(w) except error.FatalRuntime: pass alias_first = val.s debug_f.log('alias_first: %s', alias_first) if num_partial == 0: # should never happen because of Lit_CompDummy raise AssertionError() elif num_partial == 1: base_opts, user_spec = self.comp_lookup.GetFirstSpec() # Display/replace since the beginning of the first word. Note: this # is non-zero in the case of # echo $(gr and # echo `gr span_id = word_.LeftMostSpanForWord(trail.words[0]) span = arena.GetLineSpan(span_id) self.comp_ui_state.display_pos = span.col self.debug_f.log('** DISPLAY_POS = %d', self.comp_ui_state.display_pos) else: base_opts, user_spec = self.comp_lookup.GetSpecForName( first) if not user_spec and alias_first: base_opts, user_spec = self.comp_lookup.GetSpecForName( alias_first) if user_spec: # Pass the aliased command to the user-defined function, and use # it for retries. first = alias_first if not user_spec: base_opts, user_spec = self.comp_lookup.GetFallback() # Display since the beginning span_id = word_.LeftMostSpanForWord(trail.words[-1]) span = arena.GetLineSpan(span_id) self.comp_ui_state.display_pos = span.col self.debug_f.log('words[-1]: %r', trail.words[-1]) self.debug_f.log('display_pos %d', self.comp_ui_state.display_pos) # Update the API for user-defined functions. index = len( partial_argv) - 1 # COMP_CWORD is -1 when it's empty prev = '' if index == 0 else partial_argv[index - 1] comp.Update(first=first, to_complete=partial_argv[-1], prev=prev, index=index, partial_argv=partial_argv) # This happens in the case of [[ and ((, or a syntax error like 'echo < >'. if not user_spec: debug_f.log("Didn't find anything to complete") return # Reset it back to what was registered. User-defined functions can mutate # it. dynamic_opts = {} self.compopt_state.dynamic_opts = dynamic_opts self.compopt_state.currently_completing = True try: done = False while not done: try: for candidate in self._PostProcess(base_opts, dynamic_opts, user_spec, comp): yield candidate except _RetryCompletion as e: debug_f.log('Got 124, trying again ...') # Get another user_spec. The ShellFuncAction may have 'sourced' code # and run 'complete' to mutate comp_lookup, and we want to get that # new entry. if num_partial == 0: raise AssertionError() elif num_partial == 1: base_opts, user_spec = self.comp_lookup.GetFirstSpec() else: # (already processed alias_first) base_opts, user_spec = self.comp_lookup.GetSpecForName( first) if not user_spec: base_opts, user_spec = self.comp_lookup.GetFallback( ) else: done = True # exhausted candidates without getting a retry finally: self.compopt_state.currently_completing = False
def Eval(self, line): """Returns an expanded line.""" if not self.readline_mod: return line tokens = list(HISTORY_LEXER.Tokens(line)) # Common case: no history expansion. if all(id_ == Id.History_Other for (id_, _) in tokens): return line history_len = self.readline_mod.get_current_history_length() if history_len <= 0: # no commands to expand return line self.debug_f.log('history length = %d', history_len) parts = [] for id_, val in tokens: if id_ == Id.History_Other: out = val elif id_ == Id.History_Op: prev = self.readline_mod.get_history_item(history_len) ch = val[1] if ch == '!': out = prev else: self.parse_ctx.trail.Clear() # not strictly necessary? line_reader = reader.StringLineReader( prev, self.parse_ctx.arena) c_parser = self.parse_ctx.MakeOshParser(line_reader) try: c_parser.ParseLogicalLine() except util.ParseError as e: #from core import ui #ui.PrettyPrintError(e, self.parse_ctx.arena) # Invalid command in history. TODO: We should never enter these. self.debug_f.log( "Couldn't parse historical command %r: %s", prev, e) # NOTE: We're using the trail rather than the return value of # ParseLogicalLine because it handles cases like # $ for i in 1 2 3; do sleep ${i}; done # $ echo !$ # which should expand to 'echo ${i}' words = self.parse_ctx.trail.words #self.debug_f.log('TRAIL WORDS: %s', words) if ch == '^': try: w = words[1] except IndexError: raise util.HistoryError("No first word in %r", prev) spid1 = word_.LeftMostSpanForWord(w) spid2 = word_.RightMostSpanForWord(w) elif ch == '$': try: w = words[-1] except IndexError: raise util.HistoryError("No last word in %r", prev) spid1 = word_.LeftMostSpanForWord(w) spid2 = word_.RightMostSpanForWord(w) elif ch == '*': try: w1 = words[1] w2 = words[-1] except IndexError: raise util.HistoryError( "Couldn't find words in %r", prev) spid1 = word_.LeftMostSpanForWord(w1) spid2 = word_.RightMostSpanForWord(w2) else: raise AssertionError(ch) arena = self.parse_ctx.arena span1 = arena.GetLineSpan(spid1) span2 = arena.GetLineSpan(spid2) begin = span1.col end = span2.col + span2.length out = prev[begin:end] elif id_ == Id.History_Num: index = int( val[1:]) # regex ensures this. Maybe have - on the front. if index < 0: num = history_len + 1 + index else: num = index out = self.readline_mod.get_history_item(num) if out is None: # out of range raise util.HistoryError('%s: not found', val) elif id_ == Id.History_Search: # Remove the required space at the end and save it. A simple hack than # the one bash has. last_char = val[-1] val = val[:-1] # Search backward prefix = None substring = None if val[1] == '?': substring = val[2:] else: prefix = val[1:] out = None for i in xrange(history_len, 1, -1): cmd = self.readline_mod.get_history_item(i) if prefix and cmd.startswith(prefix): out = cmd if substring and substring in cmd: out = cmd if out is not None: out += last_char # restore required space break if out is None: raise util.HistoryError('%r found no results', val) else: raise AssertionError(id_) parts.append(out) line = ''.join(parts) # show what we expanded to sys.stdout.write('! %s' % line) return line