def record_source(self, word_ob, text, target):
pos = word_.LeftMostSpanForWord(word_ob)
logger.info("Recording source: %r -> %r", text, target)
logger.debug(" position: %d, word object: %r", pos, word_ob)
self.word_obs[text] = word_ob
self.sources[text].append(pos)
self.resolved_source[text] = target
def _ValToIntOrError(self, val, blame_word=None, span_id=runtime.NO_SPID):
# type: (value_t, word_t, int) -> int
if span_id == runtime.NO_SPID and blame_word:
span_id = word_.LeftMostSpanForWord(blame_word)
#log('_ValToIntOrError span=%s blame=%s', span_id, blame_word)
try:
if val.tag == value_e.Undef: # 'nounset' already handled before got here
# Happens upon a[undefined]=42, which unfortunately turns into a[0]=42.
#log('blame_word %s arena %s', blame_word, self.arena)
e_die('Undefined value in arithmetic context', span_id=span_id)
if val.tag == value_e.Int:
return val.i
if val.tag == value_e.Str:
return _StringToInteger(val.s, span_id=span_id) # calls e_die
except error.FatalRuntime as e:
if self.exec_opts.strict_arith:
raise
else:
span_id = word_.SpanIdFromError(e)
self.errfmt.PrettyPrintError(e, prefix='warning: ')
return 0
# Arrays and associative arrays always fail -- not controlled by strict_arith.
# In bash, (( a )) is like (( a[0] )), but I don't want that.
# And returning '0' gives different results.
e_die("Expected a value convertible to integer, got %s",
val,
span_id=span_id)
def _assertSpanForWord(test, word_str):
arena = test_lib.MakeArena('word_parse_test.py')
w_parser = test_lib.InitWordParser(word_str, arena=arena)
w = _assertReadWordWithArena(test, w_parser)
span_id = word_.LeftMostSpanForWord(w)
print(word_str)
print(span_id)
if span_id != runtime.NO_SPID:
span = arena.GetLineSpan(span_id)
print(span)
def SpanForArithExpr(node):
# type: (arith_expr_t) -> int
UP_node = node
with tagswitch(node) as case:
if case(arith_expr_e.VarRef):
token = cast(Token, UP_node)
return token.span_id
elif case(arith_expr_e.Word):
w = cast(compound_word, UP_node)
return word_.LeftMostSpanForWord(w)
return runtime.NO_SPID
def SpanForArithExpr(node):
# type: (arith_expr_t) -> int
UP_node = node
with tagswitch(node) as case:
if case(arith_expr_e.VarRef):
# TODO: VarRef should store a token instead of a string!
return runtime.NO_SPID
elif case(arith_expr_e.ArithWord):
node = cast(arith_expr__ArithWord, UP_node)
return word_.LeftMostSpanForWord(node.w)
return runtime.NO_SPID
def _ValToArithOrError(self, val, blame_word=None, span_id=runtime.NO_SPID):
if span_id == runtime.NO_SPID and blame_word:
span_id = word_.LeftMostSpanForWord(blame_word)
#log('_ValToArithOrError span=%s blame=%s', span_id, blame_word)
try:
i = self._ValToArith(val, span_id)
except util.FatalRuntimeError as e:
if self.exec_opts.strict_arith:
raise
else:
i = 0
span_id = word_.SpanIdFromError(e)
self.errfmt.PrettyPrintError(e, prefix='warning: ')
return i
def _StringToIntegerOrError(self, s, blame_word=None,
span_id=runtime.NO_SPID):
"""Used by both [[ $x -gt 3 ]] and (( $x ))."""
if span_id == runtime.NO_SPID and blame_word:
span_id = word_.LeftMostSpanForWord(blame_word)
try:
i = _StringToInteger(s, span_id=span_id)
except util.FatalRuntimeError as e:
if self.exec_opts.strict_arith:
raise
else:
self.errfmt.PrettyPrintError(e, prefix='warning: ')
i = 0
return i
def record_word(self, word_ob, text):
pos = word_.LeftMostSpanForWord(word_ob)
self.word_obs[text] = word_ob
if (consts.LookupSpecialBuiltin(text) == consts.NO_INDEX
and consts.LookupAssignBuiltin(text) == consts.NO_INDEX
and consts.LookupNormalBuiltin(text) == consts.NO_INDEX):
logger.info("Recording command: %r", text)
logger.debug(" position: %d, word object: %r", pos, word_ob)
self.commands[text].append(pos)
else:
# TODO: no immediate use since I'm no longer patching builtins
# but there may still be utility in recording builtins a script
# depends on. This would support spotting function/alias
# clashes and such.
self.builtins[text].append(pos)
def _StringToIntegerOrError(self, s, blame_word=None):
# type: (str, Optional[word_t]) -> int
"""Used by both [[ $x -gt 3 ]] and (( $x ))."""
if blame_word:
span_id = word_.LeftMostSpanForWord(blame_word)
else:
span_id = runtime.NO_SPID
try:
i = self._StringToInteger(s, span_id=span_id)
except error.Strict as e:
if self.always_strict or self.exec_opts.strict_arith():
raise
else:
i = 0
return i
def _VarRefOrWord(self, anode):
# type: (arith_expr_t) -> Tuple[Optional[str], int]
"""
Returns (var_name, span_id) if the arith node can be interpreted that way
"""
UP_anode = anode
with tagswitch(anode) as case:
if case(arith_expr_e.VarRef):
tok = cast(Token, UP_anode)
return (tok.val, tok.span_id)
elif case(arith_expr_e.Word):
w = cast(compound_word, UP_anode)
var_name = self.EvalWordToString(w)
span_id = word_.LeftMostSpanForWord(w)
return (var_name, span_id)
no_str = None # type: str
return (no_str, runtime.NO_SPID)
def DoCommand(self, node, local_symbols, at_top_level=False):
if node.tag == command_e.CommandList:
# TODO: How to distinguish between echo hi; echo bye; and on separate
# lines
for child in node.children:
self.DoCommand(child, local_symbols, at_top_level=at_top_level)
elif node.tag == command_e.Simple:
# How to preserve spaces between words? Do you want to do it?
# Well you need to test this:
#
# echo foo \
# bar
# TODO: Need to print until the left most part of the phrase? the phrase
# is a word, binding, redirect.
#self.cursor.PrintUntil()
if node.more_env:
(left_spid, ) = node.more_env[0].spids
self.cursor.PrintUntil(left_spid)
self.f.write('env ')
# We only need to transform the right side, not left side.
for pair in node.more_env:
self.DoWordInCommand(pair.val, local_symbols)
# More translations:
# - . to source
# - eval to sh-eval
if node.words:
first_word = node.words[0]
ok, val, quoted = word_.StaticEval(first_word)
word0_spid = word_.LeftMostSpanForWord(first_word)
if ok and not quoted:
if val == '[':
last_word = node.words[-1]
# Check if last word is ]
ok, val, quoted = word_.StaticEval(last_word)
if ok and not quoted and val == ']':
# Replace [ with 'test'
self.cursor.PrintUntil(word0_spid)
self.cursor.SkipUntil(word0_spid + 1)
self.f.write('test')
for w in node.words[1:-1]:
self.DoWordInCommand(w, local_symbols)
# Now omit ]
last_spid = word_.LeftMostSpanForWord(last_word)
self.cursor.PrintUntil(last_spid -
1) # Get the space before
self.cursor.SkipUntil(last_spid +
1) # ] takes one spid
return
else:
raise RuntimeError('Got [ without ]')
elif val == '.':
self.cursor.PrintUntil(word0_spid)
self.cursor.SkipUntil(word0_spid + 1)
self.f.write('source')
return
for w in node.words:
self.DoWordInCommand(w, local_symbols)
# NOTE: This will change to "phrase"? Word or redirect.
for r in node.redirects:
self.DoRedirect(r, local_symbols)
# TODO: Print the terminator. Could be \n or ;
# Need to print env like PYTHONPATH = 'foo' && ls
# Need to print redirects:
# < > are the same. << is here string, and >> is assignment.
# append is >+
# TODO: static_eval of simple command
# - [ -> "test". Eliminate trailing ].
# - . -> source, etc.
elif node.tag == command_e.ShAssignment:
self.DoShAssignment(node, at_top_level, local_symbols)
elif node.tag == command_e.Pipeline:
# Obscure: |& turns into |- or |+ for stderr.
# TODO:
# if ! true; then -> if not true {
# if ! echo | grep; then -> if not { echo | grep } {
# }
# not is like do {}, but it negates the return value I guess.
for child in node.children:
self.DoCommand(child, local_symbols)
elif node.tag == command_e.AndOr:
for child in node.children:
self.DoCommand(child, local_symbols)
elif node.tag == command_e.Sentence:
# 'ls &' to 'fork ls'
# Keep ; the same.
self.DoCommand(node.child, local_symbols)
# This has to be different in the function case.
elif node.tag == command_e.BraceGroup:
# { echo hi; } -> do { echo hi }
# For now it might be OK to keep 'do { echo hi; }
#left_spid, right_spid = node.spids
(left_spid, ) = node.spids
self.cursor.PrintUntil(left_spid)
self.cursor.SkipUntil(left_spid + 1)
self.f.write('do {')
for child in node.children:
self.DoCommand(child, local_symbols)
elif node.tag == command_e.Subshell:
# (echo hi) -> shell echo hi
# (echo hi; echo bye) -> shell {echo hi; echo bye}
(left_spid, right_spid) = node.spids
self.cursor.PrintUntil(left_spid)
self.cursor.SkipUntil(left_spid + 1)
self.f.write('shell {')
self.DoCommand(node.child, local_symbols)
#self._DebugSpid(right_spid)
#self._DebugSpid(right_spid + 1)
#print('RIGHT SPID', right_spid)
self.cursor.PrintUntil(right_spid)
self.cursor.SkipUntil(right_spid + 1)
self.f.write('}')
elif node.tag == command_e.DParen:
# (( a == 0 )) is sh-expr ' a == 0 '
#
# NOTE: (( n++ )) is auto-translated to sh-expr 'n++', but could be set
# n++.
left_spid, right_spid = node.spids
self.cursor.PrintUntil(left_spid)
self.cursor.SkipUntil(left_spid + 1)
self.f.write("sh-expr '")
self.cursor.PrintUntil(right_spid - 1) # before ))
self.cursor.SkipUntil(right_spid +
1) # after )) -- each one is a token
self.f.write("'")
elif node.tag == command_e.DBracket:
# [[ 1 -eq 2 ]] to (1 == 2)
self.DoBoolExpr(node.expr)
elif node.tag == command_e.ShFunction:
# TODO: skip name
#self.f.write('proc %s' % node.name)
# New symbol table for every function.
new_local_symbols = {}
# Should be the left most span, including 'function'
self.cursor.PrintUntil(node.spids[0])
self.f.write('proc ')
self.f.write(node.name)
self.cursor.SkipUntil(node.spids[2])
if node.body.tag == command_e.BraceGroup:
# Don't add "do" like a standalone brace group. Just use {}.
for child in node.body.children:
self.DoCommand(child, new_local_symbols)
else:
pass
# Add {}.
# proc foo {
# shell {echo hi; echo bye}
# }
#self.DoCommand(node.body)
elif node.tag == command_e.BraceGroup:
for child in node.children:
self.DoCommand(child, local_symbols)
elif node.tag == command_e.DoGroup:
do_spid, done_spid = node.spids
self.cursor.PrintUntil(do_spid)
self.cursor.SkipUntil(do_spid + 1)
self.f.write('{')
for child in node.children:
self.DoCommand(child, local_symbols)
self.cursor.PrintUntil(done_spid)
self.cursor.SkipUntil(done_spid + 1)
self.f.write('}')
elif node.tag == command_e.ForEach:
# Need to preserve spaces between words, because there can be line
# wrapping.
# for x in a b c \
# d e f; do
_, in_spid, semi_spid = node.spids
if in_spid == runtime.NO_SPID:
#self.cursor.PrintUntil() # 'for x' and then space
self.f.write('for %s in @Argv ' % node.iter_name)
self.cursor.SkipUntil(node.body.spids[0])
else:
self.cursor.PrintUntil(in_spid +
1) # 'for x in' and then space
self.f.write('[')
for w in node.iter_words:
self.DoWordInCommand(w, local_symbols)
self.f.write(']')
#print("SKIPPING SEMI %d" % semi_spid, file=sys.stderr)
if semi_spid != runtime.NO_SPID:
self.cursor.PrintUntil(semi_spid)
self.cursor.SkipUntil(semi_spid + 1)
self.DoCommand(node.body, local_symbols)
elif node.tag == command_e.ForExpr:
# Change (( )) to ( ), and then _FixDoGroup
pass
elif node.tag == command_e.WhileUntil:
# Skip 'until', and replace it with 'while not'
if node.keyword.id == Id.KW_Until:
kw_spid = node.keyword.span_id
self.cursor.PrintUntil(kw_spid)
self.f.write('while not')
self.cursor.SkipUntil(kw_spid + 1)
if node.cond.tag_() == condition_e.Shell:
commands = node.cond.commands
# Skip the semi-colon in the condition, which is ususally a Sentence
if len(commands) == 1 and commands[0].tag_(
) == command_e.Sentence:
self.DoCommand(commands[0].child, local_symbols)
semi_spid = commands[0].terminator.span_id
self.cursor.SkipUntil(semi_spid + 1)
self.DoCommand(node.body, local_symbols)
elif node.tag == command_e.If:
else_spid, fi_spid = node.spids
# if foo; then -> if foo {
# elif foo; then -> } elif foo {
for i, arm in enumerate(node.arms):
elif_spid, then_spid = arm.spids
if i != 0: # 'if' not 'elif' on the first arm
self.cursor.PrintUntil(elif_spid)
self.f.write('} ')
cond = arm.cond
if cond.tag_() == condition_e.Shell:
if len(
cond.commands
) == 1 and cond.commands[0].tag == command_e.Sentence:
sentence = cond.commands[0]
self.DoCommand(sentence, local_symbols)
# Remove semi-colon
semi_spid = sentence.terminator.span_id
self.cursor.PrintUntil(semi_spid)
self.cursor.SkipUntil(semi_spid + 1)
else:
for child in cond.commands:
self.DoCommand(child, local_symbols)
self.cursor.PrintUntil(then_spid)
self.cursor.SkipUntil(then_spid + 1)
self.f.write('{')
for child in arm.action:
self.DoCommand(child, local_symbols)
# else -> } else {
if node.else_action:
self.cursor.PrintUntil(else_spid)
self.f.write('} ')
self.cursor.PrintUntil(else_spid + 1)
self.f.write(' {')
for child in node.else_action:
self.DoCommand(child, local_symbols)
# fi -> }
self.cursor.PrintUntil(fi_spid)
self.cursor.SkipUntil(fi_spid + 1)
self.f.write('}')
elif node.tag == command_e.Case:
case_spid, in_spid, esac_spid = node.spids
self.cursor.PrintUntil(case_spid)
self.cursor.SkipUntil(case_spid + 1)
self.f.write('match')
# Reformat "$1" to $1
self.DoWordInCommand(node.to_match, local_symbols)
self.cursor.PrintUntil(in_spid)
self.cursor.SkipUntil(in_spid + 1)
self.f.write('{') # matchstr $var {
# each arm needs the ) and the ;; node to skip over?
for arm in node.arms:
left_spid, rparen_spid, dsemi_spid, last_spid = arm.spids
#print(left_spid, rparen_spid, dsemi_spid)
self.cursor.PrintUntil(left_spid)
# Hm maybe keep | because it's semi-deprecated? You acn use
# reload|force-relaod {
# }
# e/reload|force-reload/ {
# }
# / 'reload' or 'force-reload' / {
# }
#
# Yeah it's the more abbreviated syntax.
# change | to 'or'
for pat in arm.pat_list:
pass
self.f.write('with ')
# Remove the )
self.cursor.PrintUntil(rparen_spid)
self.cursor.SkipUntil(rparen_spid + 1)
for child in arm.action:
self.DoCommand(child, local_symbols)
if dsemi_spid != runtime.NO_SPID:
# Remove ;;
self.cursor.PrintUntil(dsemi_spid)
self.cursor.SkipUntil(dsemi_spid + 1)
elif last_spid != runtime.NO_SPID:
self.cursor.PrintUntil(last_spid)
else:
raise AssertionError(
"Expected with dsemi_spid or last_spid in case arm")
self.cursor.PrintUntil(esac_spid)
self.cursor.SkipUntil(esac_spid + 1)
self.f.write('}') # strmatch $var {
elif node.tag == command_e.NoOp:
pass
elif node.tag == command_e.ControlFlow:
# No change for break / return / continue
pass
elif node.tag == command_e.TimeBlock:
self.DoCommand(node.pipeline, local_symbols)
else:
#log('Command not handled: %s', node)
raise AssertionError(node.__class__.__name__)
def DoShAssignment(self, node, at_top_level, local_symbols):
"""
local_symbols:
- Add every 'local' declaration to it
- problem: what if you have local in an "if" ?
- we could treat it like nested scope and see what happens? Do any
programs have a problem with it?
case/if/for/while/BraceGroup all define scopes or what?
You don't want inconsistency of variables that could be defined at
any point.
- or maybe you only need it within "if / case" ? Well I guess
for/while can break out of the loop and cause problems. A break is
an "if".
- for subsequent
"""
# Change RHS to expression language. Bare words not allowed. foo -> 'foo'
has_rhs = False # TODO: This is on a per-variable basis.
# local foo -> var foo = ''
# readonly foo -> setconst foo
# export foo -> export foo
# TODO:
# - This depends on self.mode.
# - And we also need the enclosing ShFunction node to analyze.
# - or we need a symbol table for the current function. Forget about
#
# Oil keywords:
# - global : scope qualifier
# - var, const : mutability
# - export : state mutation
# - setconst -- make a variable mutable. or maybe freeze var?
#
# NOTE: Bash also has "unset". Does anyone use it?
# You can use "delete" like Python I guess. It's not the opposite of
# set.
# NOTE:
# - We CAN tell if a variable has been defined locally.
# - We CANNOT tell if it's been defined globally, because different files
# share the same global namespace, and we can't statically figure out what
# files are in the program.
defined_locally = False # is it a local variable in this function?
# can't tell if global
# DISABLED after doing dynamic assignments. We could reconstruct these
# from SimpleCommand? Look at argv[0] and then do static parsing to
# assign_pair?
if 0:
# Assume that 'local' it's a declaration. In osh, it's an error if
# locals are redefined. In bash, it's OK to do 'local f=1; local f=2'.
# Could have a flag if enough people do this.
if at_top_level:
raise RuntimeError('local at top level is invalid')
if defined_locally:
raise RuntimeError("Can't redefine local")
keyword_spid = node.spids[0]
self.cursor.PrintUntil(keyword_spid)
self.cursor.SkipUntil(keyword_spid + 1)
self.f.write('var')
if local_symbols is not None:
for pair in node.pairs:
# NOTE: Not handling local a[b]=c
if pair.lhs.tag == sh_lhs_expr_e.Name:
#print("REGISTERED %s" % pair.lhs.name)
local_symbols[pair.lhs.name] = True
elif 1:
self.cursor.PrintUntil(node.spids[0])
# For now, just detect whether the FIRST assignment on the line has been
# declared locally. We might want to split every line into separate
# statements.
if local_symbols is not None:
lhs0 = node.pairs[0].lhs
if lhs0.tag == sh_lhs_expr_e.Name and lhs0.name in local_symbols:
defined_locally = True
#print("CHECKING NAME", lhs0.name, defined_locally, local_symbols)
has_array = any(pair.lhs.tag == sh_lhs_expr_e.UnparsedIndex
for pair in node.pairs)
# need semantic analysis.
# Would be nice to assume that it's a local though.
if has_array:
self.f.write('compat ') # 'compat array-assign' syntax
elif at_top_level:
self.f.write('setglobal ')
elif defined_locally:
self.f.write('set ')
#self.f.write('[local mutated]')
else:
# We're in a function, but it's not defined locally, so we must be
# mutatting a global.
self.f.write('setglobal ')
elif 0:
# Explicit const. Assume it can't be redefined.
# Verb.
#
# Top level;
# readonly FOO=bar -> const FOO = 'bar'
# readonly FOO -> freeze FOO
# function level:
# readonly FOO=bar -> const global FOO ::= 'bar'
# readonly FOO -> freeze FOO
keyword_spid = node.spids[0]
if at_top_level:
self.cursor.PrintUntil(keyword_spid)
self.cursor.SkipUntil(keyword_spid + 1)
self.f.write('const')
elif defined_locally:
# TODO: Actually we might want 'freeze here. In bash, you can make a
# variable readonly after its defined.
raise RuntimeError("Constant redefined locally")
else:
# Same as global level
self.cursor.PrintUntil(keyword_spid)
self.cursor.SkipUntil(keyword_spid + 1)
self.f.write('const')
elif 0:
# declare -rx foo spam=eggs
# export foo
# setconst foo
#
# spam = eggs
# export spam
# Have to parse the flags
self.f.write('TODO ')
# foo=bar spam=eggs -> foo = 'bar', spam = 'eggs'
n = len(node.pairs)
for i, pair in enumerate(node.pairs):
if pair.lhs.tag == sh_lhs_expr_e.Name:
left_spid = pair.spids[0]
self.cursor.PrintUntil(left_spid)
# Assume skipping over one Lit_VarLike token
self.cursor.SkipUntil(left_spid + 1)
# Replace name. I guess it's Lit_Chars.
self.f.write(pair.lhs.name)
self.f.write(' = ')
# TODO: This should be translated from Empty.
if pair.rhs.tag == word_e.Empty:
self.f.write("''") # local i -> var i = ''
else:
self.DoWordAsExpr(pair.rhs, local_symbols)
elif pair.lhs.tag == sh_lhs_expr_e.UnparsedIndex:
# NOTES:
# - parse_ctx.one_pass_parse should be on, so the span invariant
# is accurate
# - Then do the following translation:
# a[x+1]="foo $bar" ->
# compat array-assign a 'x+1' "$foo $bar"
# This avoids dealing with nested arenas.
#
# TODO: This isn't great when there are multiple assignments.
# a[x++]=1 b[y++]=2
#
# 'compat' could apply to the WHOLE statement, with multiple
# assignments.
self.f.write("array-assign %s '%s' " %
(pair.lhs.name, pair.lhs.index))
if pair.rhs.tag == word_e.Empty:
self.f.write("''") # local i -> var i = ''
else:
rhs_spid = word_.LeftMostSpanForWord(pair.rhs)
self.cursor.SkipUntil(rhs_spid)
self.DoWordAsExpr(pair.rhs, local_symbols)
else:
raise AssertionError(pair.lhs.__class__.__name__)
if i != n - 1:
self.f.write(',')
def DoRedirect(self, node, local_symbols):
#print(node, file=sys.stderr)
op_spid = node.op.span_id
op_id = node.op.id
self.cursor.PrintUntil(op_spid)
# TODO:
# - Do < and <& the same way.
# - How to handle here docs and here docs?
# - >> becomes >+ or >-, or maybe >>>
#if node.tag == redir_e.Redir:
if False:
if node.fd == runtime.NO_SPID:
if op_id == Id.Redir_Great:
self.f.write('>') # Allow us to replace the operator
self.cursor.SkipUntil(op_spid + 1)
elif op_id == Id.Redir_GreatAnd:
self.f.write('> !') # Replace >& 2 with > !2
spid = word_.LeftMostSpanForWord(node.arg_word)
self.cursor.SkipUntil(spid)
#self.DoWordInCommand(node.arg_word)
else:
# NOTE: Spacing like !2>err.txt vs !2 > err.txt can be done in the
# formatter.
self.f.write('!%d ' % node.fd)
if op_id == Id.Redir_Great:
self.f.write('>')
self.cursor.SkipUntil(op_spid + 1)
elif op_id == Id.Redir_GreatAnd:
self.f.write('> !') # Replace 1>& 2 with !1 > !2
spid = word_.LeftMostSpanForWord(node.arg_word)
self.cursor.SkipUntil(spid)
self.DoWordInCommand(node.arg_word, local_symbols)
#elif node.tag == redir_e.HereDoc:
elif False:
ok, delimiter, delim_quoted = word_.StaticEval(node.here_begin)
if not ok:
p_die('Invalid here doc delimiter', word=node.here_begin)
# Turn everything into <<. We just change the quotes
self.f.write('<<')
#here_begin_spid2 = word_.RightMostSpanForWord(node.here_begin)
if delim_quoted:
self.f.write(" '''")
else:
self.f.write(' """')
delim_end_spid = word_.RightMostSpanForWord(node.here_begin)
self.cursor.SkipUntil(delim_end_spid + 1)
#self.cursor.SkipUntil(here_begin_spid + 1)
# Now print the lines. TODO: Have a flag to indent these to the level of
# the owning command, e.g.
# cat <<EOF
# EOF
# Or since most here docs are the top level, you could just have a hack
# for a fixed indent? TODO: Look at real use cases.
for part in node.stdin_parts:
self.DoWordPart(part, local_symbols)
self.cursor.SkipUntil(node.here_end_span_id + 1)
if delim_quoted:
self.f.write("'''\n")
else:
self.f.write('"""\n')
# Need
#self.cursor.SkipUntil(here_end_spid2)
else:
raise AssertionError(node.__class__.__name__)
# <<< 'here word'
# << 'here word'
#
# 2> out.txt
# !2 > out.txt
# cat 1<< EOF
# hello $name
# EOF
# cat !1 << """
# hello $name
# """
#
# cat << 'EOF'
# no expansion
# EOF
# cat <<- 'EOF'
# no expansion and indented
#
# cat << '''
# no expansion
# '''
# cat << '''
# no expansion and indented
# '''
# Warn about multiple here docs on a line.
# As an obscure feature, allow
# cat << \'ONE' << \"TWO"
# 123
# ONE
# 234
# TWO
# The _ is an indicator that it's not a string to be piped in.
pass
def Matches(self, comp):
# type: (Api) -> Iterator[Union[Iterator, Iterator[str]]]
"""
Args:
comp: Callback args from readline. Readline uses set_completer_delims to
tokenize the string.
Returns a list of matches relative to readline's completion_delims.
We have to post-process the output of various completers.
"""
arena = self.parse_ctx.arena # Used by inner functions
# Pass the original line "out of band" to the completion callback.
line_until_tab = comp.line[:comp.end]
self.comp_ui_state.line_until_tab = line_until_tab
self.parse_ctx.trail.Clear()
line_reader = reader.StringLineReader(line_until_tab,
self.parse_ctx.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader,
emit_comp_dummy=True)
# We want the output from parse_ctx, so we don't use the return value.
try:
c_parser.ParseLogicalLine()
except error.Parse as e:
# e.g. 'ls | ' will not parse. Now inspect the parser state!
pass
debug_f = self.debug_f
trail = self.parse_ctx.trail
if 1:
trail.PrintDebugString(debug_f)
#
# First try completing the shell language itself.
#
# NOTE: We get Eof_Real in the command state, but not in the middle of a
# BracedVarSub. This is due to the difference between the CommandParser
# and WordParser.
tokens = trail.tokens
last = -1
if tokens[-1].id == Id.Eof_Real:
last -= 1 # ignore it
try:
t1 = tokens[last]
except IndexError:
t1 = None
try:
t2 = tokens[last - 1]
except IndexError:
t2 = None
debug_f.log('line: %r', comp.line)
debug_f.log('rl_slice from byte %d to %d: %r', comp.begin, comp.end,
comp.line[comp.begin:comp.end])
debug_f.log('t1 %s', t1)
debug_f.log('t2 %s', t2)
# Each of the 'yield' statements below returns a fully-completed line, to
# appease the readline library. The root cause of this dance: If there's
# one candidate, readline is responsible for redrawing the input line. OSH
# only displays candidates and never redraws the input line.
def _TokenStart(tok):
# type: (Token) -> int
span = arena.GetLineSpan(tok.span_id)
return span.col
if t2: # We always have t1?
# echo $
if IsDollar(t2) and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $
for name in self.mem.VarNames():
yield line_until_tab + name # no need to quote var names
return
# echo ${
if t2.id == Id.Left_DollarBrace and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(
t2) + 2 # 2 for ${
for name in self.mem.VarNames():
yield line_until_tab + name # no need to quote var names
return
# echo $P
if t2.id == Id.VSub_DollarName and IsDummy(t1):
# Example: ${undef:-$P
# readline splits at ':' so we have to prepend '-$' to every completed
# variable name.
self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $
to_complete = t2.val[1:]
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[
n:] # no need to quote var names
return
# echo ${P
if t2.id == Id.VSub_Name and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) # no offset
to_complete = t2.val
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[
n:] # no need to quote var names
return
# echo $(( VAR
if t2.id == Id.Lit_ArithVarLike and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) # no offset
to_complete = t2.val
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[
n:] # no need to quote var names
return
if trail.words:
# echo ~<TAB>
# echo ~a<TAB> $(home dirs)
# This must be done at a word level, and TildeDetectAll() does NOT help
# here, because they don't have trailing slashes yet! We can't do it on
# tokens, because otherwise f~a will complete. Looking at word_part is
# EXACTLY what we want.
parts = trail.words[-1].parts
if (len(parts) == 2 and parts[0].tag_() == word_part_e.Literal
and parts[1].tag_() == word_part_e.Literal
and parts[0].id == Id.Lit_TildeLike
and parts[1].id == Id.Lit_CompDummy):
t2 = parts[0]
# +1 for ~
self.comp_ui_state.display_pos = _TokenStart(parts[0]) + 1
to_complete = t2.val[1:]
n = len(to_complete)
for u in pwd.getpwall(): # catch errors?
name = u.pw_name
if name.startswith(to_complete):
yield line_until_tab + ShellQuoteB(name[n:]) + '/'
return
# echo hi > f<TAB> (complete redirect arg)
if trail.redirects:
r = trail.redirects[-1]
# Only complete 'echo >', but not 'echo >&' or 'cat <<'
# TODO: Don't complete <<< 'h'
if (r.arg.tag_() == redir_param_e.Word
and consts.RedirArgType(r.op.id) == redir_arg_type_e.Path):
arg_word = r.arg
if WordEndsWithCompDummy(arg_word):
debug_f.log('Completing redirect arg')
try:
val = self.word_ev.EvalWordToString(r.arg)
except error.FatalRuntime as e:
debug_f.log('Error evaluating redirect word: %s', e)
return
if val.tag_() != value_e.Str:
debug_f.log("Didn't get a string from redir arg")
return
span_id = word_.LeftMostSpanForWord(arg_word)
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
comp.Update(
to_complete=val.s) # FileSystemAction uses only this
n = len(val.s)
action = FileSystemAction(add_slash=True)
for name in action.Matches(comp):
yield line_until_tab + ShellQuoteB(name[n:])
return
#
# We're not completing the shell language. Delegate to user-defined
# completion for external tools.
#
# Set below, and set on retries.
base_opts = None
user_spec = None
# Used on retries.
partial_argv = []
num_partial = -1
first = None
if trail.words:
# Now check if we're completing a word!
if WordEndsWithCompDummy(trail.words[-1]):
debug_f.log('Completing words')
#
# It didn't look like we need to complete var names, tilde, redirects,
# etc. Now try partial_argv, which may involve invoking PLUGINS.
# needed to complete paths with ~
words2 = word_.TildeDetectAll(trail.words)
if 0:
debug_f.log('After tilde detection')
for w in words2:
print(w, file=debug_f)
if 0:
debug_f.log('words2:')
for w2 in words2:
debug_f.log(' %s', w2)
for w in words2:
try:
# TODO:
# - Should we call EvalWordSequence? But turn globbing off? It
# can do splitting and such.
# - We could have a variant to eval TildeSub to ~ ?
val = self.word_ev.EvalWordToString(w)
except error.FatalRuntime:
# Why would it fail?
continue
if val.tag_() == value_e.Str:
partial_argv.append(val.s)
else:
pass
debug_f.log('partial_argv: %s', partial_argv)
num_partial = len(partial_argv)
first = partial_argv[0]
alias_first = None
debug_f.log('alias_words: %s', trail.alias_words)
if trail.alias_words:
w = trail.alias_words[0]
try:
val = self.word_ev.EvalWordToString(w)
except error.FatalRuntime:
pass
alias_first = val.s
debug_f.log('alias_first: %s', alias_first)
if num_partial == 0: # should never happen because of Lit_CompDummy
raise AssertionError()
elif num_partial == 1:
base_opts, user_spec = self.comp_lookup.GetFirstSpec()
# Display/replace since the beginning of the first word. Note: this
# is non-zero in the case of
# echo $(gr and
# echo `gr
span_id = word_.LeftMostSpanForWord(trail.words[0])
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
self.debug_f.log('** DISPLAY_POS = %d',
self.comp_ui_state.display_pos)
else:
base_opts, user_spec = self.comp_lookup.GetSpecForName(
first)
if not user_spec and alias_first:
base_opts, user_spec = self.comp_lookup.GetSpecForName(
alias_first)
if user_spec:
# Pass the aliased command to the user-defined function, and use
# it for retries.
first = alias_first
if not user_spec:
base_opts, user_spec = self.comp_lookup.GetFallback()
# Display since the beginning
span_id = word_.LeftMostSpanForWord(trail.words[-1])
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
self.debug_f.log('words[-1]: %r', trail.words[-1])
self.debug_f.log('display_pos %d',
self.comp_ui_state.display_pos)
# Update the API for user-defined functions.
index = len(
partial_argv) - 1 # COMP_CWORD is -1 when it's empty
prev = '' if index == 0 else partial_argv[index - 1]
comp.Update(first=first,
to_complete=partial_argv[-1],
prev=prev,
index=index,
partial_argv=partial_argv)
# This happens in the case of [[ and ((, or a syntax error like 'echo < >'.
if not user_spec:
debug_f.log("Didn't find anything to complete")
return
# Reset it back to what was registered. User-defined functions can mutate
# it.
dynamic_opts = {}
self.compopt_state.dynamic_opts = dynamic_opts
self.compopt_state.currently_completing = True
try:
done = False
while not done:
try:
for candidate in self._PostProcess(base_opts, dynamic_opts,
user_spec, comp):
yield candidate
except _RetryCompletion as e:
debug_f.log('Got 124, trying again ...')
# Get another user_spec. The ShellFuncAction may have 'sourced' code
# and run 'complete' to mutate comp_lookup, and we want to get that
# new entry.
if num_partial == 0:
raise AssertionError()
elif num_partial == 1:
base_opts, user_spec = self.comp_lookup.GetFirstSpec()
else:
# (already processed alias_first)
base_opts, user_spec = self.comp_lookup.GetSpecForName(
first)
if not user_spec:
base_opts, user_spec = self.comp_lookup.GetFallback(
)
else:
done = True # exhausted candidates without getting a retry
finally:
self.compopt_state.currently_completing = False
def Eval(self, line):
"""Returns an expanded line."""
if not self.readline_mod:
return line
tokens = list(HISTORY_LEXER.Tokens(line))
# Common case: no history expansion.
if all(id_ == Id.History_Other for (id_, _) in tokens):
return line
history_len = self.readline_mod.get_current_history_length()
if history_len <= 0: # no commands to expand
return line
self.debug_f.log('history length = %d', history_len)
parts = []
for id_, val in tokens:
if id_ == Id.History_Other:
out = val
elif id_ == Id.History_Op:
prev = self.readline_mod.get_history_item(history_len)
ch = val[1]
if ch == '!':
out = prev
else:
self.parse_ctx.trail.Clear() # not strictly necessary?
line_reader = reader.StringLineReader(
prev, self.parse_ctx.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
try:
c_parser.ParseLogicalLine()
except util.ParseError as e:
#from core import ui
#ui.PrettyPrintError(e, self.parse_ctx.arena)
# Invalid command in history. TODO: We should never enter these.
self.debug_f.log(
"Couldn't parse historical command %r: %s", prev,
e)
# NOTE: We're using the trail rather than the return value of
# ParseLogicalLine because it handles cases like
# $ for i in 1 2 3; do sleep ${i}; done
# $ echo !$
# which should expand to 'echo ${i}'
words = self.parse_ctx.trail.words
#self.debug_f.log('TRAIL WORDS: %s', words)
if ch == '^':
try:
w = words[1]
except IndexError:
raise util.HistoryError("No first word in %r",
prev)
spid1 = word_.LeftMostSpanForWord(w)
spid2 = word_.RightMostSpanForWord(w)
elif ch == '$':
try:
w = words[-1]
except IndexError:
raise util.HistoryError("No last word in %r", prev)
spid1 = word_.LeftMostSpanForWord(w)
spid2 = word_.RightMostSpanForWord(w)
elif ch == '*':
try:
w1 = words[1]
w2 = words[-1]
except IndexError:
raise util.HistoryError(
"Couldn't find words in %r", prev)
spid1 = word_.LeftMostSpanForWord(w1)
spid2 = word_.RightMostSpanForWord(w2)
else:
raise AssertionError(ch)
arena = self.parse_ctx.arena
span1 = arena.GetLineSpan(spid1)
span2 = arena.GetLineSpan(spid2)
begin = span1.col
end = span2.col + span2.length
out = prev[begin:end]
elif id_ == Id.History_Num:
index = int(
val[1:]) # regex ensures this. Maybe have - on the front.
if index < 0:
num = history_len + 1 + index
else:
num = index
out = self.readline_mod.get_history_item(num)
if out is None: # out of range
raise util.HistoryError('%s: not found', val)
elif id_ == Id.History_Search:
# Remove the required space at the end and save it. A simple hack than
# the one bash has.
last_char = val[-1]
val = val[:-1]
# Search backward
prefix = None
substring = None
if val[1] == '?':
substring = val[2:]
else:
prefix = val[1:]
out = None
for i in xrange(history_len, 1, -1):
cmd = self.readline_mod.get_history_item(i)
if prefix and cmd.startswith(prefix):
out = cmd
if substring and substring in cmd:
out = cmd
if out is not None:
out += last_char # restore required space
break
if out is None:
raise util.HistoryError('%r found no results', val)
else:
raise AssertionError(id_)
parts.append(out)
line = ''.join(parts)
# show what we expanded to
sys.stdout.write('! %s' % line)
return line
