def _MakeSimpleCommand(preparsed_list, suffix_words, redirects):
"""Create an command.SimpleCommand node."""
# FOO=(1 2 3) ls is not allowed.
for _, _, _, w in preparsed_list:
if word.HasArrayPart(w):
p_die("Environment bindings can't contain array literals", word=w)
# echo FOO=(1 2 3) is not allowed (but we should NOT fail on echo FOO[x]=1).
for w in suffix_words:
if word.HasArrayPart(w):
p_die("Commands can't contain array literals", word=w)
# NOTE: # In bash, {~bob,~jane}/src works, even though ~ isn't the leading
# character of the initial word.
# However, this means we must do tilde detection AFTER brace EXPANSION, not
# just after brace DETECTION like we're doing here.
# The BracedWordTree instances have to be expanded into CompoundWord
# instances for the tilde detection to work.
words2 = braces.BraceDetectAll(suffix_words)
words3 = word.TildeDetectAll(words2)
node = command.SimpleCommand()
node.words = words3
node.redirects = redirects
_AppendMoreEnv(preparsed_list, node.more_env)
return node
def _ReadArrayLiteralPart(self):
self._Next(lex_mode_e.Outer) # advance past (
self._Peek()
if self.cur_token.id != Id.Op_LParen:
p_die('Expected ( after =, got %r',
self.cur_token.val,
token=self.cur_token)
# MUST use a new word parser (with same lexer).
w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader)
words = []
while True:
w = w_parser.ReadWord(lex_mode_e.Outer)
assert w is not None
if w.tag == word_e.TokenWord:
word_id = word.CommandId(w)
if word_id == Id.Right_ArrayLiteral:
break
# Unlike command parsing, array parsing allows embedded \n.
elif word_id == Id.Op_Newline:
continue
else:
# TokenWord
p_die('Unexpected token in array literal: %r',
w.token.val,
word=w)
words.append(w)
words2 = braces.BraceDetectAll(words)
words3 = word.TildeDetectAll(words2)
return word_part.ArrayLiteralPart(words3)
def _ParseForEachLoop(self):
node = command.ForEach()
node.do_arg_iter = False
ok, iter_name, quoted = word.StaticEval(self.cur_word)
if not ok or quoted:
p_die("Loop variable name should be a constant", word=self.cur_word)
if not match.IsValidVarName(iter_name):
p_die("Invalid loop variable name", word=self.cur_word)
node.iter_name = iter_name
self._Next() # skip past name
self._NewlineOk()
in_spid = const.NO_INTEGER
semi_spid = const.NO_INTEGER
self._Peek()
if self.c_id == Id.KW_In:
self._Next() # skip in
in_spid = word.LeftMostSpanForWord(self.cur_word) + 1
iter_words, semi_spid = self.ParseForWords()
assert iter_words is not None
words2 = braces.BraceDetectAll(iter_words)
words3 = word.TildeDetectAll(words2)
node.iter_words = words3
elif self.c_id == Id.Op_Semi:
node.do_arg_iter = True # implicit for loop
self._Next()
elif self.c_id == Id.KW_Do:
node.do_arg_iter = True # implicit for loop
# do not advance
else: # for foo BAD
p_die('Unexpected word after for loop variable', word=self.cur_word)
node.spids.extend((in_spid, semi_spid))
body_node = self.ParseDoGroup()
assert body_node is not None
node.body = body_node
return node
def Matches(self, comp):
"""
Args:
comp: Callback args from readline. Readline uses set_completer_delims to
tokenize the string.
Returns a list of matches relative to readline's completion_delims.
We have to post-process the output of various completers.
"""
arena = self.parse_ctx.arena # Used by inner functions
# Pass the original line "out of band" to the completion callback.
line_until_tab = comp.line[:comp.end]
self.comp_ui_state.line_until_tab = line_until_tab
self.parse_ctx.trail.Clear()
line_reader = reader.StringLineReader(line_until_tab, self.parse_ctx.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader, emit_comp_dummy=True)
# We want the output from parse_ctx, so we don't use the return value.
try:
c_parser.ParseLogicalLine()
except util.ParseError as e:
# e.g. 'ls | ' will not parse. Now inspect the parser state!
pass
debug_f = self.debug_f
trail = self.parse_ctx.trail
if 1:
trail.PrintDebugString(debug_f)
#
# First try completing the shell language itself.
#
# NOTE: We get Eof_Real in the command state, but not in the middle of a
# BracedVarSub. This is due to the difference between the CommandParser
# and WordParser.
tokens = trail.tokens
last = -1
if tokens[-1].id == Id.Eof_Real:
last -= 1 # ignore it
try:
t1 = tokens[last]
except IndexError:
t1 = None
try:
t2 = tokens[last-1]
except IndexError:
t2 = None
debug_f.log('line: %r', comp.line)
debug_f.log('rl_slice from byte %d to %d: %r', comp.begin, comp.end,
comp.line[comp.begin:comp.end])
debug_f.log('t1 %s', t1)
debug_f.log('t2 %s', t2)
# Each of the 'yield' statements below returns a fully-completed line, to
# appease the readline library. The root cause of this dance: If there's
# one candidate, readline is responsible for redrawing the input line. OSH
# only displays candidates and never redraws the input line.
def _TokenStart(tok):
span = arena.GetLineSpan(tok.span_id)
return span.col
if t2: # We always have t1?
# echo $
if IsDollar(t2) and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $
for name in self.mem.VarNames():
yield line_until_tab + name # no need to quote var names
return
# echo ${
if t2.id == Id.Left_DollarBrace and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) + 2 # 2 for ${
for name in self.mem.VarNames():
yield line_until_tab + name # no need to quote var names
return
# echo $P
if t2.id == Id.VSub_DollarName and IsDummy(t1):
# Example: ${undef:-$P
# readline splits at ':' so we have to prepend '-$' to every completed
# variable name.
self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $
to_complete = t2.val[1:]
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[n:] # no need to quote var names
return
# echo ${P
if t2.id == Id.VSub_Name and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) # no offset
to_complete = t2.val
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[n:] # no need to quote var names
return
# echo $(( VAR
if t2.id == Id.Lit_ArithVarLike and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) # no offset
to_complete = t2.val
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[n:] # no need to quote var names
return
if trail.words:
# echo ~<TAB>
# echo ~a<TAB> $(home dirs)
# This must be done at a word level, and TildeDetectAll() does NOT help
# here, because they don't have trailing slashes yet! We can't do it on
# tokens, because otherwise f~a will complete. Looking at word_part is
# EXACTLY what we want.
parts = trail.words[-1].parts
if (len(parts) == 2 and
parts[0].tag == word_part_e.LiteralPart and
parts[1].tag == word_part_e.LiteralPart and
parts[0].token.id == Id.Lit_TildeLike and
parts[1].token.id == Id.Lit_CompDummy):
t2 = parts[0].token
# +1 for ~
self.comp_ui_state.display_pos = _TokenStart(parts[0].token) + 1
to_complete = t2.val[1:]
n = len(to_complete)
for u in pwd.getpwall(): # catch errors?
name = u.pw_name
if name.startswith(to_complete):
yield line_until_tab + ShellQuoteB(name[n:]) + '/'
return
# echo hi > f<TAB> (complete redirect arg)
if trail.redirects:
r = trail.redirects[-1]
# Only complete 'echo >', but not 'echo >&' or 'cat <<'
if (r.tag == redir_e.Redir and
REDIR_ARG_TYPES[r.op.id] == redir_arg_type_e.Path):
if WordEndsWithCompDummy(r.arg_word):
debug_f.log('Completing redirect arg')
try:
val = self.word_ev.EvalWordToString(r.arg_word)
except util.FatalRuntimeError as e:
debug_f.log('Error evaluating redirect word: %s', e)
return
if val.tag != value_e.Str:
debug_f.log("Didn't get a string from redir arg")
return
span_id = word.LeftMostSpanForWord(r.arg_word)
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
comp.Update(to_complete=val.s) # FileSystemAction uses only this
n = len(val.s)
action = FileSystemAction(add_slash=True)
for name in action.Matches(comp):
yield line_until_tab + ShellQuoteB(name[n:])
return
#
# We're not completing the shell language. Delegate to user-defined
# completion for external tools.
#
# Set below, and set on retries.
base_opts = None
user_spec = None
# Used on retries.
partial_argv = []
num_partial = -1
first = None
if trail.words:
# Now check if we're completing a word!
if WordEndsWithCompDummy(trail.words[-1]):
debug_f.log('Completing words')
#
# It didn't look like we need to complete var names, tilde, redirects,
# etc. Now try partial_argv, which may involve invoking PLUGINS.
# needed to complete paths with ~
words2 = word.TildeDetectAll(trail.words)
if 0:
debug_f.log('After tilde detection')
for w in words2:
print(w, file=debug_f)
if 0:
debug_f.log('words2:')
for w2 in words2:
debug_f.log(' %s', w2)
for w in words2:
try:
# TODO:
# - Should we call EvalWordSequence? But turn globbing off? It
# can do splitting and such.
# - We could have a variant to eval TildeSubPart to ~ ?
val = self.word_ev.EvalWordToString(w)
except util.FatalRuntimeError:
# Why would it fail?
continue
if val.tag == value_e.Str:
partial_argv.append(val.s)
else:
pass
debug_f.log('partial_argv: %s', partial_argv)
num_partial = len(partial_argv)
first = partial_argv[0]
alias_first = None
debug_f.log('alias_words: %s', trail.alias_words)
if trail.alias_words:
w = trail.alias_words[0]
try:
val = self.word_ev.EvalWordToString(w)
except util.FatalRuntimeError:
pass
alias_first = val.s
debug_f.log('alias_first: %s', alias_first)
if num_partial == 0: # should never happen because of Lit_CompDummy
raise AssertionError
elif num_partial == 1:
base_opts, user_spec = self.comp_lookup.GetFirstSpec()
# Display/replace since the beginning of the first word. Note: this
# is non-zero in the case of
# echo $(gr and
# echo `gr
span_id = word.LeftMostSpanForWord(trail.words[0])
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
self.debug_f.log('** DISPLAY_POS = %d', self.comp_ui_state.display_pos)
else:
base_opts, user_spec = self.comp_lookup.GetSpecForName(first)
if not user_spec and alias_first:
base_opts, user_spec = self.comp_lookup.GetSpecForName(alias_first)
if user_spec:
# Pass the aliased command to the user-defined function, and use
# it for retries.
first = alias_first
if not user_spec:
base_opts, user_spec = self.comp_lookup.GetFallback()
# Display since the beginning
span_id = word.LeftMostSpanForWord(trail.words[-1])
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
self.debug_f.log('words[-1]: %r', trail.words[-1])
self.debug_f.log('display_pos %d', self.comp_ui_state.display_pos)
# Update the API for user-defined functions.
index = len(partial_argv) - 1 # COMP_CWORD is -1 when it's empty
prev = '' if index == 0 else partial_argv[index-1]
comp.Update(first=first, to_complete=partial_argv[-1],
prev=prev, index=index, partial_argv=partial_argv)
# This happens in the case of [[ and ((, or a syntax error like 'echo < >'.
if not user_spec:
debug_f.log("Didn't find anything to complete")
return
# Reset it back to what was registered. User-defined functions can mutate
# it.
dynamic_opts = {}
self.compopt_state.dynamic_opts = dynamic_opts
self.compopt_state.currently_completing = True
try:
done = False
while not done:
try:
for candidate in self._PostProcess(
base_opts, dynamic_opts, user_spec, comp):
yield candidate
except _RetryCompletion as e:
debug_f.log('Got 124, trying again ...')
# Get another user_spec. The ShellFuncAction may have 'sourced' code
# and run 'complete' to mutate comp_lookup, and we want to get that
# new entry.
if num_partial == 0:
raise AssertionError
elif num_partial == 1:
base_opts, user_spec = self.comp_lookup.GetFirstSpec()
else:
# (already processed alias_first)
base_opts, user_spec = self.comp_lookup.GetSpecForName(first)
if not user_spec:
base_opts, user_spec = self.comp_lookup.GetFallback()
else:
done = True # exhausted candidates without getting a retry
finally:
self.compopt_state.currently_completing = False
def Matches(self, comp):
"""
Args:
comp: Callback args from readline. Readline uses set_completer_delims to
tokenize the string.
Returns a list of matches relative to readline's completion_delims.
We have to post-process the output of various completers.
"""
arena = self.parse_ctx.arena # Used by inner functions
self.parse_ctx.trail.Clear()
line_reader = reader.StringLineReader(comp.line, self.parse_ctx.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader,
emit_comp_dummy=True)
# We want the output from parse_ctx, so we don't use the return value.
try:
c_parser.ParseLogicalLine()
except util.ParseError as e:
# e.g. 'ls | ' will not parse. Now inspect the parser state!
pass
debug_f = self.debug_f
trail = self.parse_ctx.trail
if 1:
trail.PrintDebugString(debug_f)
# NOTE: We get Eof_Real in the command state, but not in the middle of a
# BracedVarSub. This is due to the difference between the CommandParser
# and WordParser.
tokens = trail.tokens
last = -1
if tokens[-1].id == Id.Eof_Real:
last -= 1 # ignore it
try:
t1 = tokens[last]
except IndexError:
t1 = None
try:
t2 = tokens[last - 1]
except IndexError:
t2 = None
debug_f.log('line: %r', comp.line)
debug_f.log('rl_slice from byte %d to %d: %r', comp.begin, comp.end,
comp.line[comp.begin:comp.end])
debug_f.log('t1 %s', t1)
debug_f.log('t2 %s', t2)
def _MakePrefix(tok, offset=0):
span = arena.GetLineSpan(tok.span_id)
return comp.line[comp.begin:span.col + offset]
#return comp.line[0 : span.col+offset]
if t2: # We always have t1?
if IsDollar(t2) and IsDummy(t1):
prefix = _MakePrefix(t2, offset=1)
for name in self.mem.VarNames():
yield prefix + name
return
# echo ${
if t2.id == Id.Left_VarSub and IsDummy(t1):
prefix = _MakePrefix(t2, offset=2) # 2 for ${
for name in self.mem.VarNames():
yield prefix + name
return
# echo $P
if t2.id == Id.VSub_DollarName and IsDummy(t1):
# Example: ${undef:-$P
# readline splits at ':' so we have to prepend '-$' to every completed
# variable name.
prefix = _MakePrefix(t2, offset=1) # 1 for $
to_complete = t2.val[1:]
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield prefix + name
return
# echo ${P
if t2.id == Id.VSub_Name and IsDummy(t1):
prefix = _MakePrefix(t2) # no offset
to_complete = t2.val
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield prefix + name
return
if t2.id == Id.Lit_ArithVarLike and IsDummy(t1):
prefix = _MakePrefix(t2) # no offset
to_complete = t2.val
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield prefix + name
return
# NOTE: Instead of looking at the column positions on line spans, we could
# look for IsDummy() on the rightmost LiteralPart(token) of words.
def LastColForWord(w):
span_id = word.RightMostSpanForWord(w)
span = arena.GetLineSpan(span_id)
debug_f.log('span %s', span)
debug_f.log('span col %d length %d', span.col, span.length)
return span.col + span.length
if trail.words:
# First check if we're completing a path that begins with ~.
#
# Complete tilde like 'echo ~' and 'echo ~a'. This must be done at a word
# level, and TildeDetectAll() does NOT help here, because they don't have
# trailing slashes yet! We can't do it on tokens, because otherwise f~a
# will complete. Looking at word_part is EXACTLY what we want.
parts = trail.words[-1].parts
if (len(parts) == 2 and parts[0].tag == word_part_e.LiteralPart
and parts[1].tag == word_part_e.LiteralPart
and parts[0].token.id == Id.Lit_TildeLike
and parts[1].token.id == Id.Lit_CompDummy):
t2 = parts[0].token
# NOTE: We're assuming readline does its job, and not bothering to
# compute the prefix. What are the incorrect corner cases?
prefix = '~'
to_complete = t2.val[1:]
for u in pwd.getpwall():
name = u.pw_name
if name.startswith(to_complete):
yield prefix + name + '/'
return
# Check if we should complete a redirect
if trail.redirects:
r = trail.redirects[-1]
# Only complete 'echo >', but not 'echo >&' or 'cat <<'
if (r.tag == redir_e.Redir
and REDIR_ARG_TYPES[r.op.id] == redir_arg_type_e.Path):
last_col = LastColForWord(r.arg_word)
if last_col == comp.end:
debug_f.log('Completing redirect arg')
try:
val = self.word_ev.EvalWordToString(r.arg_word)
except util.FatalRuntimeError as e:
debug_f.log('Error evaluating redirect word: %s', e)
return
if val.tag != value_e.Str:
debug_f.log("Didn't get a string from redir arg")
return
comp.Update(
to_complete=val.s) # FileSystemAction uses only this
action = FileSystemAction(add_slash=True)
for name in action.Matches(comp):
# TODO: form prefix from r.arg_word
yield name
return
base_opts = None
user_spec = None # Set below
if trail.words:
# Now check if we're completing a word!
last_col = LastColForWord(trail.words[-1])
debug_f.log('last_col for word: %d', last_col)
if last_col == comp.end: # We're not completing the last word!
debug_f.log('Completing words')
#
# It didn't look like we need to complete var names, tilde, redirects,
# etc. Now try partial_argv, which may involve invoking PLUGINS.
# needed to complete paths with ~
words2 = word.TildeDetectAll(trail.words)
if 0:
debug_f.log('After tilde detection')
for w in words2:
print(w, file=debug_f)
partial_argv = []
for w in words2:
try:
# TODO:
# - Should we call EvalWordSequence? But turn globbing off? It
# can do splitting and such.
# - We could have a variant to eval TildeSubPart to ~ ?
val = self.word_ev.EvalWordToString(w)
except util.FatalRuntimeError:
# Why would it fail?
continue
if val.tag == value_e.Str:
partial_argv.append(val.s)
else:
pass
debug_f.log('partial_argv: %s', partial_argv)
n = len(partial_argv)
# TODO: Form prefix for RootCompleter to add to user_spec candidates
if n == 0:
# We should never get this because of Lit_CompDummy.
raise AssertionError
elif n == 1:
# First
base_opts, user_spec = self.comp_lookup.GetFirstSpec()
else:
base_opts, user_spec = self.comp_lookup.GetSpecForName(
partial_argv[0])
# Update the API for user-defined functions.
index = len(
partial_argv) - 1 # COMP_CWORD is -1 when it's empty
prev = '' if index == 0 else partial_argv[index - 1]
comp.Update(first=partial_argv[0],
to_complete=partial_argv[-1],
prev=prev,
index=index,
partial_argv=partial_argv)
# This happens in the case of [[ and ((, or a syntax error like 'echo < >'.
if not user_spec:
debug_f.log("Didn't find anything to complete")
return
# Reset it back to what was registered. User-defined functions can mutate
# it.
dynamic_opts = {}
self.comp_state.dynamic_opts = dynamic_opts
self.comp_state.currently_completing = True
try:
done = False
while not done:
try:
for entry in self._PostProcess(base_opts, dynamic_opts,
user_spec, comp):
yield entry
except _RetryCompletion as e:
debug_f.log('Got 124, trying again ...')
n = len(partial_argv)
# Get another user_spec. The ShellFuncAction may have 'sourced' code
# and run 'complete' to mutate comp_lookup, and we want to get that
# new entry.
if n == 0:
raise AssertionError
elif n == 1:
# First
base_opts, user_spec = self.comp_lookup.GetFirstSpec()
else:
base_opts, user_spec = self.comp_lookup.GetSpecForName(
partial_argv[0])
else:
done = True # exhausted candidates without getting a retry
finally:
self.comp_state.currently_completing = False
def _ReadArrayLiteralPart(self):
# type: () -> word_part_t
"""
a=(1 2 3)
TODO: See osh/cmd_parse.py:164 for Id.Lit_ArrayLhsOpen, for a[x++]=1
We want:
A=(['x']=1 ["x"]=2 [$x$y]=3)
Maybe allow this as a literal string? Because I think I've seen it before?
Or maybe force people to patch to learn the rule.
A=([x]=4)
Starts with Lit_Other '[', and then it has Lit_ArrayLhsClose
Maybe enforce that ALL have keys or NONE of have keys.
"""
self._Next(lex_mode_e.ShCommand) # advance past (
self._Peek()
if self.cur_token.id != Id.Op_LParen:
p_die('Expected ( after =, got %r',
self.cur_token.val,
token=self.cur_token)
paren_spid = self.cur_token.span_id
# MUST use a new word parser (with same lexer).
w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader)
words = []
while True:
w = w_parser.ReadWord(lex_mode_e.ShCommand)
if isinstance(w, word__TokenWord):
word_id = word.CommandId(w)
if word_id == Id.Right_ArrayLiteral:
break
# Unlike command parsing, array parsing allows embedded \n.
elif word_id == Id.Op_Newline:
continue
else:
# TokenWord
p_die('Unexpected token in array literal: %r',
w.token.val,
word=w)
assert isinstance(w, word__CompoundWord) # for MyPy
words.append(w)
if not words: # a=() is empty indexed array
node = word_part.ArrayLiteralPart(
words) # type: ignore # invariant List?
node.spids.append(paren_spid)
return node
# If the first one is a key/value pair, then the rest are assumed to be.
pair = word.DetectAssocPair(words[0])
if pair:
pairs = [pair[0], pair[1]] # flat representation
n = len(words)
for i in xrange(1, n):
w = words[i]
pair = word.DetectAssocPair(w)
if not pair:
p_die("Expected associative array pair", word=w)
pairs.append(pair[0]) # flat representation
pairs.append(pair[1])
node = word_part.AssocArrayLiteral(
pairs) # type: ignore # invariant List?
node.spids.append(paren_spid)
return node
words2 = braces.BraceDetectAll(words)
words3 = word.TildeDetectAll(words2)
node = word_part.ArrayLiteralPart(words3)
node.spids.append(paren_spid)
return node
