def EvalWordToAny(self, word, glob_escape=False):
"""
Used for RHS of assignment
Also used for default value? e.g. "${a:-"a" "b"}" and so forth.
Returns:
arg_value
Or maybe just string? Whatever would go in ConstArg and GlobArg.
But you don't need to distinguish it later.
You could also have EvalWord and EvalGlobWord methods or EvalPatternWord
"""
# Special case for a=(1 2). ArrayLiteralPart won't appear in words that
# don't look like assignments.
if (len(word.parts) == 1
and word.parts[0].tag == word_part_e.ArrayLiteralPart):
array_words = word.parts[0].words
words = braces.BraceExpandWords(array_words)
strs = self._EvalWordSequence(words)
#log('ARRAY LITERAL EVALUATED TO -> %s', strs)
return runtime.StrArray(strs)
part_vals = self._EvalParts(word)
#log('part_vals %s', part_vals)
# Instead of splitting, do a trivial transformation to frag array.
# Example:
# foo="-$@-${a[@]}-" requires fragment, reframe, and simple join
frag_arrays = []
for p in part_vals:
if p.tag == part_value_e.StringPartValue:
frag_arrays.append([runtime.fragment(p.s, False, False)])
elif p.tag == part_value_e.ArrayPartValue:
frag_arrays.append(
[runtime.fragment(s, False, False) for s in p.strs])
else:
raise AssertionError
frag_arrays = _Reframe(frag_arrays)
#log('frag_arrays %s', frag_arrays)
# Simple join
args = []
for frag_array in frag_arrays:
args.append(''.join(frag.s for frag in frag_array))
# Example:
# a=(1 2)
# b=$a # one word
# c="${a[@]}" # two words
if len(args) == 1:
val = runtime.Str(args[0])
else:
# NOTE: For bash compatibility, could have an option to join them here.
# foo="-$@-${a[@]}-" -- join with IFS again, like "$*" ?
# Or maybe do that in cmd_exec in assignment.
val = runtime.StrArray(args)
return val
def GetVar(self, name, lookup_mode=scope_e.Dynamic):
assert isinstance(name, str), name
# Do lookup of system globals before looking at user variables. Note: we
# could optimize this at compile-time like $?. That would break
# ${!varref}, but it's already broken for $?.
if name == 'FUNCNAME':
# bash wants it in reverse order. This is a little inefficient but we're
# not depending on deque().
strs = list(reversed(self.func_name_stack))
# TODO: Reuse this object too?
return runtime.StrArray(strs)
if name == 'LINENO':
return self.line_num
# Instead of BASH_SOURCE. Using Oil _ convnetion.
if name == 'SOURCE_NAME':
return self.source_name
cell, _ = self._FindCellAndNamespace(name, lookup_mode, is_read=True)
if cell:
return cell.val
return runtime.Undef()
def _EvalSpecialVar(self, op_id, quoted):
# $@ is special -- it need to know whether it is in a double quoted
# context.
#
# - If it's $@ in a double quoted context, return an ARRAY.
# - If it's $@ in a normal context, return a STRING, which then will be
# subject to splitting.
if op_id in (Id.VSub_At, Id.VSub_Star):
argv = self.mem.GetArgv()
val = runtime.StrArray(argv)
if op_id == Id.VSub_At:
# "$@" evaluates to an array, $@ should be decayed
return val, not quoted
else: # $@ $* "$*"
return val, True
elif op_id == Id.VSub_QMark: # $?
# TODO: Have to parse status somewhere.
# External commands need WIFEXITED test. What about subshells?
return runtime.Str(str(self.mem.last_status)), False
elif op_id == Id.VSub_Pound: # $#
n = self.mem.GetNumArgs()
return runtime.Str(str(n)), False
else:
raise NotImplementedError(op_id)
def _ApplyUnarySuffixOp(self, val, op):
assert val.tag != value_e.Undef
op_kind = LookupKind(op.op_id)
if op_kind == Kind.VOp1:
#log('%s', op)
arg_val = self.word_ev.EvalWordToString(op.arg_word,
do_fnmatch=True)
assert arg_val.tag == value_e.Str
if val.tag == value_e.Str:
s = _DoUnarySuffixOp(val.s, op, arg_val.s)
new_val = runtime.Str(s)
else: # val.tag == value_e.StrArray:
# ${a[@]#prefix} is VECTORIZED on arrays. Oil should have this too.
strs = []
for s in val.strs:
strs.append(_DoUnarySuffixOp(s, op, arg_val.s))
new_val = runtime.StrArray(strs)
else:
raise AssertionError(op_kind)
return new_val
def _ApplyUnarySuffixOp(self, val, op):
# NOTES:
# - These are VECTORIZED on arrays
# - I want to allow this with my shell dialect: @{files|slice 1
# 2|upper} does the same thing to all of them.
# - How to do longest and shortest possible match? bash and mksh both
# call fnmatch() in a loop, with possible short-circuit optimizations.
# - TODO: Write a test program to show quadratic behavior?
# - original AT&T ksh has special glob routines that returned the match
# positions.
# Implementation:
# - Test if it is a LITERAL or a Glob. Then do a plain string
# operation.
# - If it's a glob, do the quadratic algorithm.
# - NOTE: bash also has an optimization where it extracts the LITERAL
# parts of the string, and does a prematch. If none of them match,
# then it SKIPs the quadratic algorithm.
# - The real solution is to compile a glob to RE2, but I want to avoid
# that dependency right now... libc regex is good for a bunch of
# things.
# - Bash has WIDE CHAR support for this. With wchar_t.
# - All sorts of functions like xdupmbstowcs
#
# And then pat_subst() does some special cases. Geez.
assert val.tag != value_e.Undef
op_kind = LookupKind(op.op_id)
new_val = None
# TODO: Vectorization should be factored out of all the branches.
if op_kind == Kind.VOp1:
#log('%s', op)
arg_val = self.word_ev.EvalWordToString(op.arg_word,
do_fnmatch=True)
assert arg_val.tag == value_e.Str
if val.tag == value_e.Str:
s = self._DoUnarySuffixOp(val.s, op, arg_val.s)
new_val = runtime.Str(s)
else: # val.tag == value_e.StrArray:
strs = []
for s in val.strs:
strs.append(self._DoUnarySuffixOp(s, op, arg_val.s))
new_val = runtime.StrArray(strs)
else:
raise AssertionError(op_kind)
if new_val:
return new_val
else:
return val
def GetVar(self, name, lookup_mode=scope.Dynamic):
assert isinstance(name, str), name
# Do lookup of system globals before looking at user variables. Note: we
# could optimize this at compile-time like $?. That would break
# ${!varref}, but it's already broken for $?.
if name == 'FUNCNAME':
# bash wants it in reverse order. This is a little inefficient but we're
# not depending on deque().
strs = list(reversed(self.func_name_stack))
return runtime.StrArray(strs)
cell, _ = self._FindCellAndNamespace(name, lookup_mode)
if cell:
return cell.val
return runtime.Undef()
def _EvalSpecialVar(self, op_id, quoted):
# $@ is special -- it need to know whether it is in a double quoted
# context.
#
# - If it's $@ in a double quoted context, return an ARRAY.
# - If it's $@ in a normal context, return a STRING, which then will be
# subject to splitting.
if op_id in (Id.VSub_At, Id.VSub_Star):
argv = self.mem.GetArgv()
val = runtime.StrArray(argv)
if op_id == Id.VSub_At:
# "$@" evaluates to an array, $@ should be decayed
return val, not quoted
else: # $@ $* "$*"
return val, True
elif op_id == Id.VSub_Hyphen:
s = self.exec_opts.GetDollarHyphen()
return runtime.Str(s), False
else:
val = self.mem.GetSpecialVar(op_id)
return val, False # dont' decay
def EvalRhsWord(self, word):
"""word_t -> value_t.
Used for RHS of assignment. There is no splitting.
Args:
word.CompoundWord
Returns:
value
"""
# Special case for a=(1 2). ArrayLiteralPart won't appear in words that
# don't look like assignments.
if (len(word.parts) == 1 and
word.parts[0].tag == word_part_e.ArrayLiteralPart):
array_words = word.parts[0].words
words = braces.BraceExpandWords(array_words)
strs = self._EvalWordSequence(words)
#log('ARRAY LITERAL EVALUATED TO -> %s', strs)
return runtime.StrArray(strs)
# If RHS doens't look like a=( ... ), then it must be a string.
return self.EvalWordToString(word)
def _EvalBracedVarSub(self, part, part_vals, quoted):
"""
Args:
part_vals: output param to append to.
"""
# We have four types of operator that interact.
#
# 1. Bracket: value -> (value, bool decay_array)
#
# 2. Then these four cases are mutually exclusive:
#
# a. Prefix length: value -> value
# b. Test: value -> part_value[]
# c. Other Suffix: value -> value
# d. no operator: you have a value
#
# That is, we don't have both prefix and suffix operators.
#
# 3. Process decay_array here before returning.
decay_array = False # for $*, ${a[*]}, etc.
var_name = None # For ${foo=default}
# 1. Evaluate from (var_name, var_num, token Id) -> value
if part.token.id == Id.VSub_Name:
var_name = part.token.val
val = self.mem.GetVar(var_name)
#log('EVAL NAME %s -> %s', var_name, val)
elif part.token.id == Id.VSub_Number:
var_num = int(part.token.val)
val = self._EvalVarNum(var_num)
else:
# $* decays
val, decay_array = self._EvalSpecialVar(part.token.id, quoted)
# 2. Bracket: value -> (value v, bool decay_array)
# decay_array is for joining ${a[*]} and unquoted ${a[@]} AFTER suffix ops
# are applied. If we take the length with a prefix op, the distinction is
# ignored.
if part.bracket_op:
if part.bracket_op.tag == bracket_op_e.WholeArray:
op_id = part.bracket_op.op_id
if op_id == Id.Lit_At:
if not quoted:
decay_array = True # ${a[@]} decays but "${a[@]}" doesn't
if val.tag == value_e.Undef:
val = self._EmptyStrArrayOrError(part.token)
elif val.tag == value_e.Str:
e_die("Can't index string with @: %r", val, part=part)
elif val.tag == value_e.StrArray:
val = runtime.StrArray(val.strs)
elif op_id == Id.Arith_Star:
decay_array = True # both ${a[*]} and "${a[*]}" decay
if val.tag == value_e.Undef:
val = self._EmptyStrArrayOrError(part.token)
elif val.tag == value_e.Str:
e_die("Can't index string with *: %r", val, part=part)
elif val.tag == value_e.StrArray:
# Always decay_array with ${a[*]} or "${a[*]}"
val = runtime.StrArray(val.strs)
else:
raise AssertionError(op_id) # unknown
elif part.bracket_op.tag == bracket_op_e.ArrayIndex:
anode = part.bracket_op.expr
index = self.arith_ev.Eval(anode)
if val.tag == value_e.Undef:
pass # it will be checked later
elif val.tag == value_e.Str:
# TODO: Implement this as an extension. Requires unicode support.
# Bash treats it as an array.
e_die("Can't index string %r with integer", part.token.val)
elif val.tag == value_e.StrArray:
try:
s = val.strs[index]
except IndexError:
val = runtime.Undef()
else:
val = runtime.Str(s)
else:
raise AssertionError(part.bracket_op.tag)
if part.prefix_op:
val = self._EmptyStrOrError(val) # maybe error
val = self._ApplyPrefixOp(val, part.prefix_op)
# At least for length, we can't have a test or suffix afterward.
elif part.suffix_op:
op = part.suffix_op
if op.tag == suffix_op_e.StringUnary:
if LookupKind(part.suffix_op.op_id) == Kind.VTest:
# TODO: Change style to:
# if self._ApplyTestOp(...)
# return
# It should return whether anything was done. If not, we continue to
# the end, where we might throw an error.
assign_part_vals, effect = self._ApplyTestOp(val, part.suffix_op,
quoted, part_vals)
# NOTE: Splicing part_values is necessary because of code like
# ${undef:-'a b' c 'd # e'}. Each part_value can have a different
# do_glob/do_elide setting.
if effect == Effect.SpliceParts:
return # EARLY RETURN, part_vals mutated
elif effect == Effect.SpliceAndAssign:
if var_name is None:
# TODO: error context
e_die("Can't assign to special variable")
else:
# NOTE: This decays arrays too! 'set -o strict_array' could
# avoid it.
rhs_str = _DecayPartValuesToString(assign_part_vals,
self.splitter.GetJoinChar())
state.SetLocalString(self.mem, var_name, rhs_str)
return # EARLY RETURN, part_vals mutated
elif effect == Effect.Error:
raise NotImplementedError
else:
# The old one
#val = self._EmptyStringPartOrError(part_val, quoted)
pass # do nothing, may still be undefined
else:
val = self._EmptyStrOrError(val) # maybe error
# Other suffix: value -> value
val = self._ApplyUnarySuffixOp(val, part.suffix_op)
elif op.tag == suffix_op_e.PatSub: # PatSub, vectorized
val = self._EmptyStrOrError(val)
pat_val = self.EvalWordToString(op.pat, do_fnmatch=True)
assert pat_val.tag == value_e.Str, pat_val
if op.replace:
replace_val = self.EvalWordToString(op.replace, do_fnmatch=True)
assert replace_val.tag == value_e.Str, replace_val
replace_str = replace_val.s
else:
replace_str = ''
pat = pat_val.s
if val.tag == value_e.Str:
s = libstr.PatSub(val.s, op, pat, replace_str)
val = runtime.Str(s)
elif val.tag == value_e.StrArray:
strs = []
for s in val.strs:
strs.append(libstr.PatSub(s, op, pat, replace_str))
val = runtime.StrArray(strs)
else:
raise AssertionError(val.__class__.__name__)
elif op.tag == suffix_op_e.Slice:
# NOTE: The beginning can be negative, but Python handles this. Might
# want to make it explicit.
# TODO: Check out of bounds errors? begin > end?
if op.begin:
begin = self.arith_ev.Eval(op.begin)
else:
begin = 0
if op.length:
length = self.arith_ev.Eval(op.length)
end = begin + length
else:
end = None # Python supports None as the end
if val.tag == value_e.Str: # Slice characters in a string.
# TODO: Need to support unicode? Write spec # tests.
val = runtime.Str(val.s[begin : end])
elif val.tag == value_e.StrArray: # Slice array entries.
val = runtime.StrArray(val.strs[begin : end])
else:
raise AssertionError(val.__class__.__name__)
# After applying suffixes, process decay_array here.
if decay_array:
val = self._DecayArray(val)
# No prefix or suffix ops
val = self._EmptyStrOrError(val)
# For example, ${a} evaluates to value_t.Str(), but we want a
# part_value.StringPartValue.
part_val = _ValueToPartValue(val, quoted)
part_vals.append(part_val)
def _EmptyStrArrayOrError(self):
if self.exec_opts.nounset:
self._AddErrorContext('Undefined array')
raise _EvalError()
else:
return runtime.StrArray([])
def _ApplyOtherSuffixOp(self, val, op):
# NOTES:
# - These are VECTORIZED on arrays
# - I want to allow this with my shell dialect: @{files|slice 1
# 2|upper} does the same thing to all of them.
# - How to do longest and shortest possible match? bash and mksh both
# call fnmatch() in a loop, with possible short-circuit optimizations.
# - TODO: Write a test program to show quadratic behavior?
# - original AT&T ksh has special glob routines that returned the match
# positions.
# Implementation:
# - Test if it is a LITERAL or a Glob. Then do a plain string
# operation.
# - If it's a glob, do the quadratic algorithm.
# - NOTE: bash also has an optimization where it extracts the LITERAL
# parts of the string, and does a prematch. If none of them match,
# then it SKIPs the quadratic algorithm.
# - The real solution is to compile a glob to RE2, but I want to avoid
# that dependency right now... libc regex is good for a bunch of
# things.
# - Bash has WIDE CHAR support for this. With wchar_t.
# - All sorts of functions like xdupmbstowcs
#
# And then pat_subst() does some special cases. Geez.
assert val.tag != value_e.Undef
op_kind = LookupKind(op.op_id)
new_val = None
if op_kind == Kind.VOp1:
#log('%s', op)
arg_val = self.word_ev.EvalWordToString(op.arg_word,
do_fnmatch=True)
assert arg_val.tag == value_e.Str
looks_like_glob = False
if looks_like_glob:
if op.op_id == Id.VOp1_Pound: # shortest prefix
raise NotImplementedError
elif op.op_id == Id.VOp1_DPound: # longest prefix
raise NotImplementedError
elif op.op_id == Id.VOp1_Percent: # shortest suffix
raise NotImplementedError
elif op.op_id == Id.VOp1_DPercent: # longest suffix
raise NotImplementedError
else:
raise AssertionError(op.op_id)
else:
op_str = arg_val.s
# TODO: Factor these out into a common fuction?
if op.op_id in (Id.VOp1_Pound, Id.VOp1_DPound): # const prefix
prefix = op_str
if val.tag == value_e.Str:
if val.s.startswith(prefix):
# Mutate it so we preserve the flags.
new_val = runtime.Str(val.s[len(prefix):])
else:
#log("Str: %r doesn't end with %r", val.s, suffix)
pass
elif val.tag == value_e.StrArray:
new_val = runtime.StrArray()
for i, s in enumerate(val.strs):
if s.startswith(prefix):
# Mutate it so we preserve the flags.
new_s = s[len(prefix):]
#log('%s -> %s', s, s[:-len(suffix)])
else:
new_s = s
#log("Array: %r doesn't end with %r", s, suffix)
new_val.strs.append(new_s)
elif op.op_id in (Id.VOp1_Percent,
Id.VOp1_DPercent): # const suffix
suffix = op_str
if val.tag == value_e.Str:
if val.s.endswith(suffix):
# Mutate it so we preserve the flags.
new_val = runtime.Str(val.s[:-len(suffix)])
else:
#log("Str: %r doesn't end with %r", val.s, suffix)
pass
elif val.tag == value_e.StrArray:
new_val = runtime.StrArray()
for i, s in enumerate(val.strs):
if s.endswith(suffix):
# Mutate it so we preserve the flags.
new_s = s[:-len(suffix)]
#log('%s -> %s', s, s[:-len(suffix)])
else:
new_s = s
#log("Array: %r doesn't end with %r", s, suffix)
new_val.strs.append(new_s)
else:
raise AssertionError(op.op_id)
elif op_kind == Kind.VOp2:
if op.op_id == Id.VOp2_Slash: # PatSub, vectorized
raise NotImplementedError
# Either string slicing or array slicing. However string slicing has a
# unicode problem? TODO: Test bash out. We need utf-8 parsing in C++?
#
# Or maybe have a different operator for byte slice and char slice.
elif op.op_id == Id.VOp2_Colon:
raise NotImplementedError
else:
raise NotImplementedError(op)
if new_val:
return new_val
else:
return val
def _EmptyStrArrayOrError(self, token):
assert token is not None
if self.exec_opts.nounset:
e_die('Undefined array %r', token.val, token=token)
else:
return runtime.StrArray([])
def SetGlobalArray(self, name, a): """Helper for completion.""" assert isinstance(a, list) val = runtime.StrArray(a) pairs = [(ast.LeftVar(name), val)] self.SetGlobals(pairs)
def _Dispatch(self, node, fork_external):
argv0 = None # for error message
check_errexit = True # for errexit
if node.tag == command_e.SimpleCommand:
# PROBLEM: We want to log argv in 'xtrace' mode, but we may have already
# redirected here, which screws up loggnig. For example, 'echo hi
# >/dev/null 2>&1'. We want to evaluate argv and log it BEFORE applying
# redirects.
# Another problem:
# - tracing can be called concurrently from multiple processes, leading
# to overlap. Maybe have a mode that creates a file per process.
# xtrace-proc
# - line numbers for every command would be very nice. But then you have
# to print the filename too.
words = braces.BraceExpandWords(node.words)
argv = self.ev.EvalWordSequence(words)
if argv:
argv0 = argv[0]
environ = self.mem.GetExported()
self._EvalEnv(node.more_env, environ)
if self.exec_opts.xtrace:
log('+ %s', argv)
#print('+ %s' % argv, file=sys.stderr)
#print('+ %s' % argv, file=self.XFILE)
#os.write(2, '+ %s\n' % argv)
status = self._RunSimpleCommand(argv, environ, fork_external)
if self.exec_opts.xtrace:
#log('+ %s -> %d', argv, status)
pass
elif node.tag == command_e.Sentence:
if node.terminator.id == Id.Op_Semi:
# Don't check_errexit since this isn't a real node!
check_errexit = False
status = self._Execute(node.child)
else:
status = self._RunJobInBackground(node.child)
elif node.tag == command_e.Pipeline:
if node.stderr_indices:
raise NotImplementedError('|&')
if node.negated:
self._PushErrExit()
try:
status2 = self._RunPipeline(node)
finally:
self._PopErrExit()
# errexit is disabled for !.
check_errexit = False
status = 1 if status2 == 0 else 0
else:
status = self._RunPipeline(node)
elif node.tag == command_e.Subshell:
# This makes sure we don't waste a process if we'd launch one anyway.
p = self._MakeProcess(node.child)
status = p.Run(self.waiter)
elif node.tag == command_e.DBracket:
result = self.bool_ev.Eval(node.expr)
status = 0 if result else 1
elif node.tag == command_e.DParen:
i = self.arith_ev.Eval(node.child)
status = 0 if i != 0 else 1
elif node.tag == command_e.Assignment:
pairs = []
if node.keyword == Id.Assign_Local:
lookup_mode = scope.LocalOnly
flags = ()
elif node.keyword == Id.Assign_Declare:
# declare is like local, except it can also be used outside functions?
lookup_mode = scope.LocalOnly
# TODO: Respect flags. -r and -x matter, but -a and -A might be
# implicit in the RHS?
flags = ()
elif node.keyword == Id.Assign_Readonly:
lookup_mode = scope.Dynamic
flags = (var_flags.ReadOnly,)
elif node.keyword == Id.Assign_None: # mutate existing local or global
lookup_mode = scope.Dynamic
flags = ()
else:
# TODO: typeset, declare, etc. Those are dynamic though.
raise NotImplementedError(node.keyword)
for pair in node.pairs:
if pair.rhs:
# RHS can be a string or array.
val = self.ev.EvalWordToAny(pair.rhs)
assert isinstance(val, runtime.value), val
else:
# 'local x' is equivalent to local x=""
val = runtime.Str('')
if pair.op == assign_op.PlusEqual:
old_val, lval = expr_eval.EvalLhs(pair.lhs, self.arith_ev, self.mem,
self.exec_opts)
sig = (old_val.tag, val.tag)
if sig == (value_e.Str, value_e.Str):
val = runtime.Str(old_val.s + val.s)
elif sig == (value_e.Str, value_e.StrArray):
e_die("Can't append array to string")
elif sig == (value_e.StrArray, value_e.Str):
e_die("Can't append string to array")
elif sig == (value_e.StrArray, value_e.StrArray):
val = runtime.StrArray(old_val.strs + val.strs)
else:
lval = self._EvalLhs(pair.lhs)
#log('ASSIGNING %s -> %s', lval, val)
self.mem.SetVar(lval, val, flags, lookup_mode)
# TODO: This should be eval of RHS, unlike bash!
status = 0
elif node.tag == command_e.ControlFlow:
if node.arg_word: # Evaluate the argument
val = self.ev.EvalWordToString(node.arg_word)
assert val.tag == value_e.Str
arg = int(val.s) # They all take integers
else:
arg = 0 # return 0, break 0 levels, etc.
# NOTE: always raises so we don't set status.
raise _ControlFlow(node.token, arg)
# The only difference between these two is that CommandList has no
# redirects. We already took care of that above.
elif node.tag in (command_e.CommandList, command_e.BraceGroup):
status = self._ExecuteList(node.children)
elif node.tag == command_e.AndOr:
#print(node.children)
left, right = node.children
# This is everything except the last one.
self._PushErrExit()
try:
status = self._Execute(left)
finally:
self._PopErrExit()
if node.op_id == Id.Op_DPipe:
if status != 0:
status = self._Execute(right)
elif node.op_id == Id.Op_DAmp:
if status == 0:
status = self._Execute(right)
else:
raise AssertionError
elif node.tag in (command_e.While, command_e.Until):
# TODO: Compile this out?
if node.tag == command_e.While:
_DonePredicate = lambda status: status != 0
else:
_DonePredicate = lambda status: status == 0
status = 0
while True:
self._PushErrExit()
try:
cond_status = self._ExecuteList(node.cond)
finally:
self._PopErrExit()
done = cond_status != 0
if _DonePredicate(cond_status):
break
try:
status = self._Execute(node.body) # last one wins
except _ControlFlow as e:
if e.IsBreak():
status = 0
break
elif e.IsContinue():
status = 0
continue
else: # return needs to pop up more
raise
elif node.tag == command_e.ForEach:
iter_name = node.iter_name
if node.do_arg_iter:
iter_list = self.mem.GetArgv()
else:
words = braces.BraceExpandWords(node.iter_words)
iter_list = self.ev.EvalWordSequence(words)
# We need word splitting and so forth
# NOTE: This expands globs too. TODO: We should pass in a Globber()
# object.
status = 0 # in case we don't loop
for x in iter_list:
#log('> ForEach setting %r', x)
state.SetLocalString(self.mem, iter_name, x)
#log('<')
try:
status = self._Execute(node.body) # last one wins
except _ControlFlow as e:
if e.IsBreak():
status = 0
break
elif e.IsContinue():
status = 0
continue
else: # return needs to pop up more
raise
elif node.tag == command_e.ForExpr:
raise NotImplementedError(node.tag)
elif node.tag == command_e.DoGroup:
status = self._ExecuteList(node.children)
elif node.tag == command_e.FuncDef:
# NOTE: Would it make sense to evaluate the redirects BEFORE entering?
# It will save time on function calls.
self.funcs[node.name] = node
status = 0
elif node.tag == command_e.If:
done = False
for arm in node.arms:
self._PushErrExit()
try:
status = self._ExecuteList(arm.cond)
finally:
self._PopErrExit()
if status == 0:
status = self._ExecuteList(arm.action)
done = True
break
# TODO: The compiler should flatten this
if not done and node.else_action is not None:
status = self._ExecuteList(node.else_action)
elif node.tag == command_e.NoOp:
status = 0 # make it true
elif node.tag == command_e.Case:
val = self.ev.EvalWordToString(node.to_match)
to_match = val.s
status = 0 # If there are no arms, it should be zero?
done = False
for arm in node.arms:
for pat_word in arm.pat_list:
# NOTE: Is it OK that we're evaluating these as we go?
pat_val = self.ev.EvalWordToString(pat_word, do_fnmatch=True)
#log('Matching word %r against pattern %r', to_match, pat_val.s)
if libc.fnmatch(pat_val.s, to_match):
status = self._ExecuteList(arm.action)
done = True # TODO: Parse ;;& and for fallthrough and such?
if done:
break
elif node.tag == command_e.TimeBlock:
# TODO:
# - When do we need RUSAGE_CHILDREN?
# - Respect TIMEFORMAT environment variable.
# "If this variable is not set, Bash acts as if it had the value"
# $'\nreal\t%3lR\nuser\t%3lU\nsys\t%3lS'
# "A trailing newline is added when the format string is displayed."
start_t = time.time() # calls gettimeofday() under the hood
start_u = resource.getrusage(resource.RUSAGE_SELF)
status = self._Execute(node.pipeline)
end_t = time.time()
end_u = resource.getrusage(resource.RUSAGE_SELF)
real = end_t - start_t
user = end_u.ru_utime - start_u.ru_utime
sys_ = end_u.ru_stime - start_u.ru_stime
print('real\t%.3f' % real, file=sys.stderr)
print('user\t%.3f' % user, file=sys.stderr)
print('sys\t%.3f' % sys_, file=sys.stderr)
else:
raise AssertionError(node.tag)
return status, check_errexit
def testSetVarClearFlag(self):
mem = state.Mem('', [], {})
print(mem)
mem.PushCall('my-func', ['ONE'])
self.assertEqual(2, len(mem.var_stack)) # internal details
# local x=y
mem.SetVar(runtime.LhsName('x'), runtime.Str('y'), (), scope.LocalOnly)
self.assertEqual('y', mem.var_stack[-1]['x'].val.s)
# New frame
mem.PushCall('my-func', ['TWO'])
self.assertEqual(3, len(mem.var_stack)) # internal details
# x=y -- test out dynamic scope
mem.SetVar(runtime.LhsName('x'), runtime.Str('YYY'), (), scope.Dynamic)
self.assertEqual('YYY', mem.var_stack[-2]['x'].val.s)
self.assertEqual(None, mem.var_stack[-1].get('x'))
# myglobal=g
mem.SetVar(runtime.LhsName('myglobal'), runtime.Str('g'), (),
scope.Dynamic)
self.assertEqual('g', mem.var_stack[0]['myglobal'].val.s)
self.assertEqual(False, mem.var_stack[0]['myglobal'].exported)
# 'export PYTHONPATH=/'
mem.SetVar(runtime.LhsName('PYTHONPATH'), runtime.Str('/'),
(var_flags.Exported, ), scope.Dynamic)
self.assertEqual('/', mem.var_stack[0]['PYTHONPATH'].val.s)
self.assertEqual(True, mem.var_stack[0]['PYTHONPATH'].exported)
self.assertEqual({'PYTHONPATH': '/'}, mem.GetExported())
mem.SetVar(runtime.LhsName('PYTHONPATH'), None, (var_flags.Exported, ),
scope.Dynamic)
self.assertEqual(True, mem.var_stack[0]['PYTHONPATH'].exported)
# 'export myglobal'. None means don't touch it. Undef would be confusing
# because it might mean "unset", but we have a separated API for that.
mem.SetVar(runtime.LhsName('myglobal'), None, (var_flags.Exported, ),
scope.Dynamic)
self.assertEqual(True, mem.var_stack[0]['myglobal'].exported)
# export g2 -- define and export empty
mem.SetVar(runtime.LhsName('g2'), None, (var_flags.Exported, ),
scope.Dynamic)
self.assertEqual('', mem.var_stack[0]['g2'].val.s)
self.assertEqual(True, mem.var_stack[0]['g2'].exported)
# readonly myglobal
self.assertEqual(False, mem.var_stack[0]['myglobal'].readonly)
mem.SetVar(runtime.LhsName('myglobal'), None, (var_flags.ReadOnly, ),
scope.Dynamic)
self.assertEqual(True, mem.var_stack[0]['myglobal'].readonly)
mem.SetVar(runtime.LhsName('PYTHONPATH'), runtime.Str('/lib'), (),
scope.Dynamic)
self.assertEqual('/lib', mem.var_stack[0]['PYTHONPATH'].val.s)
self.assertEqual(True, mem.var_stack[0]['PYTHONPATH'].exported)
# COMPREPLY=(1 2 3)
# invariant to enforce: arrays can't be exported
mem.SetVar(runtime.LhsName('COMPREPLY'),
runtime.StrArray(['1', '2', '3']), (), scope.GlobalOnly)
self.assertEqual(['1', '2', '3'],
mem.var_stack[0]['COMPREPLY'].val.strs)
# export COMPREPLY
try:
mem.SetVar(runtime.LhsName('COMPREPLY'), None,
(var_flags.Exported, ), scope.Dynamic)
except util.FatalRuntimeError as e:
pass
else:
self.fail("Expected failure")
# readonly r=1
mem.SetVar(runtime.LhsName('r'), runtime.Str('1'),
(var_flags.ReadOnly, ), scope.Dynamic)
self.assertEqual('1', mem.var_stack[0]['r'].val.s)
self.assertEqual(False, mem.var_stack[0]['r'].exported)
self.assertEqual(True, mem.var_stack[0]['r'].readonly)
print(mem)
# r=newvalue
try:
mem.SetVar(runtime.LhsName('r'), runtime.Str('newvalue'), (),
scope.Dynamic)
except util.FatalRuntimeError as e:
pass
else:
self.fail("Expected failure")
# readonly r2 -- define empty readonly
mem.SetVar(runtime.LhsName('r2'), None, (var_flags.ReadOnly, ),
scope.Dynamic)
self.assertEqual('', mem.var_stack[0]['r2'].val.s)
self.assertEqual(True, mem.var_stack[0]['r2'].readonly)
# export -n PYTHONPATH
# Remove the exported property. NOTE: scope is LocalOnly for Oil?
self.assertEqual(True, mem.var_stack[0]['PYTHONPATH'].exported)
mem.ClearFlag('PYTHONPATH', var_flags.Exported, scope.Dynamic)
self.assertEqual(False, mem.var_stack[0]['PYTHONPATH'].exported)
# a[2]=2
mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('2'), (),
scope.Dynamic)
self.assertEqual(['', '2'], mem.var_stack[0]['a'].val.strs)
# a[2]=3
mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('3'), (),
scope.Dynamic)
self.assertEqual(['', '3'], mem.var_stack[0]['a'].val.strs)
# a[2]=(x y z) # illegal
try:
mem.SetVar(runtime.LhsIndexedName('a', 1),
runtime.StrArray(['x', 'y', 'z']), (), scope.Dynamic)
except util.FatalRuntimeError as e:
pass
else:
self.fail("Expected failure")
# readonly a
mem.SetVar(runtime.LhsName('a'), None, (var_flags.ReadOnly, ),
scope.Dynamic)
self.assertEqual(True, mem.var_stack[0]['a'].readonly)
try:
# a[2]=3
mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('3'), (),
scope.Dynamic)
except util.FatalRuntimeError as e:
pass
else:
self.fail("Expected failure")
def _EvalBracedVarSub(self, part, quoted):
"""
Returns:
part_value[]
"""
# We have four types of operator that interact.
#
# 1. Bracket: value -> (value, bool decay_array)
#
# 2. Then these four cases are mutually exclusive:
#
# a. Prefix length: value -> value
# b. Test: value -> part_value[]
# c. Other Suffix: value -> value
# d. no operator: you have a value
#
# That is, we don't have both prefix and suffix operators.
#
# 3. Process decay_array here before returning.
decay_array = False # for $*, ${a[*]}, etc.
# 1. Evaluate from (var_name, var_num, token Id) -> value
if part.token.id == Id.VSub_Name:
var_name = part.token.val
val = self.mem.Get(var_name)
#log('EVAL NAME %s -> %s', var_name, val)
elif part.token.id == Id.VSub_Number:
var_num = int(part.token.val)
val = self._EvalVarNum(var_num)
else:
# $* decays
val, decay_array = self._EvalSpecialVar(part.token.id, quoted)
# 2. Bracket: value -> (value v, bool decay_array)
# decay_array is for joining ${a[*]} and unquoted ${a[@]} AFTER suffix ops
# are applied. If we take the length with a prefix op, the distinction is
# ignored.
if part.bracket_op:
if part.bracket_op.tag == bracket_op_e.WholeArray:
op_id = part.bracket_op.op_id
if op_id == Id.Lit_At:
if not quoted:
decay_array = True # ${a[@]} decays but "${a[@]}" doesn't
if val.tag == value_e.Undef:
val = self._EmptyStrArrayOrError(part.token)
elif val.tag == value_e.Str:
raise RuntimeError("Can't index string with @")
elif val.tag == value_e.StrArray:
val = runtime.StrArray(val.strs)
elif op_id == Id.Arith_Star:
decay_array = True # both ${a[*]} and "${a[*]}" decay
if val.tag == value_e.Undef:
val = self._EmptyStrArrayOrError(part.token)
elif val.tag == value_e.Str:
raise RuntimeError("Can't index string with *")
elif val.tag == value_e.StrArray:
# Always decay_array with ${a[*]} or "${a[*]}"
val = runtime.StrArray(val.strs)
else:
raise AssertionError(op_id) # unknown
elif part.bracket_op.tag == bracket_op_e.ArrayIndex:
anode = part.bracket_op.expr
# TODO: This should propagate errors
arith_ev = expr_eval.ArithEvaluator(self.mem, self.word_ev)
index = arith_ev.Eval(anode)
if val.tag == value_e.Undef:
pass # it will be checked later
elif val.tag == value_e.Str:
# TODO: Implement this as an extension. Requires unicode support.
# Bash treats it as an array.
e_die("Can't index string %r with integer", part.token.val)
elif val.tag == value_e.StrArray:
try:
s = val.strs[index]
except IndexError:
val = runtime.Undef()
else:
val = runtime.Str(s)
else:
raise AssertionError(part.bracket_op.tag)
if part.prefix_op:
val = self._EmptyStrOrError(val) # maybe error
val = self._ApplyPrefixOp(val, part.prefix_op)
# At least for length, we can't have a test or suffix afterward.
elif part.suffix_op:
out_part_vals = []
if LookupKind(part.suffix_op.op_id) == Kind.VTest:
# VTest: value -> part_value[]
new_part_vals, effect = self._ApplyTestOp(
val, part.suffix_op, quoted)
# NOTE: Splicing part_values is necessary because of code like
# ${undef:-'a b' c 'd # e'}. Each part_value can have a different
# do_glob/do_elide setting.
if effect == Effect.SpliceParts:
return new_part_vals # EARLY RETURN
elif effect == Effect.SpliceAndAssign:
raise NotImplementedError
elif effect == Effect.Error:
raise NotImplementedError
else:
# The old one
#val = self._EmptyStringPartOrError(part_val, quoted)
#out_part_vals.append(part_val)
pass # do nothing, may still be undefined
else:
val = self._EmptyStrOrError(val) # maybe error
# Other suffix: value -> value
val = self._ApplyOtherSuffixOp(val, part.suffix_op)
# After applying suffixes, process decay_array here.
if decay_array:
val = self._DecayArray(val)
# No prefix or suffix ops
val = self._EmptyStrOrError(val)
return [_ValueToPartValue(val, quoted)]
def _EvalBracedVarSub(self, part, quoted):
"""
Returns:
part_value[]
"""
# We have four types of operator that interact.
#
# 1. Bracket: value -> (value, bool decay_array)
#
# 2. Then these four cases are mutually exclusive:
#
# a. Prefix length: value -> value
# b. Test: value -> part_value[]
# c. Other Suffix: value -> value
# d. no operator: you have a value
#
# That is, we don't have both prefix and suffix operators.
#
# 3. Process decay_array here before returning.
decay_array = False # for $*, ${a[*]}, etc.
var_name = None # For ${foo=default}
# 1. Evaluate from (var_name, var_num, token Id) -> value
if part.token.id == Id.VSub_Name:
var_name = part.token.val
val = self.mem.GetVar(var_name)
#log('EVAL NAME %s -> %s', var_name, val)
elif part.token.id == Id.VSub_Number:
var_num = int(part.token.val)
val = self._EvalVarNum(var_num)
else:
# $* decays
val, decay_array = self._EvalSpecialVar(part.token.id, quoted)
# 2. Bracket: value -> (value v, bool decay_array)
# decay_array is for joining ${a[*]} and unquoted ${a[@]} AFTER suffix ops
# are applied. If we take the length with a prefix op, the distinction is
# ignored.
if part.bracket_op:
if part.bracket_op.tag == bracket_op_e.WholeArray:
op_id = part.bracket_op.op_id
if op_id == Id.Lit_At:
if not quoted:
decay_array = True # ${a[@]} decays but "${a[@]}" doesn't
if val.tag == value_e.Undef:
val = self._EmptyStrArrayOrError(part.token)
elif val.tag == value_e.Str:
raise RuntimeError("Can't index string with @")
elif val.tag == value_e.StrArray:
val = runtime.StrArray(val.strs)
elif op_id == Id.Arith_Star:
decay_array = True # both ${a[*]} and "${a[*]}" decay
if val.tag == value_e.Undef:
val = self._EmptyStrArrayOrError(part.token)
elif val.tag == value_e.Str:
raise RuntimeError("Can't index string with *")
elif val.tag == value_e.StrArray:
# Always decay_array with ${a[*]} or "${a[*]}"
val = runtime.StrArray(val.strs)
else:
raise AssertionError(op_id) # unknown
elif part.bracket_op.tag == bracket_op_e.ArrayIndex:
anode = part.bracket_op.expr
index = self.arith_ev.Eval(anode)
if val.tag == value_e.Undef:
pass # it will be checked later
elif val.tag == value_e.Str:
# TODO: Implement this as an extension. Requires unicode support.
# Bash treats it as an array.
e_die("Can't index string %r with integer", part.token.val)
elif val.tag == value_e.StrArray:
try:
s = val.strs[index]
except IndexError:
val = runtime.Undef()
else:
val = runtime.Str(s)
else:
raise AssertionError(part.bracket_op.tag)
if part.prefix_op:
val = self._EmptyStrOrError(val) # maybe error
val = self._ApplyPrefixOp(val, part.prefix_op)
# At least for length, we can't have a test or suffix afterward.
elif part.suffix_op:
out_part_vals = []
op = part.suffix_op
if op.tag == suffix_op_e.StringUnary:
if LookupKind(part.suffix_op.op_id) == Kind.VTest:
# VTest: value -> part_value[]
new_part_vals, effect = self._ApplyTestOp(
val, part.suffix_op, quoted)
# NOTE: Splicing part_values is necessary because of code like
# ${undef:-'a b' c 'd # e'}. Each part_value can have a different
# do_glob/do_elide setting.
if effect == Effect.SpliceParts:
return new_part_vals # EARLY RETURN
elif effect == Effect.SpliceAndAssign:
if var_name is None:
# TODO: error context
e_die("Can't assign to special variable")
else:
# NOTE: This decays arrays too! 'set -o strict_array' could
# avoid it.
rhs_str = _DecayPartValuesToString(
new_part_vals, _GetJoinChar(self.mem))
state.SetLocalString(self.mem, var_name, rhs_str)
return new_part_vals
elif effect == Effect.Error:
raise NotImplementedError
else:
# The old one
#val = self._EmptyStringPartOrError(part_val, quoted)
#out_part_vals.append(part_val)
pass # do nothing, may still be undefined
else:
val = self._EmptyStrOrError(val) # maybe error
# Other suffix: value -> value
val = self._ApplyUnarySuffixOp(val, part.suffix_op)
elif op.tag == suffix_op_e.PatSub: # PatSub, vectorized
pat_val = self.word_ev.EvalWordToString(op.pat,
do_fnmatch=True)
assert pat_val.tag == value_e.Str, pat_val
if op.replace:
replace_val = self.word_ev.EvalWordToString(
op.replace, do_fnmatch=True)
assert replace_val.tag == value_e.Str, replace_val
replace_str = replace_val.s
else:
replace_str = ''
pat = pat_val.s
if val.tag == value_e.Str:
s = self._PatSub(val.s, op, pat, replace_str)
val = runtime.Str(s)
elif val.tag == value_e.StrArray:
strs = []
for s in val.strs:
strs.append(self._PatSub(s, op, pat, replace_str))
val = runtime.StrArray(strs)
else:
raise AssertionError(val.tag)
elif op.tag == suffix_op_e.Slice:
# Either string slicing or array slicing. However string slicing has
# a unicode problem?
# Or maybe have a different operator for byte slice and char slice.
raise NotImplementedError(op)
# After applying suffixes, process decay_array here.
if decay_array:
val = self._DecayArray(val)
# No prefix or suffix ops
val = self._EmptyStrOrError(val)
return [_ValueToPartValue(val, quoted)]
def SetVar(self, lval, value, new_flags, lookup_mode):
"""
Args:
lval: lvalue
val: value, or None if only changing flags
new_flags: tuple of flags to set: ReadOnly | Exported
() means no flags to start with
None means unchanged?
scope:
Local | Global | Dynamic - for builtins, PWD, etc.
NOTE: in bash, PWD=/ changes the directory. But not in dash.
"""
# STRICTNESS / SANENESS:
#
# 1) Don't create arrays automatically, e.g. a[1000]=x
# 2) Never change types? yeah I think that's a good idea, at least for oil
# (not sh, for compatibility). set -o strict-types or something. That
# means arrays have to be initialized with let arr = [], which is fine.
# This helps with stuff like IFS. It starts off as a string, and assigning
# it to a list is en error. I guess you will have to turn this no for
# bash?
assert new_flags is not None
if lval.tag == lvalue_e.LhsName:
# Maybe this should return one of (cell, scope). existing cell, or the
# scope to put it in?
# _FindCellOrScope
cell, namespace = self._FindCellAndNamespace(lval.name, lookup_mode)
if cell:
if value is not None:
if cell.readonly:
# TODO: error context
e_die("Can't assign to readonly value %r", lval.name)
cell.val = value
if var_flags.Exported in new_flags:
cell.exported = True
if var_flags.ReadOnly in new_flags:
cell.readonly = True
else:
if value is None:
value = runtime.Str('') # export foo, readonly foo
cell = runtime.cell(value,
var_flags.Exported in new_flags ,
var_flags.ReadOnly in new_flags )
namespace[lval.name] = cell
if (cell.val is not None and cell.val.tag == value_e.StrArray and
cell.exported):
e_die("Can't export array") # TODO: error context
elif lval.tag == lvalue_e.LhsIndexedName:
# a[1]=(1 2 3)
if value.tag == value_e.StrArray:
e_die("Can't assign array to array member") # TODO: error context
cell, namespace = self._FindCellAndNamespace(lval.name, lookup_mode)
if cell:
if cell.val.tag != value_e.StrArray:
# s=x
# s[1]=y
e_die("Can't index non-array") # TODO: error context
if cell.readonly:
e_die("Can't assign to readonly value")
strs = cell.val.strs
try:
strs[lval.index] = value.s
except IndexError:
# Fill it in with None. It could look like this:
# ['1', 2, 3, None, None, '4', None]
# Then ${#a[@]} counts the entries that are not None.
#
# TODO: strict-array for Oil arrays won't auto-fill.
n = len(strs) - lval.index + 1
strs.extend([None] * n)
strs[lval.index] = value.s
else:
# TODO:
# - This is a bug, because a[2]=2 creates an array of length ONE, even
# though the index is two.
# - Maybe represent as hash table? Then it's not an ASDL type?
# representations:
# - array_item.Str array_item.Undef
# - parallel array: val.strs, val.undefs
# - or change ASDL type checking
# - ASDL language does not allow: StrArray(string?* strs)
# - or add dict to ASDL? Didn't it support obj?
# - finding the max index is linear time?
# - also you have to sort the indices
#
# array ops:
# a=(1 2)
# a[1]=x
# a+=(1 2)
# ${a[@]} - get all
# ${#a[@]} - length
# ${!a[@]} - keys
# That seems pretty minimal.
items = [''] * lval.index
items.append(value.s)
new_value = runtime.StrArray(items)
# arrays can't be exported
cell = runtime.cell(new_value, False,
var_flags.ReadOnly in new_flags)
namespace[lval.name] = cell
else:
raise AssertionError
def SetGlobalArray(mem, name, a): """Helper for completion.""" assert isinstance(a, list) mem.SetVar(ast.LhsName(name), runtime.StrArray(a), (), scope.GlobalOnly)
