def EvalLhs(node, arith_ev, mem, exec_opts):
"""Evaluate the operand for a++ a[0]++ as an R-value.
Used by Executor as well.
Args:
node: osh_ast.lhs_expr
Returns:
runtime.value, runtime.lvalue
"""
#log('lhs_expr NODE %s', node)
assert isinstance(node, ast.lhs_expr), node
if node.tag == lhs_expr_e.LhsName: # a = b
# Problem: It can't be an array?
# a=(1 2)
# (( a++ ))
lval = runtime.LhsName(node.name)
val = _LookupVar(node.name, mem, exec_opts)
elif node.tag == lhs_expr_e.LhsIndexedName: # a[1] = b
# See tdop.IsIndexable for valid values:
# - ArithVarRef (not LhsName): a[1]
# - FuncCall: f(x), 1
# - ArithBinary LBracket: f[1][1] -- no semantics for this?
index = arith_ev.Eval(node.index)
lval = runtime.LhsIndexedName(node.name, index)
val = mem.GetVar(node.name)
if val.tag == value_e.Str:
e_die("Can't assign to characters of string %r", node.name)
elif val.tag == value_e.Undef:
# It would make more sense for 'nounset' to control this, but bash
# doesn't work that way.
#if self.exec_opts.strict_arith:
# e_die('Undefined array %r', node.name) # TODO: error location
val = runtime.Str('')
elif val.tag == value_e.StrArray:
#log('ARRAY %s -> %s, index %d', node.name, array, index)
array = val.strs
# NOTE: Similar logic in RHS Arith_LBracket
try:
item = array[index]
except IndexError:
item = None
if item is None:
val = runtime.Str('')
else:
assert isinstance(item, str), item
val = runtime.Str(item)
else:
raise AssertionError(val.tag)
else:
raise AssertionError(node.tag)
return val, lval
def SetOption(self, opt_name, b):
""" For set -o, set +o, or shopt -s/-u -o. """
self._SetOption(opt_name, b)
val = self.mem.GetVar('SHELLOPTS')
assert val.tag == value_e.Str
shellopts = val.s
# Now check if SHELLOPTS needs to be updated. It may be exported.
#
# NOTE: It might be better to skip rewriting SEHLLOPTS in the common case
# where it is not used. We could do it lazily upon GET.
# Also, it would be slightly more efficient to update SHELLOPTS if
# settings were batched, Examples:
# - set -eu
# - shopt -s foo bar
if b:
if opt_name not in shellopts:
new_val = runtime.Str('%s:%s' % (shellopts, opt_name))
self.mem.InternalSetGlobal('SHELLOPTS', new_val)
else:
if opt_name in shellopts:
names = [n for n in shellopts.split(':') if n != opt_name]
new_val = runtime.Str(':'.join(names))
self.mem.InternalSetGlobal('SHELLOPTS', new_val)
def __init__(self, dollar0, argv, environ, arena, has_main=False):
self.dollar0 = dollar0
self.argv_stack = [_ArgFrame(argv)]
self.var_stack = [_StackFrame()]
# The debug_stack isn't strictly necessary for execution. We use it for
# crash dumps and for 3 parallel arrays: FUNCNAME, BASH_SOURCE,
# BASH_LINENO. The First frame points at the global vars and argv.
self.debug_stack = [(None, None, const.NO_INTEGER, 0, 0)]
self.bash_source = [] # for implementing BASH_SOURCE
self.has_main = has_main
if has_main:
self.bash_source.append(dollar0) # e.g. the filename
self.current_spid = const.NO_INTEGER
# Note: we're reusing these objects because they change on every single
# line! Don't want to allocate more than necsesary.
self.source_name = runtime.Str('')
self.line_num = runtime.Str('')
self.last_status = 0 # Mutable public variable
self.last_job_id = -1 # Uninitialized value mutable public variable
# Done ONCE on initialization
self.root_pid = os.getpid()
self._InitDefaults()
self._InitVarsFromEnv(environ)
self.arena = arena
def testPushTemp(self):
mem = _InitMem()
# x=1
mem.SetVar(runtime.LhsName('x'), runtime.Str('1'), (), scope_e.Dynamic)
self.assertEqual('1', mem.var_stack[-1].vars['x'].val.s)
mem.PushTemp()
self.assertEqual(2, len(mem.var_stack))
# Temporary frame is readonly
self.assertEqual(True, mem.var_stack[-1].readonly)
self.assertEqual(False, mem.var_stack[-2].readonly)
# x=temp E=3 read x <<< 'line'
mem.SetVar(runtime.LhsName('x'), runtime.Str('temp'), (),
scope_e.TempEnv)
mem.SetVar(runtime.LhsName('E'), runtime.Str('3'), (), scope_e.TempEnv)
mem.SetVar(runtime.LhsName('x'), runtime.Str('line'), (),
scope_e.LocalOnly)
self.assertEqual('3', mem.var_stack[-1].vars['E'].val.s)
self.assertEqual('temp', mem.var_stack[-1].vars['x'].val.s)
self.assertEqual('line', mem.var_stack[-2].vars['x'].val.s)
mem.PopTemp()
self.assertEqual(1, len(mem.var_stack))
self.assertEqual('line', mem.var_stack[-1].vars['x'].val.s)
def EvalWordToString(self, word, do_fnmatch=False, do_ere=False):
"""
Args:
word: CompoundWord
Used for redirect arg, ControlFlow arg, ArithWord, BoolWord, etc.
do_fnmatch is true for case $pat and RHS of [[ == ]].
pat="*.py"
case $x in
$pat) echo 'matches glob pattern' ;;
"$pat") echo 'equal to glob string' ;; # must be glob escaped
esac
TODO: Raise AssertionError if it has ExtGlobPart.
"""
if word.tag == word_e.EmptyWord:
return runtime.Str('')
part_vals = []
for p in word.parts:
self._EvalWordPart(p, part_vals, quoted=False)
strs = []
for part_val in part_vals:
if part_val.tag == part_value_e.StringPartValue:
# [[ foo == */"*".py ]] or case *.py) ... esac
if do_fnmatch and not part_val.do_split_glob:
s = glob_.GlobEscape(part_val.s)
elif do_ere and not part_val.do_split_glob:
s = glob_.ExtendedRegexEscape(part_val.s)
else:
s = part_val.s
else:
if self.exec_opts.strict_array:
# Examples: echo f > "$@"; local foo="$@"
# TODO: This attributes too coarsely, to the word rather than the
# parts. Problem: the word is a TREE of parts, but we only have a
# flat list of part_vals. The only case where we really get arrays
# is "$@", "${a[@]}", "${a[@]//pat/replace}", etc.
e_die(
"This word should evaluate to a string, but part of it was an "
"array",
word=word)
# TODO: Maybe add detail like this.
#e_die('RHS of assignment should only have strings. '
# 'To assign arrays, use b=( "${a[@]}" )')
else:
# It appears to not respect IFS
s = ' '.join(s for s in part_val.strs if s is not None)
strs.append(s)
return runtime.Str(''.join(strs))
def testPrompt(self):
arena = test_lib.MakeArena('<ui_test.py>')
ex = test_lib.InitExecutor(arena=arena)
p = ui.Prompt(arena, ex.parse_ctx, ex)
# Rgression for caching bug!
self.assertEqual('foo', p.EvalPrompt(runtime.Str('foo')))
self.assertEqual('foo', p.EvalPrompt(runtime.Str('foo')))
def testGetVar(self):
mem = _InitMem()
# readonly a=x
mem.SetVar(runtime.LhsName('a'), runtime.Str('x'),
(var_flags_e.ReadOnly, ), scope_e.Dynamic)
val = mem.GetVar('a', scope_e.Dynamic)
test_lib.AssertAsdlEqual(self, runtime.Str('x'), val)
val = mem.GetVar('undef', scope_e.Dynamic)
test_lib.AssertAsdlEqual(self, runtime.Undef(), val)
def SetStringDynamic(mem, name, s):
"""Set a string by looking up the stack.
Used for getopts.
"""
assert isinstance(s, str)
mem.SetVar(ast.LhsName(name), runtime.Str(s), (), scope_e.Dynamic)
def _ApplyPrefixOp(self, val, op_id):
"""
Returns:
value
"""
assert val.tag != value_e.Undef
if op_id == Id.VSub_Pound: # LENGTH
if val.tag == value_e.Str:
unicode_val = val.s.decode('utf-8')
length = len(unicode_val)
# length = len(val.s)
elif val.tag == value_e.StrArray:
# There can be empty placeholder values in the array.
length = sum(1 for s in val.strs if s is not None)
return runtime.Str(str(length))
elif op_id == Id.VSub_Bang:
# NOTES:
# - Could translate to eval('$' + name) or eval("\$$name")
# - ${!array[@]} means something completely different. TODO: implement
# that.
# - It might make sense to suggest implementing this with associative
# arrays?
# Treat the value of the variable as a variable name.
return self.mem.GetVar(val.s)
else:
raise AssertionError(op_id)
def EvalRhsWord(self, word):
"""word_t -> value_t.
Used for RHS of assignment. There is no splitting.
Args:
ast.word_t
Returns:
runtime.value_t
"""
if word.tag == word_e.EmptyWord:
return runtime.Str('')
# Special case for a=(1 2). ArrayLiteralPart won't appear in words that
# don't look like assignments.
if (len(word.parts) == 1
and word.parts[0].tag == word_part_e.ArrayLiteralPart):
array_words = word.parts[0].words
words = braces.BraceExpandWords(array_words)
strs = self._EvalWordSequence(words)
#log('ARRAY LITERAL EVALUATED TO -> %s', strs)
return runtime.StrArray(strs)
# If RHS doens't look like a=( ... ), then it must be a string.
return self.EvalWordToString(word)
def _ApplyUnarySuffixOp(self, val, op):
assert val.tag != value_e.Undef
op_kind = LookupKind(op.op_id)
if op_kind == Kind.VOp1:
#log('%s', op)
arg_val = self.EvalWordToString(op.arg_word, do_fnmatch=True)
assert arg_val.tag == value_e.Str
if val.tag == value_e.Str:
s = libstr.DoUnarySuffixOp(val.s, op, arg_val.s)
new_val = runtime.Str(s)
else: # val.tag == value_e.StrArray:
# ${a[@]#prefix} is VECTORIZED on arrays. Oil should have this too.
strs = []
for s in val.strs:
if s is not None:
strs.append(libstr.DoUnarySuffixOp(s, op, arg_val.s))
new_val = runtime.StrArray(strs)
else:
raise AssertionError(op_kind)
return new_val
def _EvalSpecialVar(self, op_id, quoted):
"""Returns (val, bool maybe_decay_array).
TODO: Should that boolean be part of the value?
"""
# $@ is special -- it need to know whether it is in a double quoted
# context.
#
# - If it's $@ in a double quoted context, return an ARRAY.
# - If it's $@ in a normal context, return a STRING, which then will be
# subject to splitting.
if op_id in (Id.VSub_At, Id.VSub_Star):
argv = self.mem.GetArgv()
val = runtime.StrArray(argv)
if op_id == Id.VSub_At:
# "$@" evaluates to an array, $@ should be decayed
return val, not quoted
else: # $@ $* "$*"
return val, True
elif op_id == Id.VSub_Hyphen:
s = self.exec_opts.GetDollarHyphen()
return runtime.Str(s), False
else:
val = self.mem.GetSpecialVar(op_id)
return val, False # don't decay
def _InitVarsFromEnv(self, environ):
# This is the way dash and bash work -- at startup, they turn everything in
# 'environ' variable into shell variables. Bash has an export_env
# variable. Dash has a loop through environ in init.c
for n, v in environ.iteritems():
self.SetVar(ast.LhsName(n), runtime.Str(v),
(var_flags_e.Exported, ), scope_e.GlobalOnly)
# If it's not in the environment, initialize it. This makes it easier to
# update later in ExecOpts.
# TODO: IFS, PWD, etc. should follow this pattern. Maybe need a SysCall
# interface? self.syscall.getcwd() etc.
v = self.GetVar('SHELLOPTS')
if v.tag == value_e.Undef:
SetGlobalString(self, 'SHELLOPTS', '')
# Now make it readonly
self.SetVar(ast.LhsName('SHELLOPTS'), None, (var_flags_e.ReadOnly, ),
scope_e.GlobalOnly)
v = self.GetVar('HOME')
if v.tag == value_e.Undef:
home_dir = util.GetHomeDir() or '~' # No expansion if not found?
SetGlobalString(self, 'HOME', home_dir)
def Export(argv, mem):
arg, i = EXPORT_SPEC.Parse(argv)
if arg.n:
for name in argv[i:]:
m = match.IsValidVarName(name)
if not m:
raise args.UsageError('export: Invalid variable name %r' %
name)
# NOTE: bash doesn't care if it wasn't found.
mem.ClearFlag(name, var_flags_e.Exported, scope_e.Dynamic)
else:
for arg in argv[i:]:
parts = arg.split('=', 1)
if len(parts) == 1:
name = parts[0]
val = None # Creates an empty variable
else:
name, s = parts
val = runtime.Str(s)
m = match.IsValidVarName(name)
if not m:
raise args.UsageError('export: Invalid variable name %r' %
name)
#log('%s %s', name, val)
mem.SetVar(runtime.LhsName(name), val, (var_flags_e.Exported, ),
scope_e.Dynamic)
return 0
def SetLocalString(mem, name, s):
"""Set a local string.
Used for:
1) for loop iteration variables
2) temporary environments like FOO=bar BAR=$FOO cmd,
3) read builtin
"""
assert isinstance(s, str)
mem.SetVar(ast.LhsName(name), runtime.Str(s), (), scope_e.LocalOnly)
def testExportThenAssign(self):
"""Regression Test"""
mem = _InitMem()
# export U
mem.SetVar(runtime.LhsName('U'), None, (var_flags_e.Exported, ),
scope_e.Dynamic)
print(mem)
# U=u
mem.SetVar(runtime.LhsName('U'), runtime.Str('u'), (), scope_e.Dynamic)
print(mem)
e = mem.GetExported()
self.assertEqual({'U': 'u'}, e)
def _EmptyStrOrError(self, val, token=None):
assert isinstance(val, runtime.value), val
if val.tag == value_e.Undef:
if self.exec_opts.nounset:
if token is None:
e_die('Undefined variable')
else:
name = token.val[1:] if token.val.startswith('$') else token.val
e_die('Undefined variable %r', name, token=token)
else:
return runtime.Str('')
else:
return val
def __init__(self, argv0, argv, environ, arena):
top = _StackFrame()
self.var_stack = [top]
self.argv0 = argv0
self.argv_stack = [_ArgFrame(argv)]
# NOTE: could use deque and appendleft/popleft, but:
# 1. ASDL type checking of StrArray doesn't allow it (could be fixed)
# 2. We don't otherwise depend on the collections module
self.func_name_stack = []
# Note: we're reusing these objects because they change on every single
# line! Don't want to allocate more than necsesary.
self.source_name = runtime.Str('')
self.line_num = runtime.Str('')
self.last_status = 0 # Mutable public variable
self.last_job_id = -1 # Uninitialized value mutable public variable
# Done ONCE on initialization
self.root_pid = os.getpid()
self._InitDefaults()
self._InitVarsFromEnv(environ)
self.arena = arena
def EvalWordToString(self, word, do_fnmatch=False, decay=False):
"""
Used for redirect arg, ControlFlow arg, ArithWord, BoolWord, etc.
do_fnmatch is true for case $pat and RHS of [[ == ]].
pat="*.py"
case $x in
$pat) echo 'matches glob pattern' ;;
"$pat") echo 'equal to glob string' ;; // must be glob escaped
esac
"""
part_vals = []
for part in word.parts:
self._EvalWordPart(part, part_vals, quoted=False)
strs = []
for part_val in part_vals:
# TODO: if decay, then allow string part. e.g. for here word or here
# doc with "$@".
if part_val.tag == part_value_e.StringPartValue:
# [[ foo == */"*".py ]] or case *.py) ... esac
if do_fnmatch and not part_val.do_split_glob:
s = glob_.GlobEscape(part_val.s)
else:
s = part_val.s
else:
if self.exec_opts.strict_array:
# Examples: echo f > "$@"; local foo="$@"
e_die("Expected string, got %s", part_val, word=word)
# TODO: Maybe add detail like this.
#e_die('RHS of assignment should only have strings. '
# 'To assign arrays, using b=( "${a[@]}" )')
else:
# It appears to not respect IFS
s = ' '.join(s for s in part_val.strs if s is not None)
strs.append(s)
return runtime.Str(''.join(strs))
def GetSpecialVar(self, op_id):
if op_id == Id.VSub_Bang: # $!
n = self.last_job_id
if n == -1:
return runtime.Undef() # could be an error
elif op_id == Id.VSub_QMark: # $?
# TODO: Have to parse status somewhere.
# External commands need WIFEXITED test. What about subshells?
n = self.last_status
elif op_id == Id.VSub_Pound: # $#
n = self.argv_stack[-1].GetNumArgs()
elif op_id == Id.VSub_Dollar: # $$
n = self.root_pid
else:
raise NotImplementedError(op_id)
return runtime.Str(str(n))
def _EvalBracedVarSub(self, part, part_vals, quoted):
"""
Args:
part_vals: output param to append to.
"""
# We have four types of operator that interact.
#
# 1. Bracket: value -> (value, bool maybe_decay_array)
#
# 2. Then these four cases are mutually exclusive:
#
# a. Prefix length: value -> value
# b. Test: value -> part_value[]
# c. Other Suffix: value -> value
# d. no operator: you have a value
#
# That is, we don't have both prefix and suffix operators.
#
# 3. Process maybe_decay_array here before returning.
maybe_decay_array = False # for $*, ${a[*]}, etc.
var_name = None # For ${foo=default}
# 1. Evaluate from (var_name, var_num, token Id) -> value
if part.token.id == Id.VSub_Name:
var_name = part.token.val
val = self.mem.GetVar(var_name)
#log('EVAL NAME %s -> %s', var_name, val)
elif part.token.id == Id.VSub_Number:
var_num = int(part.token.val)
val = self._EvalVarNum(var_num)
else:
# $* decays
val, maybe_decay_array = self._EvalSpecialVar(
part.token.id, quoted)
# 2. Bracket: value -> (value v, bool maybe_decay_array)
# maybe_decay_array is for joining ${a[*]} and unquoted ${a[@]} AFTER
# suffix ops are applied. If we take the length with a prefix op, the
# distinction is ignored.
if part.bracket_op:
if part.bracket_op.tag == bracket_op_e.WholeArray:
op_id = part.bracket_op.op_id
if op_id == Id.Lit_At:
if not quoted:
maybe_decay_array = True # ${a[@]} decays but "${a[@]}" doesn't
if val.tag == value_e.Undef:
val = self._EmptyStrArrayOrError(part.token)
elif val.tag == value_e.Str:
e_die("Can't index string with @: %r", val, part=part)
elif val.tag == value_e.StrArray:
# TODO: Is this a no-op? Just leave 'val' alone.
val = runtime.StrArray(val.strs)
elif op_id == Id.Arith_Star:
maybe_decay_array = True # both ${a[*]} and "${a[*]}" decay
if val.tag == value_e.Undef:
val = self._EmptyStrArrayOrError(part.token)
elif val.tag == value_e.Str:
e_die("Can't index string with *: %r", val, part=part)
elif val.tag == value_e.StrArray:
# TODO: Is this a no-op? Just leave 'val' alone.
# ${a[*]} or "${a[*]}" : maybe_decay_array is always true
val = runtime.StrArray(val.strs)
else:
raise AssertionError(op_id) # unknown
elif part.bracket_op.tag == bracket_op_e.ArrayIndex:
anode = part.bracket_op.expr
if val.tag == value_e.Undef:
pass # it will be checked later
elif val.tag == value_e.Str:
# Bash treats any string as an array, so we can't add our own
# behavior here without making valid OSH invalid bash.
e_die("Can't index string %r with integer",
part.token.val,
token=part.token)
elif val.tag == value_e.StrArray:
index = self.arith_ev.Eval(anode)
try:
# could be None because representation is sparse
s = val.strs[index]
except IndexError:
s = None
if s is None:
val = runtime.Undef()
else:
val = runtime.Str(s)
elif val.tag == value_e.AssocArray:
key = self.arith_ev.Eval(anode, int_coerce=False)
try:
val = runtime.Str(val.d[key])
except KeyError:
val = runtime.Undef()
else:
raise AssertionError(val.__class__.__name__)
else:
raise AssertionError(part.bracket_op.tag)
if part.prefix_op:
val = self._EmptyStrOrError(val) # maybe error
val = self._ApplyPrefixOp(val, part.prefix_op)
# NOTE: When applying the length operator, we can't have a test or
# suffix afterward. And we don't want to decay the array
elif part.suffix_op:
op = part.suffix_op
if op.tag == suffix_op_e.StringNullary:
if op.op_id == Id.VOp0_P:
# TODO: Use dependency injection
#val = self.prompt._EvalPS1(val)
prompt = ui.PROMPT.EvalPrompt(val)
val = runtime.Str(prompt)
else:
raise NotImplementedError(op.op_id)
elif op.tag == suffix_op_e.StringUnary:
if LookupKind(part.suffix_op.op_id) == Kind.VTest:
# TODO: Change style to:
# if self._ApplyTestOp(...)
# return
# It should return whether anything was done. If not, we continue to
# the end, where we might throw an error.
assign_part_vals, effect = self._ApplyTestOp(
val, part.suffix_op, quoted, part_vals)
# NOTE: Splicing part_values is necessary because of code like
# ${undef:-'a b' c 'd # e'}. Each part_value can have a different
# do_glob/do_elide setting.
if effect == effect_e.SpliceParts:
return # EARLY RETURN, part_vals mutated
elif effect == effect_e.SpliceAndAssign:
if var_name is None:
# TODO: error context
e_die("Can't assign to special variable")
else:
# NOTE: This decays arrays too! 'set -o strict_array' could
# avoid it.
rhs_str = _DecayPartValuesToString(
assign_part_vals, self.splitter.GetJoinChar())
state.SetLocalString(self.mem, var_name, rhs_str)
return # EARLY RETURN, part_vals mutated
elif effect == effect_e.Error:
raise NotImplementedError
else:
# The old one
#val = self._EmptyStringPartOrError(part_val, quoted)
pass # do nothing, may still be undefined
else:
val = self._EmptyStrOrError(val) # maybe error
# Other suffix: value -> value
val = self._ApplyUnarySuffixOp(val, part.suffix_op)
elif op.tag == suffix_op_e.PatSub: # PatSub, vectorized
val = self._EmptyStrOrError(val) # ${undef//x/y}
pat_val = self.EvalWordToString(op.pat, do_fnmatch=True)
assert pat_val.tag == value_e.Str, pat_val
if op.replace:
replace_val = self.EvalWordToString(op.replace,
do_fnmatch=True)
assert replace_val.tag == value_e.Str, replace_val
replace_str = replace_val.s
else:
replace_str = ''
# Either GlobReplacer or ConstStringReplacer
replacer = libstr.MakeReplacer(pat_val.s, replace_str,
op.spids[0])
if val.tag == value_e.Str:
s = replacer.Replace(val.s, op)
val = runtime.Str(s)
elif val.tag == value_e.StrArray:
strs = []
for s in val.strs:
if s is not None:
strs.append(replacer.Replace(s, op))
val = runtime.StrArray(strs)
else:
raise AssertionError(val.__class__.__name__)
elif op.tag == suffix_op_e.Slice:
val = self._EmptyStrOrError(val) # ${undef:3:1}
if op.begin:
begin = self.arith_ev.Eval(op.begin)
else:
begin = 0
if op.length:
length = self.arith_ev.Eval(op.length)
else:
length = None
if val.tag == value_e.Str: # Slice UTF-8 characters in a string.
s = val.s
try:
if begin < 0:
# It could be negative if we compute unicode length, but that's
# confusing.
# TODO: Instead of attributing it to the word part, it would be
# better if we attributed it to arith_expr begin.
raise util.InvalidSlice(
"The start index of a string slice can't be negative: %d",
begin,
part=part)
byte_begin = libstr.AdvanceUtf8Chars(s, begin, 0)
if length is None:
byte_end = len(s)
else:
if length < 0:
# TODO: Instead of attributing it to the word part, it would be
# better if we attributed it to arith_expr begin.
raise util.InvalidSlice(
"The length of a string slice can't be negative: %d",
length,
part=part)
byte_end = libstr.AdvanceUtf8Chars(
s, length, byte_begin)
except (util.InvalidSlice, util.InvalidUtf8) as e:
if self.exec_opts.strict_word_eval:
raise
else:
# TODO:
# - We don't see the error location here, but we see it when set
# -o strict-word-eval.
# - Doesn't make the command exit with 1. It just sets the word
# to empty string.
util.warn(e.UserErrorString())
substr = '' # error condition
else:
substr = s[byte_begin:byte_end]
val = runtime.Str(substr)
elif val.tag == value_e.StrArray: # Slice array entries.
# NOTE: unset elements don't count towards the length.
strs = []
for s in val.strs[begin:]:
if s is not None:
strs.append(s)
if len(
strs
) == length: # never true for unspecified length
break
val = runtime.StrArray(strs)
else:
raise AssertionError(
val.__class__.__name__) # Not possible
# After applying suffixes, process maybe_decay_array here.
if maybe_decay_array and val.tag == value_e.StrArray:
val = self._DecayArray(val)
# For the case where there are no prefix or suffix ops.
val = self._EmptyStrOrError(val)
# For example, ${a} evaluates to value_t.Str(), but we want a
# part_value.StringPartValue.
part_val = _ValueToPartValue(val, quoted)
part_vals.append(part_val)
def _DecayArray(self, val):
assert val.tag == value_e.StrArray, val
sep = self.splitter.GetJoinChar()
return runtime.Str(sep.join(s for s in val.strs if s is not None))
def _ApplyPrefixOp(self, val, op_id):
"""
Returns:
value
"""
assert val.tag != value_e.Undef
if op_id == Id.VSub_Pound: # LENGTH
if val.tag == value_e.Str:
# NOTE: Whether bash counts bytes or chars is affected by LANG
# environment variables.
# Should we respect that, or another way to select? set -o
# count-bytes?
# https://stackoverflow.com/questions/17368067/length-of-string-in-bash
try:
length = libstr.CountUtf8Chars(val.s)
except util.InvalidUtf8 as e:
# TODO: Add location info from 'part'? Only the caller has it.
if self.exec_opts.strict_word_eval:
raise
else:
# NOTE: Doesn't make the command exit with 1; it just returns a
# length of -1.
util.warn(e.UserErrorString())
return runtime.Str('-1')
elif val.tag == value_e.StrArray:
# There can be empty placeholder values in the array.
length = sum(1 for s in val.strs if s is not None)
return runtime.Str(str(length))
elif op_id == Id.VSub_Bang:
# NOTES:
# - Could translate to eval('$' + name) or eval("\$$name")
# - ${!array[@]} means something completely different. TODO: implement
# that.
# - It might make sense to suggest implementing this with associative
# arrays?
# Treat the value of the variable as a variable name.
if val.tag == value_e.Str:
try:
# e.g. ${!OPTIND} gives $1 when OPTIND is 1
arg_num = int(val.s)
return self.mem.GetArgNum(arg_num)
except ValueError:
if not match.IsValidVarName(val.s):
# TODO: location information.
# Also note that bash doesn't consider this fatal. It makes the
# command exit with '1', but we don't have that ability yet?
e_die('Bad variable name %r in var ref', val.s)
return self.mem.GetVar(val.s)
elif val.tag == value_e.StrArray:
raise NotImplementedError(
'${!a[@]}') # bash gets keys this way
else:
raise AssertionError
else:
raise AssertionError(op_id)
def SetGlobalString(mem, name, s):
"""Helper for completion, $PWD, etc."""
assert isinstance(s, str)
val = runtime.Str(s)
mem.SetVar(ast.LhsName(name), val, (), scope_e.GlobalOnly)
def EvalWordToString(self, w, do_fnmatch=False, do_ere=False):
# do_fnmatch: for the [[ == ]] semantics which we don't have!
# I think I need another type of node
# Maybe it should be BuiltinEqual and BuiltinDEqual? Parse it into a
# different tree.
return runtime.Str(w.s)
def GetArgNum(self, arg_num):
index = self.num_shifted + arg_num - 1
if index >= len(self.argv):
return runtime.Undef()
return runtime.Str(str(self.argv[index]))
def _Store(self, lval, new_int): val = runtime.Str(str(new_int)) self.mem.SetVar(lval, val, (), scope_e.Dynamic)
def testSetVarClearFlag(self):
mem = _InitMem()
print(mem)
mem.PushCall('my-func', ['ONE'])
self.assertEqual(2, len(mem.var_stack)) # internal details
# local x=y
mem.SetVar(runtime.LhsName('x'), runtime.Str('y'), (),
scope_e.LocalOnly)
self.assertEqual('y', mem.var_stack[-1].vars['x'].val.s)
# New frame
mem.PushCall('my-func', ['TWO'])
self.assertEqual(3, len(mem.var_stack)) # internal details
# x=y -- test out dynamic scope
mem.SetVar(runtime.LhsName('x'), runtime.Str('YYY'), (),
scope_e.Dynamic)
self.assertEqual('YYY', mem.var_stack[-2].vars['x'].val.s)
self.assertEqual(None, mem.var_stack[-1].vars.get('x'))
# myglobal=g
mem.SetVar(runtime.LhsName('myglobal'), runtime.Str('g'), (),
scope_e.Dynamic)
self.assertEqual('g', mem.var_stack[0].vars['myglobal'].val.s)
self.assertEqual(False, mem.var_stack[0].vars['myglobal'].exported)
# 'export PYTHONPATH=/'
mem.SetVar(runtime.LhsName('PYTHONPATH'), runtime.Str('/'),
(var_flags_e.Exported, ), scope_e.Dynamic)
self.assertEqual('/', mem.var_stack[0].vars['PYTHONPATH'].val.s)
self.assertEqual(True, mem.var_stack[0].vars['PYTHONPATH'].exported)
self.assertEqual({'PYTHONPATH': '/'}, mem.GetExported())
mem.SetVar(runtime.LhsName('PYTHONPATH'), None,
(var_flags_e.Exported, ), scope_e.Dynamic)
self.assertEqual(True, mem.var_stack[0].vars['PYTHONPATH'].exported)
# 'export myglobal'. None means don't touch it. Undef would be confusing
# because it might mean "unset", but we have a separated API for that.
mem.SetVar(runtime.LhsName('myglobal'), None, (var_flags_e.Exported, ),
scope_e.Dynamic)
self.assertEqual(True, mem.var_stack[0].vars['myglobal'].exported)
# export g2 -- define and export empty
mem.SetVar(runtime.LhsName('g2'), None, (var_flags_e.Exported, ),
scope_e.Dynamic)
self.assertEqual(value_e.Undef, mem.var_stack[0].vars['g2'].val.tag)
self.assertEqual(True, mem.var_stack[0].vars['g2'].exported)
# readonly myglobal
self.assertEqual(False, mem.var_stack[0].vars['myglobal'].readonly)
mem.SetVar(runtime.LhsName('myglobal'), None, (var_flags_e.ReadOnly, ),
scope_e.Dynamic)
self.assertEqual(True, mem.var_stack[0].vars['myglobal'].readonly)
mem.SetVar(runtime.LhsName('PYTHONPATH'), runtime.Str('/lib'), (),
scope_e.Dynamic)
self.assertEqual('/lib', mem.var_stack[0].vars['PYTHONPATH'].val.s)
self.assertEqual(True, mem.var_stack[0].vars['PYTHONPATH'].exported)
# COMPREPLY=(1 2 3)
# invariant to enforce: arrays can't be exported
mem.SetVar(runtime.LhsName('COMPREPLY'),
runtime.StrArray(['1', '2', '3']), (), scope_e.GlobalOnly)
self.assertEqual(['1', '2', '3'],
mem.var_stack[0].vars['COMPREPLY'].val.strs)
# export COMPREPLY
try:
mem.SetVar(runtime.LhsName('COMPREPLY'), None,
(var_flags_e.Exported, ), scope_e.Dynamic)
except util.FatalRuntimeError as e:
pass
else:
self.fail("Expected failure")
# readonly r=1
mem.SetVar(runtime.LhsName('r'), runtime.Str('1'),
(var_flags_e.ReadOnly, ), scope_e.Dynamic)
self.assertEqual('1', mem.var_stack[0].vars['r'].val.s)
self.assertEqual(False, mem.var_stack[0].vars['r'].exported)
self.assertEqual(True, mem.var_stack[0].vars['r'].readonly)
print(mem)
# r=newvalue
try:
mem.SetVar(runtime.LhsName('r'), runtime.Str('newvalue'), (),
scope_e.Dynamic)
except util.FatalRuntimeError as e:
pass
else:
self.fail("Expected failure")
# readonly r2 -- define empty readonly
mem.SetVar(runtime.LhsName('r2'), None, (var_flags_e.ReadOnly, ),
scope_e.Dynamic)
self.assertEqual(value_e.Undef, mem.var_stack[0].vars['r2'].val.tag)
self.assertEqual(True, mem.var_stack[0].vars['r2'].readonly)
# export -n PYTHONPATH
# Remove the exported property. NOTE: scope is LocalOnly for Oil?
self.assertEqual(True, mem.var_stack[0].vars['PYTHONPATH'].exported)
mem.ClearFlag('PYTHONPATH', var_flags_e.Exported, scope_e.Dynamic)
self.assertEqual(False, mem.var_stack[0].vars['PYTHONPATH'].exported)
# a[1]=2
mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('2'), (),
scope_e.Dynamic)
self.assertEqual([None, '2'], mem.var_stack[0].vars['a'].val.strs)
# a[1]=3
mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('3'), (),
scope_e.Dynamic)
self.assertEqual([None, '3'], mem.var_stack[0].vars['a'].val.strs)
# a[1]=(x y z) # illegal
try:
mem.SetVar(runtime.LhsIndexedName('a', 1),
runtime.StrArray(['x', 'y', 'z']), (), scope_e.Dynamic)
except util.FatalRuntimeError as e:
pass
else:
self.fail("Expected failure")
# readonly a
mem.SetVar(runtime.LhsName('a'), None, (var_flags_e.ReadOnly, ),
scope_e.Dynamic)
self.assertEqual(True, mem.var_stack[0].vars['a'].readonly)
try:
# a[2]=3
mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('3'), (),
scope_e.Dynamic)
except util.FatalRuntimeError as e:
pass
else:
self.fail("Expected failure")
def _Dispatch(self, node, fork_external):
# If we call RunCommandSub in a recursive call to the executor, this will
# be set true (if strict-errexit is false). But it only lasts for one
# command.
self.check_command_sub_status = False
#argv0 = None # for error message
check_errexit = False # for errexit
if node.tag == command_e.SimpleCommand:
check_errexit = True
# Find span_id for a basic implementation of $LINENO, e.g.
# PS4='+$SOURCE_NAME:$LINENO:'
# NOTE: osh2oil uses node.more_env, but we don't need that.
span_id = const.NO_INTEGER
if node.words:
first_word = node.words[0]
span_id = word.LeftMostSpanForWord(first_word)
self.mem.SetCurrentSpanId(span_id)
# PROBLEM: We want to log argv in 'xtrace' mode, but we may have already
# redirected here, which screws up logging. For example, 'echo hi
# >/dev/null 2>&1'. We want to evaluate argv and log it BEFORE applying
# redirects.
# Another problem:
# - tracing can be called concurrently from multiple processes, leading
# to overlap. Maybe have a mode that creates a file per process.
# xtrace-proc
# - line numbers for every command would be very nice. But then you have
# to print the filename too.
words = braces.BraceExpandWords(node.words)
argv = self.word_ev.EvalWordSequence(words)
# This comes before evaluating env, in case there are problems evaluating
# it. We could trace the env separately? Also trace unevaluated code
# with set-o verbose?
self.tracer.OnSimpleCommand(argv)
if node.more_env:
self.mem.PushTemp()
try:
for env_pair in node.more_env:
val = self.word_ev.EvalWordToString(env_pair.val)
# Set each var so the next one can reference it. Example:
# FOO=1 BAR=$FOO ls /
self.mem.SetVar(ast.LhsName(env_pair.name), val,
(var_flags_e.Exported,), scope_e.TempEnv)
# NOTE: This might never return! In the case of fork_external=False.
status = self._RunSimpleCommand(argv, fork_external, span_id)
finally:
if node.more_env:
self.mem.PopTemp()
elif node.tag == command_e.Sentence:
# Don't check_errexit since this isn't a real node!
if node.terminator.id == Id.Op_Semi:
status = self._Execute(node.child)
else:
status = self._RunJobInBackground(node.child)
elif node.tag == command_e.Pipeline:
check_errexit = True
if node.stderr_indices:
raise NotImplementedError('|&')
if node.negated:
self._PushErrExit()
try:
status2 = self._RunPipeline(node)
finally:
self._PopErrExit()
# errexit is disabled for !.
check_errexit = False
status = 1 if status2 == 0 else 0
else:
status = self._RunPipeline(node)
elif node.tag == command_e.Subshell:
check_errexit = True
# This makes sure we don't waste a process if we'd launch one anyway.
p = self._MakeProcess(node.child)
status = p.Run(self.waiter)
elif node.tag == command_e.DBracket:
check_errexit = True
result = self.bool_ev.Eval(node.expr)
status = 0 if result else 1
elif node.tag == command_e.DParen:
check_errexit = True
i = self.arith_ev.Eval(node.child)
status = 0 if i != 0 else 1
elif node.tag == command_e.Assignment:
flags = word_compile.ParseAssignFlags(node.flags)
if node.keyword == Id.Assign_Local:
lookup_mode = scope_e.LocalOnly
# typeset and declare are synonyms? I see typeset -a a=() the most.
elif node.keyword in (Id.Assign_Declare, Id.Assign_Typeset):
# declare is like local, except it can also be used outside functions?
if var_flags_e.Global in flags:
lookup_mode = scope_e.GlobalOnly
else:
lookup_mode = scope_e.LocalOnly
elif node.keyword == Id.Assign_Readonly:
lookup_mode = scope_e.Dynamic
flags.append(var_flags_e.ReadOnly)
elif node.keyword == Id.Assign_None: # mutate existing local or global
lookup_mode = scope_e.Dynamic
else:
raise AssertionError(node.keyword)
for pair in node.pairs:
if pair.op == assign_op_e.PlusEqual:
assert pair.rhs, pair.rhs # I don't think a+= is valid?
val = self.word_ev.EvalRhsWord(pair.rhs)
old_val, lval = expr_eval.EvalLhsAndLookup(pair.lhs, self.arith_ev,
self.mem, self.exec_opts)
sig = (old_val.tag, val.tag)
if sig == (value_e.Undef, value_e.Str):
pass # val is RHS
elif sig == (value_e.Undef, value_e.StrArray):
pass # val is RHS
elif sig == (value_e.Str, value_e.Str):
val = runtime.Str(old_val.s + val.s)
elif sig == (value_e.Str, value_e.StrArray):
e_die("Can't append array to string")
elif sig == (value_e.StrArray, value_e.Str):
e_die("Can't append string to array")
elif sig == (value_e.StrArray, value_e.StrArray):
val = runtime.StrArray(old_val.strs + val.strs)
else: # plain assignment
spid = pair.spids[0] # Source location for tracing
lval = self._EvalLhs(pair.lhs, spid, lookup_mode)
# RHS can be a string or array.
if pair.rhs:
val = self.word_ev.EvalRhsWord(pair.rhs)
assert isinstance(val, runtime.value), val
else: # e.g. 'readonly x' or 'local x'
val = None
# NOTE: In bash and mksh, declare -a myarray makes an empty cell with
# Undef value, but the 'array' attribute.
#log('setting %s to %s with flags %s', lval, val, flags)
self.mem.SetVar(lval, val, flags, lookup_mode,
strict_array=self.exec_opts.strict_array)
# Assignment always appears to have a spid.
if node.spids:
current_spid = node.spids[0]
else:
current_spid = const.NO_INTEGER
self.mem.SetCurrentSpanId(current_spid)
self.tracer.OnAssignment(lval, pair.op, val, flags, lookup_mode)
# PATCH to be compatible with existing shells: If the assignment had a
# command sub like:
#
# s=$(echo one; false)
#
# then its status will be in mem.last_status, and we can check it here.
# If there was NOT a command sub in the assignment, then we don't want to
# check it.
if node.keyword == Id.Assign_None: # mutate existing local or global
# Only do this if there was a command sub? How? Look at node?
# Set a flag in mem? self.mem.last_status or
if self.check_command_sub_status:
self._CheckStatus(self.mem.last_status, node)
# A global assignment shouldn't clear $?.
status = self.mem.last_status
else:
status = 0
else:
# To be compatible with existing shells, local assignments DO clear
# $?. Even in strict mode, we don't need to bother setting
# check_errexit = True, because we would have already checked the
# command sub in RunCommandSub.
status = 0
# TODO: maybe we should have a "sane-status" that respects this:
# false; echo $?; local f=x; echo $?
elif node.tag == command_e.ControlFlow:
if node.arg_word: # Evaluate the argument
val = self.word_ev.EvalWordToString(node.arg_word)
assert val.tag == value_e.Str
arg = int(val.s) # They all take integers
else:
arg = 0 # return 0, exit 0, break 0 levels, etc.
# NOTE: We don't do anything about a top-level 'return' here. Unlike in
# bash, that is OK. If you can return from a sourced script, it makes
# sense to return from a main script.
ok = True
tok = node.token
if (tok.id in (Id.ControlFlow_Break, Id.ControlFlow_Continue) and
self.loop_level == 0):
ok = False
msg = 'Invalid control flow at top level'
if ok:
raise _ControlFlow(tok, arg)
if self.exec_opts.strict_control_flow:
e_die(msg, token=tok)
else:
# Only print warnings, never fatal.
# Bash oddly only exits 1 for 'return', but no other shell does.
ui.PrintFilenameAndLine(tok.span_id, self.arena)
util.warn(msg)
status = 0
# The only difference between these two is that CommandList has no
# redirects. We already took care of that above.
elif node.tag in (command_e.CommandList, command_e.BraceGroup):
status = self._ExecuteList(node.children)
check_errexit = False
elif node.tag == command_e.AndOr:
# NOTE: && and || have EQUAL precedence in command mode. See case #13
# in dbracket.test.sh.
left = node.children[0]
# Suppress failure for every child except the last one.
self._PushErrExit()
try:
status = self._Execute(left)
finally:
self._PopErrExit()
i = 1
n = len(node.children)
while i < n:
#log('i %d status %d', i, status)
child = node.children[i]
op_id = node.ops[i-1]
#log('child %s op_id %s', child, op_id)
if op_id == Id.Op_DPipe and status == 0:
i += 1
continue # short circuit
elif op_id == Id.Op_DAmp and status != 0:
i += 1
continue # short circuit
if i == n - 1: # errexit handled differently for last child
status = self._Execute(child)
check_errexit = True
else:
self._PushErrExit()
try:
status = self._Execute(child)
finally:
self._PopErrExit()
i += 1
elif node.tag == command_e.WhileUntil:
if node.keyword.id == Id.KW_While:
_DonePredicate = lambda status: status != 0
else:
_DonePredicate = lambda status: status == 0
status = 0
self.loop_level += 1
try:
while True:
self._PushErrExit()
try:
cond_status = self._ExecuteList(node.cond)
finally:
self._PopErrExit()
done = cond_status != 0
if _DonePredicate(cond_status):
break
try:
status = self._Execute(node.body) # last one wins
except _ControlFlow as e:
if e.IsBreak():
status = 0
break
elif e.IsContinue():
status = 0
continue
else: # return needs to pop up more
raise
finally:
self.loop_level -= 1
elif node.tag == command_e.ForEach:
iter_name = node.iter_name
if node.do_arg_iter:
iter_list = self.mem.GetArgv()
else:
words = braces.BraceExpandWords(node.iter_words)
iter_list = self.word_ev.EvalWordSequence(words)
# We need word splitting and so forth
# NOTE: This expands globs too. TODO: We should pass in a Globber()
# object.
status = 0 # in case we don't loop
self.loop_level += 1
try:
for x in iter_list:
#log('> ForEach setting %r', x)
state.SetLocalString(self.mem, iter_name, x)
#log('<')
try:
status = self._Execute(node.body) # last one wins
except _ControlFlow as e:
if e.IsBreak():
status = 0
break
elif e.IsContinue():
status = 0
else: # return needs to pop up more
raise
finally:
self.loop_level -= 1
elif node.tag == command_e.ForExpr:
status = 0
init, cond, body, update = node.init, node.cond, node.body, node.update
if init:
self.arith_ev.Eval(init)
self.loop_level += 1
try:
while True:
if cond:
b = self.arith_ev.Eval(cond)
if not b:
break
try:
status = self._Execute(body)
except _ControlFlow as e:
if e.IsBreak():
status = 0
break
elif e.IsContinue():
status = 0
else: # return needs to pop up more
raise
if update:
self.arith_ev.Eval(update)
finally:
self.loop_level -= 1
elif node.tag == command_e.DoGroup:
status = self._ExecuteList(node.children)
check_errexit = False # not real statements
elif node.tag == command_e.FuncDef:
# NOTE: Would it make sense to evaluate the redirects BEFORE entering?
# It will save time on function calls.
self.funcs[node.name] = node
status = 0
elif node.tag == command_e.If:
done = False
for arm in node.arms:
self._PushErrExit()
try:
status = self._ExecuteList(arm.cond)
finally:
self._PopErrExit()
if status == 0:
status = self._ExecuteList(arm.action)
done = True
break
# TODO: The compiler should flatten this
if not done and node.else_action is not None:
status = self._ExecuteList(node.else_action)
elif node.tag == command_e.NoOp:
status = 0 # make it true
elif node.tag == command_e.Case:
val = self.word_ev.EvalWordToString(node.to_match)
to_match = val.s
status = 0 # If there are no arms, it should be zero?
done = False
for arm in node.arms:
for pat_word in arm.pat_list:
# NOTE: Is it OK that we're evaluating these as we go?
# TODO: case "$@") shouldn't succeed? That's a type error?
# That requires strict-array?
pat_val = self.word_ev.EvalWordToString(pat_word, do_fnmatch=True)
#log('Matching word %r against pattern %r', to_match, pat_val.s)
if libc.fnmatch(pat_val.s, to_match):
status = self._ExecuteList(arm.action)
done = True # TODO: Parse ;;& and for fallthrough and such?
break # Only execute action ONCE
if done:
break
elif node.tag == command_e.TimeBlock:
# TODO:
# - When do we need RUSAGE_CHILDREN?
# - Respect TIMEFORMAT environment variable.
# "If this variable is not set, Bash acts as if it had the value"
# $'\nreal\t%3lR\nuser\t%3lU\nsys\t%3lS'
# "A trailing newline is added when the format string is displayed."
start_t = time.time() # calls gettimeofday() under the hood
start_u = resource.getrusage(resource.RUSAGE_SELF)
status = self._Execute(node.pipeline)
end_t = time.time()
end_u = resource.getrusage(resource.RUSAGE_SELF)
real = end_t - start_t
user = end_u.ru_utime - start_u.ru_utime
sys_ = end_u.ru_stime - start_u.ru_stime
libc.print_time(real, user, sys_)
else:
raise NotImplementedError(node.__class__.__name__)
return status, check_errexit
def GetArgNum(self, arg_num):
if arg_num == 0:
return runtime.Str(self.argv0)
return self.argv_stack[-1].GetArgNum(arg_num)
