def EvalLhs(node, arith_ev, mem, exec_opts): """Evaluate the operand for a++ a[0]++ as an R-value. Used by Executor as well. Args: node: osh_ast.lhs_expr Returns: runtime.value, runtime.lvalue """ #log('lhs_expr NODE %s', node) assert isinstance(node, ast.lhs_expr), node if node.tag == lhs_expr_e.LhsName: # a = b # Problem: It can't be an array? # a=(1 2) # (( a++ )) lval = runtime.LhsName(node.name) val = _LookupVar(node.name, mem, exec_opts) elif node.tag == lhs_expr_e.LhsIndexedName: # a[1] = b # See tdop.IsIndexable for valid values: # - ArithVarRef (not LhsName): a[1] # - FuncCall: f(x), 1 # - ArithBinary LBracket: f[1][1] -- no semantics for this? index = arith_ev.Eval(node.index) lval = runtime.LhsIndexedName(node.name, index) val = mem.GetVar(node.name) if val.tag == value_e.Str: e_die("Can't assign to characters of string %r", node.name) elif val.tag == value_e.Undef: # It would make more sense for 'nounset' to control this, but bash # doesn't work that way. #if self.exec_opts.strict_arith: # e_die('Undefined array %r', node.name) # TODO: error location val = runtime.Str('') elif val.tag == value_e.StrArray: #log('ARRAY %s -> %s, index %d', node.name, array, index) array = val.strs # NOTE: Similar logic in RHS Arith_LBracket try: item = array[index] except IndexError: item = None if item is None: val = runtime.Str('') else: assert isinstance(item, str), item val = runtime.Str(item) else: raise AssertionError(val.tag) else: raise AssertionError(node.tag) return val, lval
def SetOption(self, opt_name, b): """ For set -o, set +o, or shopt -s/-u -o. """ self._SetOption(opt_name, b) val = self.mem.GetVar('SHELLOPTS') assert val.tag == value_e.Str shellopts = val.s # Now check if SHELLOPTS needs to be updated. It may be exported. # # NOTE: It might be better to skip rewriting SEHLLOPTS in the common case # where it is not used. We could do it lazily upon GET. # Also, it would be slightly more efficient to update SHELLOPTS if # settings were batched, Examples: # - set -eu # - shopt -s foo bar if b: if opt_name not in shellopts: new_val = runtime.Str('%s:%s' % (shellopts, opt_name)) self.mem.InternalSetGlobal('SHELLOPTS', new_val) else: if opt_name in shellopts: names = [n for n in shellopts.split(':') if n != opt_name] new_val = runtime.Str(':'.join(names)) self.mem.InternalSetGlobal('SHELLOPTS', new_val)
def __init__(self, dollar0, argv, environ, arena, has_main=False): self.dollar0 = dollar0 self.argv_stack = [_ArgFrame(argv)] self.var_stack = [_StackFrame()] # The debug_stack isn't strictly necessary for execution. We use it for # crash dumps and for 3 parallel arrays: FUNCNAME, BASH_SOURCE, # BASH_LINENO. The First frame points at the global vars and argv. self.debug_stack = [(None, None, const.NO_INTEGER, 0, 0)] self.bash_source = [] # for implementing BASH_SOURCE self.has_main = has_main if has_main: self.bash_source.append(dollar0) # e.g. the filename self.current_spid = const.NO_INTEGER # Note: we're reusing these objects because they change on every single # line! Don't want to allocate more than necsesary. self.source_name = runtime.Str('') self.line_num = runtime.Str('') self.last_status = 0 # Mutable public variable self.last_job_id = -1 # Uninitialized value mutable public variable # Done ONCE on initialization self.root_pid = os.getpid() self._InitDefaults() self._InitVarsFromEnv(environ) self.arena = arena
def testPushTemp(self): mem = _InitMem() # x=1 mem.SetVar(runtime.LhsName('x'), runtime.Str('1'), (), scope_e.Dynamic) self.assertEqual('1', mem.var_stack[-1].vars['x'].val.s) mem.PushTemp() self.assertEqual(2, len(mem.var_stack)) # Temporary frame is readonly self.assertEqual(True, mem.var_stack[-1].readonly) self.assertEqual(False, mem.var_stack[-2].readonly) # x=temp E=3 read x <<< 'line' mem.SetVar(runtime.LhsName('x'), runtime.Str('temp'), (), scope_e.TempEnv) mem.SetVar(runtime.LhsName('E'), runtime.Str('3'), (), scope_e.TempEnv) mem.SetVar(runtime.LhsName('x'), runtime.Str('line'), (), scope_e.LocalOnly) self.assertEqual('3', mem.var_stack[-1].vars['E'].val.s) self.assertEqual('temp', mem.var_stack[-1].vars['x'].val.s) self.assertEqual('line', mem.var_stack[-2].vars['x'].val.s) mem.PopTemp() self.assertEqual(1, len(mem.var_stack)) self.assertEqual('line', mem.var_stack[-1].vars['x'].val.s)
def EvalWordToString(self, word, do_fnmatch=False, do_ere=False): """ Args: word: CompoundWord Used for redirect arg, ControlFlow arg, ArithWord, BoolWord, etc. do_fnmatch is true for case $pat and RHS of [[ == ]]. pat="*.py" case $x in $pat) echo 'matches glob pattern' ;; "$pat") echo 'equal to glob string' ;; # must be glob escaped esac TODO: Raise AssertionError if it has ExtGlobPart. """ if word.tag == word_e.EmptyWord: return runtime.Str('') part_vals = [] for p in word.parts: self._EvalWordPart(p, part_vals, quoted=False) strs = [] for part_val in part_vals: if part_val.tag == part_value_e.StringPartValue: # [[ foo == */"*".py ]] or case *.py) ... esac if do_fnmatch and not part_val.do_split_glob: s = glob_.GlobEscape(part_val.s) elif do_ere and not part_val.do_split_glob: s = glob_.ExtendedRegexEscape(part_val.s) else: s = part_val.s else: if self.exec_opts.strict_array: # Examples: echo f > "$@"; local foo="$@" # TODO: This attributes too coarsely, to the word rather than the # parts. Problem: the word is a TREE of parts, but we only have a # flat list of part_vals. The only case where we really get arrays # is "$@", "${a[@]}", "${a[@]//pat/replace}", etc. e_die( "This word should evaluate to a string, but part of it was an " "array", word=word) # TODO: Maybe add detail like this. #e_die('RHS of assignment should only have strings. ' # 'To assign arrays, use b=( "${a[@]}" )') else: # It appears to not respect IFS s = ' '.join(s for s in part_val.strs if s is not None) strs.append(s) return runtime.Str(''.join(strs))
def testPrompt(self): arena = test_lib.MakeArena('<ui_test.py>') ex = test_lib.InitExecutor(arena=arena) p = ui.Prompt(arena, ex.parse_ctx, ex) # Rgression for caching bug! self.assertEqual('foo', p.EvalPrompt(runtime.Str('foo'))) self.assertEqual('foo', p.EvalPrompt(runtime.Str('foo')))
def testGetVar(self): mem = _InitMem() # readonly a=x mem.SetVar(runtime.LhsName('a'), runtime.Str('x'), (var_flags_e.ReadOnly, ), scope_e.Dynamic) val = mem.GetVar('a', scope_e.Dynamic) test_lib.AssertAsdlEqual(self, runtime.Str('x'), val) val = mem.GetVar('undef', scope_e.Dynamic) test_lib.AssertAsdlEqual(self, runtime.Undef(), val)
def SetStringDynamic(mem, name, s): """Set a string by looking up the stack. Used for getopts. """ assert isinstance(s, str) mem.SetVar(ast.LhsName(name), runtime.Str(s), (), scope_e.Dynamic)
def _ApplyPrefixOp(self, val, op_id): """ Returns: value """ assert val.tag != value_e.Undef if op_id == Id.VSub_Pound: # LENGTH if val.tag == value_e.Str: unicode_val = val.s.decode('utf-8') length = len(unicode_val) # length = len(val.s) elif val.tag == value_e.StrArray: # There can be empty placeholder values in the array. length = sum(1 for s in val.strs if s is not None) return runtime.Str(str(length)) elif op_id == Id.VSub_Bang: # NOTES: # - Could translate to eval('$' + name) or eval("\$$name") # - ${!array[@]} means something completely different. TODO: implement # that. # - It might make sense to suggest implementing this with associative # arrays? # Treat the value of the variable as a variable name. return self.mem.GetVar(val.s) else: raise AssertionError(op_id)
def EvalRhsWord(self, word): """word_t -> value_t. Used for RHS of assignment. There is no splitting. Args: ast.word_t Returns: runtime.value_t """ if word.tag == word_e.EmptyWord: return runtime.Str('') # Special case for a=(1 2). ArrayLiteralPart won't appear in words that # don't look like assignments. if (len(word.parts) == 1 and word.parts[0].tag == word_part_e.ArrayLiteralPart): array_words = word.parts[0].words words = braces.BraceExpandWords(array_words) strs = self._EvalWordSequence(words) #log('ARRAY LITERAL EVALUATED TO -> %s', strs) return runtime.StrArray(strs) # If RHS doens't look like a=( ... ), then it must be a string. return self.EvalWordToString(word)
def _ApplyUnarySuffixOp(self, val, op): assert val.tag != value_e.Undef op_kind = LookupKind(op.op_id) if op_kind == Kind.VOp1: #log('%s', op) arg_val = self.EvalWordToString(op.arg_word, do_fnmatch=True) assert arg_val.tag == value_e.Str if val.tag == value_e.Str: s = libstr.DoUnarySuffixOp(val.s, op, arg_val.s) new_val = runtime.Str(s) else: # val.tag == value_e.StrArray: # ${a[@]#prefix} is VECTORIZED on arrays. Oil should have this too. strs = [] for s in val.strs: if s is not None: strs.append(libstr.DoUnarySuffixOp(s, op, arg_val.s)) new_val = runtime.StrArray(strs) else: raise AssertionError(op_kind) return new_val
def _EvalSpecialVar(self, op_id, quoted): """Returns (val, bool maybe_decay_array). TODO: Should that boolean be part of the value? """ # $@ is special -- it need to know whether it is in a double quoted # context. # # - If it's $@ in a double quoted context, return an ARRAY. # - If it's $@ in a normal context, return a STRING, which then will be # subject to splitting. if op_id in (Id.VSub_At, Id.VSub_Star): argv = self.mem.GetArgv() val = runtime.StrArray(argv) if op_id == Id.VSub_At: # "$@" evaluates to an array, $@ should be decayed return val, not quoted else: # $@ $* "$*" return val, True elif op_id == Id.VSub_Hyphen: s = self.exec_opts.GetDollarHyphen() return runtime.Str(s), False else: val = self.mem.GetSpecialVar(op_id) return val, False # don't decay
def _InitVarsFromEnv(self, environ): # This is the way dash and bash work -- at startup, they turn everything in # 'environ' variable into shell variables. Bash has an export_env # variable. Dash has a loop through environ in init.c for n, v in environ.iteritems(): self.SetVar(ast.LhsName(n), runtime.Str(v), (var_flags_e.Exported, ), scope_e.GlobalOnly) # If it's not in the environment, initialize it. This makes it easier to # update later in ExecOpts. # TODO: IFS, PWD, etc. should follow this pattern. Maybe need a SysCall # interface? self.syscall.getcwd() etc. v = self.GetVar('SHELLOPTS') if v.tag == value_e.Undef: SetGlobalString(self, 'SHELLOPTS', '') # Now make it readonly self.SetVar(ast.LhsName('SHELLOPTS'), None, (var_flags_e.ReadOnly, ), scope_e.GlobalOnly) v = self.GetVar('HOME') if v.tag == value_e.Undef: home_dir = util.GetHomeDir() or '~' # No expansion if not found? SetGlobalString(self, 'HOME', home_dir)
def Export(argv, mem): arg, i = EXPORT_SPEC.Parse(argv) if arg.n: for name in argv[i:]: m = match.IsValidVarName(name) if not m: raise args.UsageError('export: Invalid variable name %r' % name) # NOTE: bash doesn't care if it wasn't found. mem.ClearFlag(name, var_flags_e.Exported, scope_e.Dynamic) else: for arg in argv[i:]: parts = arg.split('=', 1) if len(parts) == 1: name = parts[0] val = None # Creates an empty variable else: name, s = parts val = runtime.Str(s) m = match.IsValidVarName(name) if not m: raise args.UsageError('export: Invalid variable name %r' % name) #log('%s %s', name, val) mem.SetVar(runtime.LhsName(name), val, (var_flags_e.Exported, ), scope_e.Dynamic) return 0
def SetLocalString(mem, name, s): """Set a local string. Used for: 1) for loop iteration variables 2) temporary environments like FOO=bar BAR=$FOO cmd, 3) read builtin """ assert isinstance(s, str) mem.SetVar(ast.LhsName(name), runtime.Str(s), (), scope_e.LocalOnly)
def testExportThenAssign(self): """Regression Test""" mem = _InitMem() # export U mem.SetVar(runtime.LhsName('U'), None, (var_flags_e.Exported, ), scope_e.Dynamic) print(mem) # U=u mem.SetVar(runtime.LhsName('U'), runtime.Str('u'), (), scope_e.Dynamic) print(mem) e = mem.GetExported() self.assertEqual({'U': 'u'}, e)
def _EmptyStrOrError(self, val, token=None): assert isinstance(val, runtime.value), val if val.tag == value_e.Undef: if self.exec_opts.nounset: if token is None: e_die('Undefined variable') else: name = token.val[1:] if token.val.startswith('$') else token.val e_die('Undefined variable %r', name, token=token) else: return runtime.Str('') else: return val
def __init__(self, argv0, argv, environ, arena): top = _StackFrame() self.var_stack = [top] self.argv0 = argv0 self.argv_stack = [_ArgFrame(argv)] # NOTE: could use deque and appendleft/popleft, but: # 1. ASDL type checking of StrArray doesn't allow it (could be fixed) # 2. We don't otherwise depend on the collections module self.func_name_stack = [] # Note: we're reusing these objects because they change on every single # line! Don't want to allocate more than necsesary. self.source_name = runtime.Str('') self.line_num = runtime.Str('') self.last_status = 0 # Mutable public variable self.last_job_id = -1 # Uninitialized value mutable public variable # Done ONCE on initialization self.root_pid = os.getpid() self._InitDefaults() self._InitVarsFromEnv(environ) self.arena = arena
def EvalWordToString(self, word, do_fnmatch=False, decay=False): """ Used for redirect arg, ControlFlow arg, ArithWord, BoolWord, etc. do_fnmatch is true for case $pat and RHS of [[ == ]]. pat="*.py" case $x in $pat) echo 'matches glob pattern' ;; "$pat") echo 'equal to glob string' ;; // must be glob escaped esac """ part_vals = [] for part in word.parts: self._EvalWordPart(part, part_vals, quoted=False) strs = [] for part_val in part_vals: # TODO: if decay, then allow string part. e.g. for here word or here # doc with "$@". if part_val.tag == part_value_e.StringPartValue: # [[ foo == */"*".py ]] or case *.py) ... esac if do_fnmatch and not part_val.do_split_glob: s = glob_.GlobEscape(part_val.s) else: s = part_val.s else: if self.exec_opts.strict_array: # Examples: echo f > "$@"; local foo="$@" e_die("Expected string, got %s", part_val, word=word) # TODO: Maybe add detail like this. #e_die('RHS of assignment should only have strings. ' # 'To assign arrays, using b=( "${a[@]}" )') else: # It appears to not respect IFS s = ' '.join(s for s in part_val.strs if s is not None) strs.append(s) return runtime.Str(''.join(strs))
def GetSpecialVar(self, op_id): if op_id == Id.VSub_Bang: # $! n = self.last_job_id if n == -1: return runtime.Undef() # could be an error elif op_id == Id.VSub_QMark: # $? # TODO: Have to parse status somewhere. # External commands need WIFEXITED test. What about subshells? n = self.last_status elif op_id == Id.VSub_Pound: # $# n = self.argv_stack[-1].GetNumArgs() elif op_id == Id.VSub_Dollar: # $$ n = self.root_pid else: raise NotImplementedError(op_id) return runtime.Str(str(n))
def _EvalBracedVarSub(self, part, part_vals, quoted): """ Args: part_vals: output param to append to. """ # We have four types of operator that interact. # # 1. Bracket: value -> (value, bool maybe_decay_array) # # 2. Then these four cases are mutually exclusive: # # a. Prefix length: value -> value # b. Test: value -> part_value[] # c. Other Suffix: value -> value # d. no operator: you have a value # # That is, we don't have both prefix and suffix operators. # # 3. Process maybe_decay_array here before returning. maybe_decay_array = False # for $*, ${a[*]}, etc. var_name = None # For ${foo=default} # 1. Evaluate from (var_name, var_num, token Id) -> value if part.token.id == Id.VSub_Name: var_name = part.token.val val = self.mem.GetVar(var_name) #log('EVAL NAME %s -> %s', var_name, val) elif part.token.id == Id.VSub_Number: var_num = int(part.token.val) val = self._EvalVarNum(var_num) else: # $* decays val, maybe_decay_array = self._EvalSpecialVar( part.token.id, quoted) # 2. Bracket: value -> (value v, bool maybe_decay_array) # maybe_decay_array is for joining ${a[*]} and unquoted ${a[@]} AFTER # suffix ops are applied. If we take the length with a prefix op, the # distinction is ignored. if part.bracket_op: if part.bracket_op.tag == bracket_op_e.WholeArray: op_id = part.bracket_op.op_id if op_id == Id.Lit_At: if not quoted: maybe_decay_array = True # ${a[@]} decays but "${a[@]}" doesn't if val.tag == value_e.Undef: val = self._EmptyStrArrayOrError(part.token) elif val.tag == value_e.Str: e_die("Can't index string with @: %r", val, part=part) elif val.tag == value_e.StrArray: # TODO: Is this a no-op? Just leave 'val' alone. val = runtime.StrArray(val.strs) elif op_id == Id.Arith_Star: maybe_decay_array = True # both ${a[*]} and "${a[*]}" decay if val.tag == value_e.Undef: val = self._EmptyStrArrayOrError(part.token) elif val.tag == value_e.Str: e_die("Can't index string with *: %r", val, part=part) elif val.tag == value_e.StrArray: # TODO: Is this a no-op? Just leave 'val' alone. # ${a[*]} or "${a[*]}" : maybe_decay_array is always true val = runtime.StrArray(val.strs) else: raise AssertionError(op_id) # unknown elif part.bracket_op.tag == bracket_op_e.ArrayIndex: anode = part.bracket_op.expr if val.tag == value_e.Undef: pass # it will be checked later elif val.tag == value_e.Str: # Bash treats any string as an array, so we can't add our own # behavior here without making valid OSH invalid bash. e_die("Can't index string %r with integer", part.token.val, token=part.token) elif val.tag == value_e.StrArray: index = self.arith_ev.Eval(anode) try: # could be None because representation is sparse s = val.strs[index] except IndexError: s = None if s is None: val = runtime.Undef() else: val = runtime.Str(s) elif val.tag == value_e.AssocArray: key = self.arith_ev.Eval(anode, int_coerce=False) try: val = runtime.Str(val.d[key]) except KeyError: val = runtime.Undef() else: raise AssertionError(val.__class__.__name__) else: raise AssertionError(part.bracket_op.tag) if part.prefix_op: val = self._EmptyStrOrError(val) # maybe error val = self._ApplyPrefixOp(val, part.prefix_op) # NOTE: When applying the length operator, we can't have a test or # suffix afterward. And we don't want to decay the array elif part.suffix_op: op = part.suffix_op if op.tag == suffix_op_e.StringNullary: if op.op_id == Id.VOp0_P: # TODO: Use dependency injection #val = self.prompt._EvalPS1(val) prompt = ui.PROMPT.EvalPrompt(val) val = runtime.Str(prompt) else: raise NotImplementedError(op.op_id) elif op.tag == suffix_op_e.StringUnary: if LookupKind(part.suffix_op.op_id) == Kind.VTest: # TODO: Change style to: # if self._ApplyTestOp(...) # return # It should return whether anything was done. If not, we continue to # the end, where we might throw an error. assign_part_vals, effect = self._ApplyTestOp( val, part.suffix_op, quoted, part_vals) # NOTE: Splicing part_values is necessary because of code like # ${undef:-'a b' c 'd # e'}. Each part_value can have a different # do_glob/do_elide setting. if effect == effect_e.SpliceParts: return # EARLY RETURN, part_vals mutated elif effect == effect_e.SpliceAndAssign: if var_name is None: # TODO: error context e_die("Can't assign to special variable") else: # NOTE: This decays arrays too! 'set -o strict_array' could # avoid it. rhs_str = _DecayPartValuesToString( assign_part_vals, self.splitter.GetJoinChar()) state.SetLocalString(self.mem, var_name, rhs_str) return # EARLY RETURN, part_vals mutated elif effect == effect_e.Error: raise NotImplementedError else: # The old one #val = self._EmptyStringPartOrError(part_val, quoted) pass # do nothing, may still be undefined else: val = self._EmptyStrOrError(val) # maybe error # Other suffix: value -> value val = self._ApplyUnarySuffixOp(val, part.suffix_op) elif op.tag == suffix_op_e.PatSub: # PatSub, vectorized val = self._EmptyStrOrError(val) # ${undef//x/y} pat_val = self.EvalWordToString(op.pat, do_fnmatch=True) assert pat_val.tag == value_e.Str, pat_val if op.replace: replace_val = self.EvalWordToString(op.replace, do_fnmatch=True) assert replace_val.tag == value_e.Str, replace_val replace_str = replace_val.s else: replace_str = '' # Either GlobReplacer or ConstStringReplacer replacer = libstr.MakeReplacer(pat_val.s, replace_str, op.spids[0]) if val.tag == value_e.Str: s = replacer.Replace(val.s, op) val = runtime.Str(s) elif val.tag == value_e.StrArray: strs = [] for s in val.strs: if s is not None: strs.append(replacer.Replace(s, op)) val = runtime.StrArray(strs) else: raise AssertionError(val.__class__.__name__) elif op.tag == suffix_op_e.Slice: val = self._EmptyStrOrError(val) # ${undef:3:1} if op.begin: begin = self.arith_ev.Eval(op.begin) else: begin = 0 if op.length: length = self.arith_ev.Eval(op.length) else: length = None if val.tag == value_e.Str: # Slice UTF-8 characters in a string. s = val.s try: if begin < 0: # It could be negative if we compute unicode length, but that's # confusing. # TODO: Instead of attributing it to the word part, it would be # better if we attributed it to arith_expr begin. raise util.InvalidSlice( "The start index of a string slice can't be negative: %d", begin, part=part) byte_begin = libstr.AdvanceUtf8Chars(s, begin, 0) if length is None: byte_end = len(s) else: if length < 0: # TODO: Instead of attributing it to the word part, it would be # better if we attributed it to arith_expr begin. raise util.InvalidSlice( "The length of a string slice can't be negative: %d", length, part=part) byte_end = libstr.AdvanceUtf8Chars( s, length, byte_begin) except (util.InvalidSlice, util.InvalidUtf8) as e: if self.exec_opts.strict_word_eval: raise else: # TODO: # - We don't see the error location here, but we see it when set # -o strict-word-eval. # - Doesn't make the command exit with 1. It just sets the word # to empty string. util.warn(e.UserErrorString()) substr = '' # error condition else: substr = s[byte_begin:byte_end] val = runtime.Str(substr) elif val.tag == value_e.StrArray: # Slice array entries. # NOTE: unset elements don't count towards the length. strs = [] for s in val.strs[begin:]: if s is not None: strs.append(s) if len( strs ) == length: # never true for unspecified length break val = runtime.StrArray(strs) else: raise AssertionError( val.__class__.__name__) # Not possible # After applying suffixes, process maybe_decay_array here. if maybe_decay_array and val.tag == value_e.StrArray: val = self._DecayArray(val) # For the case where there are no prefix or suffix ops. val = self._EmptyStrOrError(val) # For example, ${a} evaluates to value_t.Str(), but we want a # part_value.StringPartValue. part_val = _ValueToPartValue(val, quoted) part_vals.append(part_val)
def _DecayArray(self, val): assert val.tag == value_e.StrArray, val sep = self.splitter.GetJoinChar() return runtime.Str(sep.join(s for s in val.strs if s is not None))
def _ApplyPrefixOp(self, val, op_id): """ Returns: value """ assert val.tag != value_e.Undef if op_id == Id.VSub_Pound: # LENGTH if val.tag == value_e.Str: # NOTE: Whether bash counts bytes or chars is affected by LANG # environment variables. # Should we respect that, or another way to select? set -o # count-bytes? # https://stackoverflow.com/questions/17368067/length-of-string-in-bash try: length = libstr.CountUtf8Chars(val.s) except util.InvalidUtf8 as e: # TODO: Add location info from 'part'? Only the caller has it. if self.exec_opts.strict_word_eval: raise else: # NOTE: Doesn't make the command exit with 1; it just returns a # length of -1. util.warn(e.UserErrorString()) return runtime.Str('-1') elif val.tag == value_e.StrArray: # There can be empty placeholder values in the array. length = sum(1 for s in val.strs if s is not None) return runtime.Str(str(length)) elif op_id == Id.VSub_Bang: # NOTES: # - Could translate to eval('$' + name) or eval("\$$name") # - ${!array[@]} means something completely different. TODO: implement # that. # - It might make sense to suggest implementing this with associative # arrays? # Treat the value of the variable as a variable name. if val.tag == value_e.Str: try: # e.g. ${!OPTIND} gives $1 when OPTIND is 1 arg_num = int(val.s) return self.mem.GetArgNum(arg_num) except ValueError: if not match.IsValidVarName(val.s): # TODO: location information. # Also note that bash doesn't consider this fatal. It makes the # command exit with '1', but we don't have that ability yet? e_die('Bad variable name %r in var ref', val.s) return self.mem.GetVar(val.s) elif val.tag == value_e.StrArray: raise NotImplementedError( '${!a[@]}') # bash gets keys this way else: raise AssertionError else: raise AssertionError(op_id)
def SetGlobalString(mem, name, s): """Helper for completion, $PWD, etc.""" assert isinstance(s, str) val = runtime.Str(s) mem.SetVar(ast.LhsName(name), val, (), scope_e.GlobalOnly)
def EvalWordToString(self, w, do_fnmatch=False, do_ere=False): # do_fnmatch: for the [[ == ]] semantics which we don't have! # I think I need another type of node # Maybe it should be BuiltinEqual and BuiltinDEqual? Parse it into a # different tree. return runtime.Str(w.s)
def GetArgNum(self, arg_num): index = self.num_shifted + arg_num - 1 if index >= len(self.argv): return runtime.Undef() return runtime.Str(str(self.argv[index]))
def _Store(self, lval, new_int): val = runtime.Str(str(new_int)) self.mem.SetVar(lval, val, (), scope_e.Dynamic)
def testSetVarClearFlag(self): mem = _InitMem() print(mem) mem.PushCall('my-func', ['ONE']) self.assertEqual(2, len(mem.var_stack)) # internal details # local x=y mem.SetVar(runtime.LhsName('x'), runtime.Str('y'), (), scope_e.LocalOnly) self.assertEqual('y', mem.var_stack[-1].vars['x'].val.s) # New frame mem.PushCall('my-func', ['TWO']) self.assertEqual(3, len(mem.var_stack)) # internal details # x=y -- test out dynamic scope mem.SetVar(runtime.LhsName('x'), runtime.Str('YYY'), (), scope_e.Dynamic) self.assertEqual('YYY', mem.var_stack[-2].vars['x'].val.s) self.assertEqual(None, mem.var_stack[-1].vars.get('x')) # myglobal=g mem.SetVar(runtime.LhsName('myglobal'), runtime.Str('g'), (), scope_e.Dynamic) self.assertEqual('g', mem.var_stack[0].vars['myglobal'].val.s) self.assertEqual(False, mem.var_stack[0].vars['myglobal'].exported) # 'export PYTHONPATH=/' mem.SetVar(runtime.LhsName('PYTHONPATH'), runtime.Str('/'), (var_flags_e.Exported, ), scope_e.Dynamic) self.assertEqual('/', mem.var_stack[0].vars['PYTHONPATH'].val.s) self.assertEqual(True, mem.var_stack[0].vars['PYTHONPATH'].exported) self.assertEqual({'PYTHONPATH': '/'}, mem.GetExported()) mem.SetVar(runtime.LhsName('PYTHONPATH'), None, (var_flags_e.Exported, ), scope_e.Dynamic) self.assertEqual(True, mem.var_stack[0].vars['PYTHONPATH'].exported) # 'export myglobal'. None means don't touch it. Undef would be confusing # because it might mean "unset", but we have a separated API for that. mem.SetVar(runtime.LhsName('myglobal'), None, (var_flags_e.Exported, ), scope_e.Dynamic) self.assertEqual(True, mem.var_stack[0].vars['myglobal'].exported) # export g2 -- define and export empty mem.SetVar(runtime.LhsName('g2'), None, (var_flags_e.Exported, ), scope_e.Dynamic) self.assertEqual(value_e.Undef, mem.var_stack[0].vars['g2'].val.tag) self.assertEqual(True, mem.var_stack[0].vars['g2'].exported) # readonly myglobal self.assertEqual(False, mem.var_stack[0].vars['myglobal'].readonly) mem.SetVar(runtime.LhsName('myglobal'), None, (var_flags_e.ReadOnly, ), scope_e.Dynamic) self.assertEqual(True, mem.var_stack[0].vars['myglobal'].readonly) mem.SetVar(runtime.LhsName('PYTHONPATH'), runtime.Str('/lib'), (), scope_e.Dynamic) self.assertEqual('/lib', mem.var_stack[0].vars['PYTHONPATH'].val.s) self.assertEqual(True, mem.var_stack[0].vars['PYTHONPATH'].exported) # COMPREPLY=(1 2 3) # invariant to enforce: arrays can't be exported mem.SetVar(runtime.LhsName('COMPREPLY'), runtime.StrArray(['1', '2', '3']), (), scope_e.GlobalOnly) self.assertEqual(['1', '2', '3'], mem.var_stack[0].vars['COMPREPLY'].val.strs) # export COMPREPLY try: mem.SetVar(runtime.LhsName('COMPREPLY'), None, (var_flags_e.Exported, ), scope_e.Dynamic) except util.FatalRuntimeError as e: pass else: self.fail("Expected failure") # readonly r=1 mem.SetVar(runtime.LhsName('r'), runtime.Str('1'), (var_flags_e.ReadOnly, ), scope_e.Dynamic) self.assertEqual('1', mem.var_stack[0].vars['r'].val.s) self.assertEqual(False, mem.var_stack[0].vars['r'].exported) self.assertEqual(True, mem.var_stack[0].vars['r'].readonly) print(mem) # r=newvalue try: mem.SetVar(runtime.LhsName('r'), runtime.Str('newvalue'), (), scope_e.Dynamic) except util.FatalRuntimeError as e: pass else: self.fail("Expected failure") # readonly r2 -- define empty readonly mem.SetVar(runtime.LhsName('r2'), None, (var_flags_e.ReadOnly, ), scope_e.Dynamic) self.assertEqual(value_e.Undef, mem.var_stack[0].vars['r2'].val.tag) self.assertEqual(True, mem.var_stack[0].vars['r2'].readonly) # export -n PYTHONPATH # Remove the exported property. NOTE: scope is LocalOnly for Oil? self.assertEqual(True, mem.var_stack[0].vars['PYTHONPATH'].exported) mem.ClearFlag('PYTHONPATH', var_flags_e.Exported, scope_e.Dynamic) self.assertEqual(False, mem.var_stack[0].vars['PYTHONPATH'].exported) # a[1]=2 mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('2'), (), scope_e.Dynamic) self.assertEqual([None, '2'], mem.var_stack[0].vars['a'].val.strs) # a[1]=3 mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('3'), (), scope_e.Dynamic) self.assertEqual([None, '3'], mem.var_stack[0].vars['a'].val.strs) # a[1]=(x y z) # illegal try: mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.StrArray(['x', 'y', 'z']), (), scope_e.Dynamic) except util.FatalRuntimeError as e: pass else: self.fail("Expected failure") # readonly a mem.SetVar(runtime.LhsName('a'), None, (var_flags_e.ReadOnly, ), scope_e.Dynamic) self.assertEqual(True, mem.var_stack[0].vars['a'].readonly) try: # a[2]=3 mem.SetVar(runtime.LhsIndexedName('a', 1), runtime.Str('3'), (), scope_e.Dynamic) except util.FatalRuntimeError as e: pass else: self.fail("Expected failure")
def _Dispatch(self, node, fork_external): # If we call RunCommandSub in a recursive call to the executor, this will # be set true (if strict-errexit is false). But it only lasts for one # command. self.check_command_sub_status = False #argv0 = None # for error message check_errexit = False # for errexit if node.tag == command_e.SimpleCommand: check_errexit = True # Find span_id for a basic implementation of $LINENO, e.g. # PS4='+$SOURCE_NAME:$LINENO:' # NOTE: osh2oil uses node.more_env, but we don't need that. span_id = const.NO_INTEGER if node.words: first_word = node.words[0] span_id = word.LeftMostSpanForWord(first_word) self.mem.SetCurrentSpanId(span_id) # PROBLEM: We want to log argv in 'xtrace' mode, but we may have already # redirected here, which screws up logging. For example, 'echo hi # >/dev/null 2>&1'. We want to evaluate argv and log it BEFORE applying # redirects. # Another problem: # - tracing can be called concurrently from multiple processes, leading # to overlap. Maybe have a mode that creates a file per process. # xtrace-proc # - line numbers for every command would be very nice. But then you have # to print the filename too. words = braces.BraceExpandWords(node.words) argv = self.word_ev.EvalWordSequence(words) # This comes before evaluating env, in case there are problems evaluating # it. We could trace the env separately? Also trace unevaluated code # with set-o verbose? self.tracer.OnSimpleCommand(argv) if node.more_env: self.mem.PushTemp() try: for env_pair in node.more_env: val = self.word_ev.EvalWordToString(env_pair.val) # Set each var so the next one can reference it. Example: # FOO=1 BAR=$FOO ls / self.mem.SetVar(ast.LhsName(env_pair.name), val, (var_flags_e.Exported,), scope_e.TempEnv) # NOTE: This might never return! In the case of fork_external=False. status = self._RunSimpleCommand(argv, fork_external, span_id) finally: if node.more_env: self.mem.PopTemp() elif node.tag == command_e.Sentence: # Don't check_errexit since this isn't a real node! if node.terminator.id == Id.Op_Semi: status = self._Execute(node.child) else: status = self._RunJobInBackground(node.child) elif node.tag == command_e.Pipeline: check_errexit = True if node.stderr_indices: raise NotImplementedError('|&') if node.negated: self._PushErrExit() try: status2 = self._RunPipeline(node) finally: self._PopErrExit() # errexit is disabled for !. check_errexit = False status = 1 if status2 == 0 else 0 else: status = self._RunPipeline(node) elif node.tag == command_e.Subshell: check_errexit = True # This makes sure we don't waste a process if we'd launch one anyway. p = self._MakeProcess(node.child) status = p.Run(self.waiter) elif node.tag == command_e.DBracket: check_errexit = True result = self.bool_ev.Eval(node.expr) status = 0 if result else 1 elif node.tag == command_e.DParen: check_errexit = True i = self.arith_ev.Eval(node.child) status = 0 if i != 0 else 1 elif node.tag == command_e.Assignment: flags = word_compile.ParseAssignFlags(node.flags) if node.keyword == Id.Assign_Local: lookup_mode = scope_e.LocalOnly # typeset and declare are synonyms? I see typeset -a a=() the most. elif node.keyword in (Id.Assign_Declare, Id.Assign_Typeset): # declare is like local, except it can also be used outside functions? if var_flags_e.Global in flags: lookup_mode = scope_e.GlobalOnly else: lookup_mode = scope_e.LocalOnly elif node.keyword == Id.Assign_Readonly: lookup_mode = scope_e.Dynamic flags.append(var_flags_e.ReadOnly) elif node.keyword == Id.Assign_None: # mutate existing local or global lookup_mode = scope_e.Dynamic else: raise AssertionError(node.keyword) for pair in node.pairs: if pair.op == assign_op_e.PlusEqual: assert pair.rhs, pair.rhs # I don't think a+= is valid? val = self.word_ev.EvalRhsWord(pair.rhs) old_val, lval = expr_eval.EvalLhsAndLookup(pair.lhs, self.arith_ev, self.mem, self.exec_opts) sig = (old_val.tag, val.tag) if sig == (value_e.Undef, value_e.Str): pass # val is RHS elif sig == (value_e.Undef, value_e.StrArray): pass # val is RHS elif sig == (value_e.Str, value_e.Str): val = runtime.Str(old_val.s + val.s) elif sig == (value_e.Str, value_e.StrArray): e_die("Can't append array to string") elif sig == (value_e.StrArray, value_e.Str): e_die("Can't append string to array") elif sig == (value_e.StrArray, value_e.StrArray): val = runtime.StrArray(old_val.strs + val.strs) else: # plain assignment spid = pair.spids[0] # Source location for tracing lval = self._EvalLhs(pair.lhs, spid, lookup_mode) # RHS can be a string or array. if pair.rhs: val = self.word_ev.EvalRhsWord(pair.rhs) assert isinstance(val, runtime.value), val else: # e.g. 'readonly x' or 'local x' val = None # NOTE: In bash and mksh, declare -a myarray makes an empty cell with # Undef value, but the 'array' attribute. #log('setting %s to %s with flags %s', lval, val, flags) self.mem.SetVar(lval, val, flags, lookup_mode, strict_array=self.exec_opts.strict_array) # Assignment always appears to have a spid. if node.spids: current_spid = node.spids[0] else: current_spid = const.NO_INTEGER self.mem.SetCurrentSpanId(current_spid) self.tracer.OnAssignment(lval, pair.op, val, flags, lookup_mode) # PATCH to be compatible with existing shells: If the assignment had a # command sub like: # # s=$(echo one; false) # # then its status will be in mem.last_status, and we can check it here. # If there was NOT a command sub in the assignment, then we don't want to # check it. if node.keyword == Id.Assign_None: # mutate existing local or global # Only do this if there was a command sub? How? Look at node? # Set a flag in mem? self.mem.last_status or if self.check_command_sub_status: self._CheckStatus(self.mem.last_status, node) # A global assignment shouldn't clear $?. status = self.mem.last_status else: status = 0 else: # To be compatible with existing shells, local assignments DO clear # $?. Even in strict mode, we don't need to bother setting # check_errexit = True, because we would have already checked the # command sub in RunCommandSub. status = 0 # TODO: maybe we should have a "sane-status" that respects this: # false; echo $?; local f=x; echo $? elif node.tag == command_e.ControlFlow: if node.arg_word: # Evaluate the argument val = self.word_ev.EvalWordToString(node.arg_word) assert val.tag == value_e.Str arg = int(val.s) # They all take integers else: arg = 0 # return 0, exit 0, break 0 levels, etc. # NOTE: We don't do anything about a top-level 'return' here. Unlike in # bash, that is OK. If you can return from a sourced script, it makes # sense to return from a main script. ok = True tok = node.token if (tok.id in (Id.ControlFlow_Break, Id.ControlFlow_Continue) and self.loop_level == 0): ok = False msg = 'Invalid control flow at top level' if ok: raise _ControlFlow(tok, arg) if self.exec_opts.strict_control_flow: e_die(msg, token=tok) else: # Only print warnings, never fatal. # Bash oddly only exits 1 for 'return', but no other shell does. ui.PrintFilenameAndLine(tok.span_id, self.arena) util.warn(msg) status = 0 # The only difference between these two is that CommandList has no # redirects. We already took care of that above. elif node.tag in (command_e.CommandList, command_e.BraceGroup): status = self._ExecuteList(node.children) check_errexit = False elif node.tag == command_e.AndOr: # NOTE: && and || have EQUAL precedence in command mode. See case #13 # in dbracket.test.sh. left = node.children[0] # Suppress failure for every child except the last one. self._PushErrExit() try: status = self._Execute(left) finally: self._PopErrExit() i = 1 n = len(node.children) while i < n: #log('i %d status %d', i, status) child = node.children[i] op_id = node.ops[i-1] #log('child %s op_id %s', child, op_id) if op_id == Id.Op_DPipe and status == 0: i += 1 continue # short circuit elif op_id == Id.Op_DAmp and status != 0: i += 1 continue # short circuit if i == n - 1: # errexit handled differently for last child status = self._Execute(child) check_errexit = True else: self._PushErrExit() try: status = self._Execute(child) finally: self._PopErrExit() i += 1 elif node.tag == command_e.WhileUntil: if node.keyword.id == Id.KW_While: _DonePredicate = lambda status: status != 0 else: _DonePredicate = lambda status: status == 0 status = 0 self.loop_level += 1 try: while True: self._PushErrExit() try: cond_status = self._ExecuteList(node.cond) finally: self._PopErrExit() done = cond_status != 0 if _DonePredicate(cond_status): break try: status = self._Execute(node.body) # last one wins except _ControlFlow as e: if e.IsBreak(): status = 0 break elif e.IsContinue(): status = 0 continue else: # return needs to pop up more raise finally: self.loop_level -= 1 elif node.tag == command_e.ForEach: iter_name = node.iter_name if node.do_arg_iter: iter_list = self.mem.GetArgv() else: words = braces.BraceExpandWords(node.iter_words) iter_list = self.word_ev.EvalWordSequence(words) # We need word splitting and so forth # NOTE: This expands globs too. TODO: We should pass in a Globber() # object. status = 0 # in case we don't loop self.loop_level += 1 try: for x in iter_list: #log('> ForEach setting %r', x) state.SetLocalString(self.mem, iter_name, x) #log('<') try: status = self._Execute(node.body) # last one wins except _ControlFlow as e: if e.IsBreak(): status = 0 break elif e.IsContinue(): status = 0 else: # return needs to pop up more raise finally: self.loop_level -= 1 elif node.tag == command_e.ForExpr: status = 0 init, cond, body, update = node.init, node.cond, node.body, node.update if init: self.arith_ev.Eval(init) self.loop_level += 1 try: while True: if cond: b = self.arith_ev.Eval(cond) if not b: break try: status = self._Execute(body) except _ControlFlow as e: if e.IsBreak(): status = 0 break elif e.IsContinue(): status = 0 else: # return needs to pop up more raise if update: self.arith_ev.Eval(update) finally: self.loop_level -= 1 elif node.tag == command_e.DoGroup: status = self._ExecuteList(node.children) check_errexit = False # not real statements elif node.tag == command_e.FuncDef: # NOTE: Would it make sense to evaluate the redirects BEFORE entering? # It will save time on function calls. self.funcs[node.name] = node status = 0 elif node.tag == command_e.If: done = False for arm in node.arms: self._PushErrExit() try: status = self._ExecuteList(arm.cond) finally: self._PopErrExit() if status == 0: status = self._ExecuteList(arm.action) done = True break # TODO: The compiler should flatten this if not done and node.else_action is not None: status = self._ExecuteList(node.else_action) elif node.tag == command_e.NoOp: status = 0 # make it true elif node.tag == command_e.Case: val = self.word_ev.EvalWordToString(node.to_match) to_match = val.s status = 0 # If there are no arms, it should be zero? done = False for arm in node.arms: for pat_word in arm.pat_list: # NOTE: Is it OK that we're evaluating these as we go? # TODO: case "$@") shouldn't succeed? That's a type error? # That requires strict-array? pat_val = self.word_ev.EvalWordToString(pat_word, do_fnmatch=True) #log('Matching word %r against pattern %r', to_match, pat_val.s) if libc.fnmatch(pat_val.s, to_match): status = self._ExecuteList(arm.action) done = True # TODO: Parse ;;& and for fallthrough and such? break # Only execute action ONCE if done: break elif node.tag == command_e.TimeBlock: # TODO: # - When do we need RUSAGE_CHILDREN? # - Respect TIMEFORMAT environment variable. # "If this variable is not set, Bash acts as if it had the value" # $'\nreal\t%3lR\nuser\t%3lU\nsys\t%3lS' # "A trailing newline is added when the format string is displayed." start_t = time.time() # calls gettimeofday() under the hood start_u = resource.getrusage(resource.RUSAGE_SELF) status = self._Execute(node.pipeline) end_t = time.time() end_u = resource.getrusage(resource.RUSAGE_SELF) real = end_t - start_t user = end_u.ru_utime - start_u.ru_utime sys_ = end_u.ru_stime - start_u.ru_stime libc.print_time(real, user, sys_) else: raise NotImplementedError(node.__class__.__name__) return status, check_errexit
def GetArgNum(self, arg_num): if arg_num == 0: return runtime.Str(self.argv0) return self.argv_stack[-1].GetArgNum(arg_num)