def SetOption(self, opt_name, b): """ For set -o, set +o, or shopt -s/-u -o. """ self._SetOption(opt_name, b) val = self.mem.GetVar('SHELLOPTS') assert val.tag == value_e.Str shellopts = val.s # Now check if SHELLOPTS needs to be updated. It may be exported. # # NOTE: It might be better to skip rewriting SEHLLOPTS in the common case # where it is not used. We could do it lazily upon GET. # Also, it would be slightly more efficient to update SHELLOPTS if # settings were batched, Examples: # - set -eu # - shopt -s foo bar if b: if opt_name not in shellopts: new_val = value.Str('%s:%s' % (shellopts, opt_name)) self.mem.InternalSetGlobal('SHELLOPTS', new_val) else: if opt_name in shellopts: names = [n for n in shellopts.split(':') if n != opt_name] new_val = value.Str(':'.join(names)) self.mem.InternalSetGlobal('SHELLOPTS', new_val)
def testPushTemp(self): mem = _InitMem() # x=1 mem.SetVar( lvalue.LhsName('x'), value.Str('1'), (), scope_e.Dynamic) self.assertEqual('1', mem.var_stack[-1]['x'].val.s) mem.PushTemp() self.assertEqual(2, len(mem.var_stack)) # x=temp E=3 read x <<< 'line' mem.SetVar( lvalue.LhsName('x'), value.Str('temp'), (), scope_e.LocalOnly) mem.SetVar( lvalue.LhsName('E'), value.Str('3'), (), scope_e.LocalOnly) mem.SetVar( lvalue.LhsName('x'), value.Str('line'), (), scope_e.LocalOnly) self.assertEqual('3', mem.var_stack[-1]['E'].val.s) self.assertEqual('line', mem.var_stack[-1]['x'].val.s) self.assertEqual('1', mem.var_stack[-2]['x'].val.s) mem.PopTemp() self.assertEqual(1, len(mem.var_stack)) self.assertEqual('1', mem.var_stack[-1]['x'].val.s)
def testPushTemp(self): mem = _InitMem() # x=1 mem.SetVar( lvalue.LhsName('x'), value.Str('1'), (), scope_e.Dynamic) self.assertEqual('1', mem.var_stack[-1].vars['x'].val.s) mem.PushTemp() self.assertEqual(2, len(mem.var_stack)) # Temporary frame is immutable self.assertEqual(False, mem.var_stack[-1].mutable) self.assertEqual(True, mem.var_stack[-2].mutable) # x=temp E=3 read x <<< 'line' mem.SetVar( lvalue.LhsName('x'), value.Str('temp'), (), scope_e.TempEnv) mem.SetVar( lvalue.LhsName('E'), value.Str('3'), (), scope_e.TempEnv) mem.SetVar( lvalue.LhsName('x'), value.Str('line'), (), scope_e.LocalOnly) self.assertEqual('3', mem.var_stack[-1].vars['E'].val.s) self.assertEqual('temp', mem.var_stack[-1].vars['x'].val.s) self.assertEqual('line', mem.var_stack[-2].vars['x'].val.s) mem.PopTemp() self.assertEqual(1, len(mem.var_stack)) self.assertEqual('line', mem.var_stack[-1].vars['x'].val.s)
def __init__(self, dollar0, argv, environ, arena, has_main=False): self.dollar0 = dollar0 self.argv_stack = [_ArgFrame(argv)] self.var_stack = [{}] # The debug_stack isn't strictly necessary for execution. We use it for # crash dumps and for 3 parallel arrays: FUNCNAME, CALL_SOURCE, # BASH_LINENO. The First frame points at the global vars and argv. self.debug_stack = [(None, None, const.NO_INTEGER, 0, 0)] self.bash_source = [] # for implementing BASH_SOURCE self.has_main = has_main if has_main: self.bash_source.append(dollar0) # e.g. the filename self.current_spid = const.NO_INTEGER # Note: we're reusing these objects because they change on every single # line! Don't want to allocate more than necsesary. self.source_name = value.Str('') self.line_num = value.Str('') self.last_status = [0] # type: List[int] # a stack self.pipe_status = [[]] # type: List[List[int]] # stack self.last_job_id = -1 # Uninitialized value mutable public variable # Done ONCE on initialization self.root_pid = posix.getpid() self._InitDefaults() self._InitVarsFromEnv(environ) self.arena = arena
def Parse(spec, arg_r): # type: (flag_spec._FlagSpec, Reader) -> _Attributes # NOTE about -: # 'set -' ignores it, vs set # 'unset -' or 'export -' seems to treat it as a variable name out = _Attributes(spec.defaults) while not arg_r.AtEnd(): arg = arg_r.Peek() if arg == '--': out.saw_double_dash = True arg_r.Next() break if arg.startswith('-') and len(arg) > 1: n = len(arg) for i in xrange(1, n): # parse flag combos like -rx ch = arg[i] if ch in spec.plus_flags: out.Set(ch, value.Str('-')) continue if ch in spec.arity0: # e.g. read -r out.SetTrue(ch) continue if ch in spec.arity1: # e.g. read -t1.0 action = spec.arity1[ch] # make sure we don't pass empty string for read -t attached_arg = arg[i + 1:] if i < n - 1 else None action.OnMatch(attached_arg, arg_r, out) break e_usage("doesn't accept flag %s" % ('-' + ch), span_id=arg_r.SpanId()) arg_r.Next() # next arg # Only accept + if there are ANY options defined, e.g. for declare +rx. elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1: n = len(arg) for i in xrange(1, n): # parse flag combos like -rx ch = arg[i] if ch in spec.plus_flags: out.Set(ch, value.Str('+')) continue e_usage("doesn't accept option %s" % ('+' + ch), span_id=arg_r.SpanId()) arg_r.Next() # next arg else: # a regular arg break return out
def EvalWordToString(self, word, do_fnmatch=False, do_ere=False): """ Args: word: CompoundWord Used for redirect arg, ControlFlow arg, ArithWord, BoolWord, etc. do_fnmatch is true for case $pat and RHS of [[ == ]]. pat="*.py" case $x in $pat) echo 'matches glob pattern' ;; "$pat") echo 'equal to glob string' ;; # must be glob escaped esac TODO: Raise AssertionError if it has ExtGlobPart. """ if word.tag == word_e.EmptyWord: return value.Str('') part_vals = [] for p in word.parts: self._EvalWordPart(p, part_vals, quoted=False) strs = [] for part_val in part_vals: if part_val.tag == part_value_e.String: # [[ foo == */"*".py ]] or case *.py) ... esac if do_fnmatch and not part_val.do_split_glob: s = glob_.GlobEscape(part_val.s) elif do_ere and not part_val.do_split_glob: s = glob_.ExtendedRegexEscape(part_val.s) else: s = part_val.s else: if self.exec_opts.strict_array: # Examples: echo f > "$@"; local foo="$@" # TODO: This attributes too coarsely, to the word rather than the # parts. Problem: the word is a TREE of parts, but we only have a # flat list of part_vals. The only case where we really get arrays # is "$@", "${a[@]}", "${a[@]//pat/replace}", etc. e_die( "This word should evaluate to a string, but part of it was an " "array", word=word) # TODO: Maybe add detail like this. #e_die('RHS of assignment should only have strings. ' # 'To assign arrays, use b=( "${a[@]}" )') else: # It appears to not respect IFS s = ' '.join(s for s in part_val.strs if s is not None) strs.append(s) return value.Str(''.join(strs))
def _SetToArg(action, suffix, arg_r, out): # type: (SetToArgAction, Optional[str], Reader, _Attributes) -> bool """ Perform the action. """ if suffix: # for the ',' in -d, arg = suffix else: arg_r.Next() arg = arg_r.Peek() if arg is None: e_usage('expected argument to %r' % ('-' + action.name), span_id=arg_r.SpanId()) # e.g. spec.LongFlag('--format', ['text', 'html']) # Should change to arg.Enum([...]) with tagswitch(action.flag_type) as case: if case(flag_type_e.Enum): alts = cast(flag_type__Enum, action.flag_type).alts if arg not in alts: e_usage('got invalid argument %r to %r, expected one of: %s' % (arg, ('-' + action.name), ', '.join(alts)), span_id=arg_r.SpanId()) val = value.Str(arg) # type: value_t elif case(flag_type_e.Str): val = value.Str(arg) elif case(flag_type_e.Int): try: i = int(arg) except ValueError: e_usage('expected integer after %s, got %r' % ('-' + action.name, arg), span_id=arg_r.SpanId()) # So far all our int values are > 0, so use -1 as the 'unset' value if i < 0: e_usage('got invalid integer for %s: %s' % ('-' + action.name, arg), span_id=arg_r.SpanId()) val = value.Int(i) elif case(flag_type_e.Float): try: val = value.Float(float(arg)) except ValueError: e_usage('expected number after %r, got %r' % ('-' + action.name, arg), span_id=arg_r.SpanId()) else: raise AssertionError() out.Set(action.name, val) return action.quit_parsing_flags
def testEvaluator(self): arena = test_lib.MakeArena('<ui_test.py>') mem = state.Mem('', [], {}, arena) ex = test_lib.InitExecutor(arena=arena) p = prompt.Evaluator('osh', arena, ex.parse_ctx, ex, mem) # Rgression for caching bug! self.assertEqual('foo', p.EvalPrompt(value.Str('foo'))) self.assertEqual('foo', p.EvalPrompt(value.Str('foo')))
def testGetVar(self): mem = _InitMem() # readonly a=x mem.SetVar(lvalue.Named('a'), value.Str('x'), (var_flags_e.ReadOnly, ), scope_e.Dynamic) val = mem.GetVar('a', scope_e.Dynamic) test_lib.AssertAsdlEqual(self, value.Str('x'), val) val = mem.GetVar('undef', scope_e.Dynamic) test_lib.AssertAsdlEqual(self, value.Undef(), val)
def testGetValue(self): mem = _InitMem() # readonly a=x mem.SetValue(lvalue.Named('a'), value.Str('x'), scope_e.Dynamic, flags=state.SetReadOnly) val = mem.GetValue('a', scope_e.Dynamic) test_lib.AssertAsdlEqual(self, value.Str('x'), val) val = mem.GetValue('undef', scope_e.Dynamic) test_lib.AssertAsdlEqual(self, value.Undef(), val)
def OnMatch(self, prefix, suffix, arg_r, out): # type: (Optional[str], Optional[str], Reader, _Attributes) -> bool """Called when the flag matches.""" if suffix: # for the ',' in -d, arg = suffix else: arg_r.Next() arg = arg_r.Peek() if arg is None: e_usage('expected argument to %r' % ('-' + self.name), span_id=arg_r.SpanId()) # e.g. spec.LongFlag('--format', ['text', 'html']) # Should change to arg.Enum([...]) with tagswitch(self.flag_type) as case: if case(flag_type_e.Enum): alts = cast(flag_type__Enum, self.flag_type).alts if arg not in alts: e_usage( 'got invalid argument %r to %r, expected one of: %s' % (arg, ('-' + self.name), ', '.join(alts)), span_id=arg_r.SpanId()) val = value.Str(arg) # type: value_t elif case(flag_type_e.Str): val = value.Str(arg) elif case(flag_type_e.Int): try: val = value.Int(int(arg)) except ValueError: e_usage('expected integer after %r, got %r' % ('-' + self.name, arg), span_id=arg_r.SpanId()) elif case(flag_type_e.Float): try: val = value.Float(float(arg)) except ValueError: e_usage('expected number after %r, got %r' % ('-' + self.name, arg), span_id=arg_r.SpanId()) else: raise AssertionError() out.Set(self.name, val) return self.quit_parsing_flags
def _InitVarsFromEnv(self, environ): # This is the way dash and bash work -- at startup, they turn everything in # 'environ' variable into shell variables. Bash has an export_env # variable. Dash has a loop through environ in init.c for n, v in environ.iteritems(): self.SetVar(lhs_expr.LhsName(n), value.Str(v), (var_flags_e.Exported, ), scope_e.GlobalOnly) # If it's not in the environment, initialize it. This makes it easier to # update later in ExecOpts. # TODO: IFS, etc. should follow this pattern. Maybe need a SysCall # interface? self.syscall.getcwd() etc. v = self.GetVar('SHELLOPTS') if v.tag == value_e.Undef: SetGlobalString(self, 'SHELLOPTS', '') # Now make it readonly self.SetVar(lhs_expr.LhsName('SHELLOPTS'), None, (var_flags_e.ReadOnly, ), scope_e.GlobalOnly) # Usually we inherit PWD from the parent shell. When it's not set, we may # compute it. v = self.GetVar('PWD') if v.tag == value_e.Undef: SetGlobalString(self, 'PWD', _GetWorkingDir()) # Now mark it exported, no matter what. This is one of few variables # EXPORTED. bash and dash both do it. (e.g. env -i -- dash -c env) self.SetVar(lhs_expr.LhsName('PWD'), None, (var_flags_e.Exported, ), scope_e.GlobalOnly)
def Export(argv, mem): arg, i = EXPORT_SPEC.Parse(argv) if arg.n: for name in argv[i:]: m = match.IsValidVarName(name) if not m: raise args.UsageError('export: Invalid variable name %r' % name) # NOTE: bash doesn't care if it wasn't found. mem.ClearFlag(name, var_flags_e.Exported, scope_e.Dynamic) else: for arg in argv[i:]: parts = arg.split('=', 1) if len(parts) == 1: name = parts[0] val = None # Creates an empty variable else: name, s = parts val = value.Str(s) m = match.IsValidVarName(name) if not m: raise args.UsageError('export: Invalid variable name %r' % name) #log('%s %s', name, val) mem.SetVar(lvalue.LhsName(name), val, (var_flags_e.Exported, ), scope_e.Dynamic) return 0
def _Value(self, arg, span_id): # type: (str, int) -> value_t if self.valid is not None and arg not in self.valid: e_usage('got invalid argument %r to %r, expected one of: %s' % (arg, ('-' + self.name), '|'.join(self.valid)), span_id=span_id) return value.Str(arg)
def testValidEscapes(self): for prompt_str in [ "\[\033[01;34m\]user\[\033[00m\] >", "\[\]\[\]\[\]", "\[\] hi \[hi\] \[\] hello"]: self.assertEqual( self.p.EvalPrompt(value.Str(prompt_str)), prompt_str.replace("\[", "\x01").replace("\]", "\x02"))
def _EvalSpecialVar(self, op_id, quoted): """Returns (val, bool maybe_decay_array). TODO: Should that boolean be part of the value? """ # $@ is special -- it need to know whether it is in a double quoted # context. # # - If it's $@ in a double quoted context, return an ARRAY. # - If it's $@ in a normal context, return a STRING, which then will be # subject to splitting. if op_id in (Id.VSub_At, Id.VSub_Star): argv = self.mem.GetArgv() val = value.StrArray(argv) if op_id == Id.VSub_At: # "$@" evaluates to an array, $@ should be decayed return val, not quoted else: # $@ $* "$*" return val, True elif op_id == Id.VSub_Hyphen: s = self.exec_opts.GetDollarHyphen() return value.Str(s), False else: val = self.mem.GetSpecialVar(op_id) return val, False # don't decay
def SetStringDynamic(mem, name, s): """Set a string by looking up the stack. Used for getopts. """ assert isinstance(s, str) mem.SetVar(lhs_expr.LhsName(name), value.Str(s), (), scope_e.Dynamic)
def _ApplyUnarySuffixOp(self, val, op): assert val.tag != value_e.Undef op_kind = LookupKind(op.op_id) if op_kind == Kind.VOp1: #log('%s', op) arg_val = self.EvalWordToString(op.arg_word, do_fnmatch=True) assert arg_val.tag == value_e.Str if val.tag == value_e.Str: s = string_ops.DoUnarySuffixOp(val.s, op, arg_val.s) new_val = value.Str(s) else: # val.tag == value_e.StrArray: # ${a[@]#prefix} is VECTORIZED on arrays. Oil should have this too. strs = [] for s in val.strs: if s is not None: strs.append(string_ops.DoUnarySuffixOp(s, op, arg_val.s)) new_val = value.StrArray(strs) else: raise AssertionError(op_kind) return new_val
def __call__(self, cmd_val): arg_r = args.Reader(cmd_val.argv, spids=cmd_val.arg_spids) arg_r.Next() arg, _ = GETLINE_SPEC.Parse(arg_r) if arg.cstr: # TODO: implement it # returns error if it can't decode raise NotImplementedError() var_name, var_spid = arg_r.ReadRequired2('requires a variable name') if var_name.startswith(':'): # optional : sigil var_name = var_name[1:] next_arg, next_spid = arg_r.Peek2() if next_arg is not None: raise args.UsageError('got extra argument', span_id=next_spid) # TODO: use a more efficient function in C line = builtin.ReadLineFromStdin() if not line: # EOF return 1 if not arg.end: if line.endswith('\r\n'): line = line[:-2] elif line.endswith('\n'): line = line[:-1] self.mem.SetVar(sh_lhs_expr.Name(var_name), value.Str(line), (), scope_e.LocalOnly) return 0
def _InitVarsFromEnv(self, environ): # This is the way dash and bash work -- at startup, they turn everything in # 'environ' variable into shell variables. Bash has an export_env # variable. Dash has a loop through environ in init.c for n, v in environ.iteritems(): self.SetVar(lhs_expr.LhsName(n), value.Str(v), (var_flags_e.Exported,), scope_e.GlobalOnly) # If it's not in the environment, initialize it. This makes it easier to # update later in ExecOpts. # TODO: IFS, PWD, etc. should follow this pattern. Maybe need a SysCall # interface? self.syscall.getcwd() etc. v = self.GetVar('SHELLOPTS') if v.tag == value_e.Undef: SetGlobalString(self, 'SHELLOPTS', '') # Now make it readonly self.SetVar( lhs_expr.LhsName('SHELLOPTS'), None, (var_flags_e.ReadOnly,), scope_e.GlobalOnly) v = self.GetVar('HOME') if v.tag == value_e.Undef: # TODO: Should lack of a home dir be an error? What does bash do? home_dir = _GetHomeDir() or '~' SetGlobalString(self, 'HOME', home_dir)
def EvalForPlugin(self, w): """Wrapper around EvalWordToString that prevents errors. Runtime errors like $(( 1 / 0 )) and mutating $? like $(exit 42) are handled here. """ self.mem.PushStatusFrame() # to "sandbox" $? and $PIPESTATUS try: val = self.EvalWordToString(w) except util.FatalRuntimeError as e: val = value.Str("<Runtime error: %s>" % e.UserErrorString()) except (OSError, IOError) as e: # This is like the catch-all in Executor.ExecuteAndCatch(). val = value.Str("<I/O error: %s>" % posix.strerror(e.errno)) finally: self.mem.PopStatusFrame() return val
def _All(self, var_name): # type: (str) -> int contents = _ReadAll() # No error conditions? lhs = lvalue.Named(var_name) self.mem.SetValue(lhs, value.Str(contents), scope_e.LocalOnly) return 0
def EvalWordToString(self, w, quote_kind=quote_e.Default): # type: (word_t, quote_t) -> value__Str # do_fnmatch: for the [[ == ]] semantics which we don't have! # I think I need another type of node # Maybe it should be BuiltinEqual and BuiltinDEqual? Parse it into a # different tree. assert w.tag_() == word_e.String string_word = cast(word__String, w) return value.Str(string_word.s)
def SetLocalString(mem, name, s): """Set a local string. Used for: 1) for loop iteration variables 2) temporary environments like FOO=bar BAR=$FOO cmd, 3) read builtin """ assert isinstance(s, str) mem.SetVar(lhs_expr.LhsName(name), value.Str(s), (), scope_e.LocalOnly)
def ShortFlag(self, short_name, arg_type=None, help=None): # type: (str, Optional[int], Optional[str]) -> None """ This is very similar to ShortFlag for FlagSpecAndMore, except we have separate arity0 and arity1 dicts. """ assert short_name.startswith('-'), short_name assert len(short_name) == 2, short_name char = short_name[1] if arg_type is None: self.arity0[char] = True else: self.arity1[char] = args.SetToArg(char, _FlagType(arg_type)) # TODO: callers should pass flag_type if arg_type is None: typ = flag_type.Bool() # type: flag_type_t default = value.Bool(False) # type: value_t elif arg_type == args.Int: typ = flag_type.Int() default = value.Int(-1) elif arg_type == args.Float: typ = flag_type.Float() default = value.Float(0.0) elif arg_type == args.String: typ = flag_type.Str() default = value.Str('') elif isinstance(arg_type, list): typ = flag_type.Enum(arg_type) default = value.Str('') # This isn't valid else: raise AssertionError(arg_type) if self.typed: self.defaults[char] = default else: # TODO: remove when all builtins converted self.defaults[char] = value.Undef() self.fields[char] = typ
def _Default(arg_type, arg_default=None): # type: (Union[None, int, List[str]], Optional[str]) -> value_t # for enum or string # note: not using this for integers yet if arg_default is not None: return value.Str(arg_default) # early return if arg_type is None: default = value.Bool(False) # type: value_t elif arg_type == args.Int: default = value.Int(-1) # positive values aren't allowed now elif arg_type == args.Float: default = value.Float(0.0) elif arg_type == args.String: default = value.Undef() # e.g. read -d '' is NOT the default elif isinstance(arg_type, list): default = value.Str('') # This isn't valid else: raise AssertionError(arg_type) return default
def __init__( self, parse_ctx, # type: ParseContext exec_opts, # type: optview.Exec mutable_opts, # type: MutableOpts mem, # type: Mem f, # type: _DebugFile ): # type: (...) -> None """ Args: parse_ctx: For parsing PS4. exec_opts: For xtrace setting mem: for retrieving PS4 word_ev: for evaluating PS4 """ self.parse_ctx = parse_ctx self.exec_opts = exec_opts self.mutable_opts = mutable_opts self.mem = mem self.f = f # can be stderr, the --debug-file, etc. self.word_ev = None # type: NormalWordEvaluator self.ind = 0 # changed by process, proc, source, eval self.indents = [''] # "pooled" to avoid allocations # PS4 value -> compound_word. PS4 is scoped. self.parse_cache = {} # type: Dict[str, compound_word] # Mutate objects to save allocations self.val_indent = value.Str('') self.val_punct = value.Str('') self.val_pid_str = value.Str('') # mutated by SetProcess # Can these be global constants? I don't think we have that in ASDL yet. self.lval_indent = lvalue.Named('SHX_indent') self.lval_punct = lvalue.Named('SHX_punct') self.lval_pid_str = lvalue.Named('SHX_pid_str')
def _EmptyStrOrError(self, val, token=None): assert isinstance(val, value_t), val if val.tag == value_e.Undef: if self.exec_opts.nounset: if token is None: e_die('Undefined variable') else: name = token.val[1:] if token.val.startswith('$') else token.val e_die('Undefined variable %r', name, token=token) else: return value.Str('') else: return val
def testExportThenAssign(self): """Regression Test""" mem = _InitMem() # export U mem.SetVar(lvalue.Named('U'), None, (var_flags_e.Exported, ), scope_e.Dynamic) print(mem) # U=u mem.SetVar(lvalue.Named('U'), value.Str('u'), (), scope_e.Dynamic) print(mem) e = mem.GetExported() self.assertEqual('u', e['U'])
def _EvalIndirectArrayExpansion(self, name, index): """Expands ${!ref} when $ref has the form `name[index]`. Args: name, index: arbitrary strings Returns: value, or None if invalid """ if not match.IsValidVarName(name): return None val = self.mem.GetVar(name) if val.tag == value_e.StrArray: if index in ('@', '*'): # TODO: maybe_decay_array return value.StrArray(val.strs) try: index_num = int(index) except ValueError: return None try: return value.Str(val.strs[index_num]) except IndexError: return value.Undef() elif val.tag == value_e.AssocArray: if index in ('@', '*'): raise NotImplementedError try: return value.Str(val.d[index]) except KeyError: return value.Undef() elif val.tag == value_e.Undef: return value.Undef() elif val.tag == value_e.Str: return None else: raise AssertionError