def _MakeProcess(self, node, job_state=None, disable_errexit=False): """ Assume we will run the node in another process. Return a process. """ if node.tag == command_e.ControlFlow: # Pipeline or subshells with control flow are invalid, e.g.: # - break | less # - continue | less # - ( return ) # NOTE: This could be done at parse time too. e_die('Invalid control flow %r in pipeline / subshell / background', node.token.val, token=node.token) # NOTE: If ErrExit(), we could be verbose about subprogram errors? This # only really matters when executing 'exit 42', because the child shell # inherits errexit and will be verbose. Other notes: # # - We might want errors to fit on a single line so they don't get # interleaved. # - We could turn the `exit` builtin into a FatalRuntimeError exception and # get this check for "free". thunk = process.SubProgramThunk(self, node, disable_errexit=disable_errexit) p = process.Process(thunk, job_state=job_state) return p
def _GetProcessForNode(self, node): """ Assume we will run the node in another process. Return a process. """ if node.tag == command_e.SimpleCommand: words = braces.BraceExpandWords(node.words) argv = self.ev.EvalWordSequence(words) more_env = self.mem.GetExported() self._EvalEnv(node.more_env, more_env) thunk = self._GetThunkForSimpleCommand(argv, more_env) elif node.tag == command_e.ControlFlow: # Pipeline or subshells with control flow are invalid, e.g.: # - break | less # - continue | less # - ( return ) # NOTE: This could be done at parse time too. e_die('Invalid control flow %r in pipeline or subshell', node.token.val, token=node.token) else: thunk = process.SubProgramThunk(self, node) redirects = self._EvalRedirects(node) p = process.Process(thunk, fd_state=self.fd_state, redirects=redirects) return p
def _MakeProcess(self, node, parent_pipeline=None, inherit_errexit=True): # type: (command_t, process.Pipeline, bool) -> process.Process """ Assume we will run the node in another process. Return a process. """ UP_node = node if node.tag_() == command_e.ControlFlow: node = cast(command__ControlFlow, UP_node) # Pipeline or subshells with control flow are invalid, e.g.: # - break | less # - continue | less # - ( return ) # NOTE: This could be done at parse time too. if node.token.id != Id.ControlFlow_Exit: e_die( 'Invalid control flow %r in pipeline / subshell / background', node.token.val, token=node.token) # NOTE: If ErrExit(), we could be verbose about subprogram errors? This # only really matters when executing 'exit 42', because the child shell # inherits errexit and will be verbose. Other notes: # # - We might want errors to fit on a single line so they don't get # interleaved. # - We could turn the `exit` builtin into a FatalRuntimeError exception and # get this check for "free". thunk = process.SubProgramThunk(self.cmd_ev, node, inherit_errexit=inherit_errexit) p = process.Process(thunk, self.job_state) p.Init_ParentPipeline(parent_pipeline) return p
def _RunSimpleCommand(self, argv, fork_external, span_id, funcs=True): """ Args: fork_external: for subshell ( ls / ) or ( command ls / ) """ # This happens when you write "$@" but have no arguments. if not argv: return 0 # status 0, or skip it? arg0 = argv[0] builtin_id = builtin.ResolveSpecial(arg0) if builtin_id != builtin_e.NONE: try: status = self._RunBuiltin(builtin_id, argv, span_id) except args.UsageError as e: ui.usage('osh %r usage error: %s', arg0, e) status = 2 # consistent error code for usage error return status # Builtins like 'true' can be redefined as functions. if funcs: func_node = self.funcs.get(arg0) if func_node is not None: # NOTE: Functions could call 'exit 42' directly, etc. status = self._RunFunc(func_node, argv[1:]) return status builtin_id = builtin.Resolve(arg0) if builtin_id == builtin_e.COMMAND: # 'command ls' suppresses function lookup n = len(argv) if n == 1: return 0 # 'command', like the 'if not argv' case above # The 'command' builtin syntax is simple enough that this is 100% # correct, not a heuristic. elif n >= 2 and argv[1] != '-v': return self._RunSimpleCommand(argv[1:], fork_external, span_id, funcs=False) if builtin_id != builtin_e.NONE: try: status = self._RunBuiltin(builtin_id, argv, span_id) except args.UsageError as e: ui.usage('osh %r usage error: %s', arg0, e) status = 2 # consistent error code for usage error return status environ = self.mem.GetExported() # Include temporary variables if fork_external: thunk = process.ExternalThunk(argv, environ) p = process.Process(thunk) status = p.Run(self.waiter) return status # NOTE: Never returns! process.ExecExternalProgram(argv, environ)
def RunSimpleCommand(self, argv, fork_external, span_id, funcs=True): """ Args: fork_external: for subshell ( ls / ) or ( command ls / ) """ # This happens when you write "$@" but have no arguments. if not argv: if self.exec_opts.strict_argv: e_die("Command evaluated to an empty argv array", span_id=span_id) else: return 0 # status 0, or skip it? arg0 = argv[0] builtin_id = builtin.ResolveSpecial(arg0) if builtin_id != builtin_e.NONE: try: status = self._RunBuiltin(builtin_id, argv, fork_external, span_id) except args.UsageError as e: ui.usage('osh %r usage error: %s', arg0, e) status = 2 # consistent error code for usage error return status # Builtins like 'true' can be redefined as functions. if funcs: func_node = self.funcs.get(arg0) if func_node is not None: # NOTE: Functions could call 'exit 42' directly, etc. status = self._RunFunc(func_node, argv[1:]) return status builtin_id = builtin.Resolve(arg0) if builtin_id != builtin_e.NONE: try: status = self._RunBuiltin(builtin_id, argv, fork_external, span_id) except args.UsageError as e: ui.usage('osh %r usage error: %s', arg0, e) status = 2 # consistent error code for usage error return status environ = self.mem.GetExported() # Include temporary variables if fork_external: thunk = process.ExternalThunk(self.ext_prog, argv, environ) p = process.Process(thunk) status = p.Run(self.waiter) return status self.ext_prog.Exec(argv, environ) # NEVER RETURNS
def _RunSimpleCommand(self, argv, fork_external): # This happens when you write "$@" but have no arguments. if not argv: return 0 # status 0, or skip it? arg0 = argv[0] builtin_id = builtin.ResolveSpecial(arg0) if builtin_id != builtin_e.NONE: try: status = self._RunBuiltin(builtin_id, argv) except args.UsageError as e: # TODO: Make this message more consistent? util.usage(str(e)) status = 2 # consistent error code for usage error return status # Builtins like 'true' can be redefined as functions. func_node = self.funcs.get(arg0) if func_node is not None: # NOTE: Functions could call 'exit 42' directly, etc. status = self.RunFunc(func_node, argv) return status builtin_id = builtin.Resolve(arg0) if builtin_id != builtin_e.NONE: try: status = self._RunBuiltin(builtin_id, argv) except args.UsageError as e: # TODO: Make this message more consistent? util.usage(str(e)) status = 2 # consistent error code for usage error return status environ = self.mem.GetExported() # Include temporary variables if fork_external: thunk = process.ExternalThunk(argv, environ) p = process.Process(thunk) status = p.Run(self.waiter) return status # NOTE: Never returns! process.ExecExternalProgram(argv, environ)
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True): # type: (cmd_value__Argv, bool, bool) -> int """ Run builtins, functions, external commands Oil and other languages might have different, simpler rules. No special builtins, etc. Oil might have OIL_PATH = @( ... ) or something. Interpreters might want to define all their own builtins. Args: procs: whether to look up procs. """ argv = cmd_val.argv span_id = cmd_val.arg_spids[0] if len(cmd_val.arg_spids) else runtime.NO_SPID # This happens when you write "$@" but have no arguments. if len(argv) == 0: if self.exec_opts.strict_argv(): e_die("Command evaluated to an empty argv array", span_id=span_id) else: return 0 # status 0, or skip it? arg0 = argv[0] builtin_id = consts.LookupAssignBuiltin(arg0) if builtin_id != consts.NO_INDEX: # command readonly is disallowed, for technical reasons. Could relax it # later. self.errfmt.Print_("Can't run assignment builtin recursively", span_id=span_id) return 1 builtin_id = consts.LookupSpecialBuiltin(arg0) if builtin_id != consts.NO_INDEX: status = self.RunBuiltin(builtin_id, cmd_val) # TODO: Enable this and fix spec test failures. # Also update _SPECIAL_BUILTINS in osh/builtin.py. #if status != 0: # e_die('special builtin failed', status=status) return status # TODO: if shopt -s namespaces, then look up in current namespace FIRST. # # Then fallback on self.procs, which should be renamed self.procs? # # honestly there is no real chance of colllision because # foo-bar() {} can't be accessed anyway # functions can have hyphens, but variables can't # Builtins like 'true' can be redefined as functions. if call_procs: proc_node = self.procs.get(arg0) if proc_node is not None: if (self.exec_opts.strict_errexit() and self.mutable_opts.ErrExitIsDisabled()): self.errfmt.Print_('errexit was disabled for this construct', span_id=self.mutable_opts.ErrExitSpanId()) self.errfmt.StderrLine('') e_die("Can't run a proc while errexit is disabled. " "Use 'run' or wrap it in a process with $0 myproc", span_id=span_id) # NOTE: Functions could call 'exit 42' directly, etc. status = self.cmd_ev.RunProc(proc_node, argv[1:]) return status # TODO: # look up arg0 in global namespace? And see if the type is value.Obj # And it's a proc? # isinstance(val.obj, objects.Proc) UP_val = self.mem.GetVar(arg0) if mylib.PYTHON: # Not reusing CPython objects if UP_val.tag_() == value_e.Obj: val = cast(value__Obj, UP_val) if isinstance(val.obj, objects.Proc): status = self.cmd_ev.RunOilProc(val.obj, argv[1:]) return status builtin_id = consts.LookupNormalBuiltin(arg0) if builtin_id != consts.NO_INDEX: return self.RunBuiltin(builtin_id, cmd_val) environ = self.mem.GetExported() # Include temporary variables if cmd_val.block: e_die('Unexpected block passed to external command %r', arg0, span_id=cmd_val.block.spids[0]) # Resolve argv[0] BEFORE forking. argv0_path = self.search_path.CachedLookup(arg0) if argv0_path is None: self.errfmt.Print_('%r not found' % arg0, span_id=span_id) return 127 # Normal case: ls / if do_fork: thunk = process.ExternalThunk(self.ext_prog, argv0_path, cmd_val, environ) p = process.Process(thunk, self.job_state) status = p.Run(self.waiter) return status # Already forked for pipeline: ls / | wc -l # TODO: count subshell? ( ls / ) vs. ( ls /; ls / ) self.ext_prog.Exec(argv0_path, cmd_val, environ) # NEVER RETURNS assert False, "This line should never be reached" # makes mypy happy
def _Execute(self, node): """ Args: node: of type AstNode """ redirects = self._EvalRedirects(node) # TODO: Only eval argv[0] once. It can have side effects! if node.tag == command_e.SimpleCommand: words = braces.BraceExpandWords(node.words) argv = self.ev.EvalWordSequence(words) more_env = self.mem.GetExported() self._EvalEnv(node.more_env, more_env) thunk = self._GetThunkForSimpleCommand(argv, more_env) # Don't waste a process if we'd launch one anyway. if thunk.IsExternal(): p = process.Process(thunk, fd_state=self.fd_state, redirects=redirects) status = p.Run() else: # Internal #log('ARGV %s', argv) # NOTE: _EvalRedirects turns LST nodes into core/process.py nodes. And # then we use polymorphism here. Does it make sense to use functional # style based on the RedirType? Might be easier to read. self.fd_state.PushFrame() for r in redirects: r.ApplyInParent(self.fd_state) status = thunk.RunInParent() restore_fd_state = thunk.ShouldRestoreFdState() # Special case for exec 1>&2 (with no args): we permanently change the # fd state. BUT we don't want to restore later. # TODO: Instead of this, maybe r.ApplyPermaent(self.fd_state)? if restore_fd_state: self.fd_state.PopAndRestore() else: self.fd_state.PopAndForget() elif node.tag == command_e.Sentence: # TODO: Compile this away. status = self._Execute(node.command) elif node.tag == command_e.Pipeline: status = self._RunPipeline(node) elif node.tag == command_e.Subshell: # This makes sure we don't waste a process if we'd launch one anyway. p = self._GetProcessForNode(node.children[0]) status = p.Run() elif node.tag == command_e.DBracket: bool_ev = expr_eval.BoolEvaluator(self.mem, self.ev) ok = bool_ev.Eval(node.expr) if ok: status = 0 if bool_ev.Result() else 1 else: e_die('Error evaluating boolean: %s' % bool_ev.Error()) elif node.tag == command_e.DParen: arith_ev = expr_eval.ArithEvaluator(self.mem, self.ev) ok = arith_ev.Eval(node.child) if ok: i = arith_ev.Result() # Negate the value: non-zero in arithmetic is true, which is zero in # shell land status = 0 if i != 0 else 1 else: e_die('Error evaluating (( )): %s' % arith_ev.Error()) elif node.tag == command_e.Assignment: pairs = [] for pair in node.pairs: if pair.rhs: # RHS can be a string or array. val = self.ev.EvalWordToAny(pair.rhs) assert isinstance(val, runtime.value), val else: # 'local x' is equivalent to local x="" val = runtime.Str('') pairs.append((pair.lhs, val)) if node.keyword == Id.Assign_Local: self.mem.SetLocals(pairs) else: # NOTE: could be readonly/export/etc. self.mem.SetLocalsOrGlobals(pairs) # TODO: This should be eval of RHS, unlike bash! status = 0 elif node.tag == command_e.ControlFlow: if node.arg_word: # Evaluate the argument _, val = self.ev.EvalWordToString(node.arg_word) assert val.tag == value_e.Str arg = int(val.s) # They all take integers else: arg = 0 # return 0, break 0 levels, etc. raise _ControlFlow(node.token, arg) # The only difference between these two is that CommandList has no # redirects. We already took care of that above. elif node.tag in (command_e.CommandList, command_e.BraceGroup): self.fd_state.PushFrame() for r in redirects: r.ApplyInParent(self.fd_state) status = 0 # for empty list for child in node.children: status = self._Execute(child) # last status wins self.fd_state.PopAndRestore() elif node.tag == command_e.AndOr: #print(node.children) left, right = node.children status = self._Execute(left) if node.op_id == Id.Op_DPipe: if status != 0: status = self._Execute(right) elif node.op_id == Id.Op_DAmp: if status == 0: status = self._Execute(right) else: raise AssertionError elif node.tag in (command_e.While, command_e.Until): # TODO: Compile this out? if node.tag == command_e.While: _DonePredicate = lambda status: status != 0 else: _DonePredicate = lambda status: status == 0 while True: status = self._Execute(node.cond) done = status != 0 if _DonePredicate(status): break try: status = self._Execute(node.body) # last one wins except _ControlFlow as e: if e.IsBreak(): status = 0 break elif e.IsContinue(): status = 0 continue else: # return needs to pop up more raise elif node.tag == command_e.ForEach: iter_name = node.iter_name if node.do_arg_iter: iter_list = self.mem.GetArgv() else: words = braces.BraceExpandWords(node.iter_words) iter_list = self.ev.EvalWordSequence(words) # We need word splitting and so forth # NOTE: This expands globs too. TODO: We should pass in a Globber() # object. status = 0 # in case we don't loop for x in iter_list: #log('> ForEach setting %r', x) self.mem.SetLocal(iter_name, runtime.Str(x)) #log('<') try: status = self._Execute(node.body) # last one wins except _ControlFlow as e: if e.IsBreak(): status = 0 break elif e.IsContinue(): status = 0 continue else: # return needs to pop up more raise elif node.tag == command_e.ForExpr: raise NotImplementedError(node.tag) elif node.tag == command_e.DoGroup: # Delegate to command list # TODO: This should be compiled out! status = self._Execute(node.child) elif node.tag == command_e.FuncDef: self.funcs[node.name] = node status = 0 elif node.tag == command_e.If: done = False for arm in node.arms: status = self._Execute(arm.cond) if status == 0: status = self._Execute(arm.action) done = True break # TODO: The compiler should flatten this if not done and node.else_action is not None: status = self._Execute(node.else_action) elif node.tag == command_e.NoOp: status = 0 # make it true elif node.tag == command_e.Case: ok, val = self.ev.EvalWordToString(node.to_match) assert ok to_match = val.s status = 0 # If there are no arms, it should be zero? done = False for arm in node.arms: for pat_word in arm.pat_list: # NOTE: Is it OK that we're evaluating these as we go? ok, pat_val = self.ev.EvalWordToString(pat_word, do_fnmatch=True) assert ok #log('Matching word %r against pattern %r', to_match, pat_val.s) if libc.fnmatch(pat_val.s, to_match): status = self._Execute(arm.action) done = True # TODO: Parse ;;& and for fallthrough and such? if done: break elif node.tag == command_e.TimeBlock: # TODO: # - When do we need RUSAGE_CHILDREN? # - Respect TIMEFORMAT environment variable. # "If this variable is not set, Bash acts as if it had the value" # $'\nreal\t%3lR\nuser\t%3lU\nsys\t%3lS' # "A trailing newline is added when the format string is displayed." start_t = time.time() # calls gettimeofday() under the hood start_u = resource.getrusage(resource.RUSAGE_SELF) status = self._Execute(node.pipeline) end_t = time.time() end_u = resource.getrusage(resource.RUSAGE_SELF) real = end_t - start_t user = end_u.ru_utime - start_u.ru_utime sys_ = end_u.ru_stime - start_u.ru_stime print('real\t%.3f' % real, file=sys.stderr) print('user\t%.3f' % user, file=sys.stderr) print('sys\t%.3f' % sys_, file=sys.stderr) else: raise AssertionError(node.tag) if self.exec_opts.errexit and status != 0: if node.tag == command_e.SimpleCommand: # TODO: Add context e_die('%r command exited with status %d (%s)', argv[0], status, node.words[0]) else: e_die('%r command exited with status %d', node.__class__.__name__, status) # TODO: Is this the right place to put it? Does it need a stack for # function calls? self.mem.last_status = status return status
def RunSimpleCommand(self, arg_vec, fork_external, funcs=True): """Public interface to run a simple command (excluding assignment) Args: fork_external: for subshell ( ls / ) or ( command ls / ) """ argv = arg_vec.strs if arg_vec.spids: span_id = arg_vec.spids[0] else: span_id = const.NO_INTEGER # This happens when you write "$@" but have no arguments. if not argv: if self.exec_opts.strict_argv: e_die("Command evaluated to an empty argv array", span_id=span_id) else: return 0 # status 0, or skip it? arg0 = argv[0] builtin_id = builtin.ResolveAssign(arg0) if builtin_id != builtin_e.NONE: # command readonly is disallowed, for technical reasons. Could relax it # later. self.errfmt.Print("Can't run assignment builtin recursively", span_id=span_id) return 1 builtin_id = builtin.ResolveSpecial(arg0) if builtin_id != builtin_e.NONE: status = self._RunBuiltin(builtin_id, arg_vec, fork_external) # TODO: Enable this and fix spec test failures. # Also update _SPECIAL_BUILTINS in osh/builtin.py. #if status != 0: # e_die('special builtin failed', status=status) return status # Builtins like 'true' can be redefined as functions. if funcs: func_node = self.funcs.get(arg0) if func_node is not None: # NOTE: Functions could call 'exit 42' directly, etc. status = self._RunFunc(func_node, argv[1:]) return status builtin_id = builtin.Resolve(arg0) if builtin_id != builtin_e.NONE: return self._RunBuiltin(builtin_id, arg_vec, fork_external) environ = self.mem.GetExported() # Include temporary variables # Resolve argv[0] BEFORE forking. argv0_path = self.search_path.CachedLookup(argv[0]) if argv0_path is None: self.errfmt.Print('%r not found', argv[0], span_id=span_id) return 127 if fork_external: thunk = process.ExternalThunk(self.ext_prog, argv0_path, arg_vec, environ) p = process.Process(thunk, self.job_state) status = p.Run(self.waiter) return status self.ext_prog.Exec(argv0_path, arg_vec, environ) # NEVER RETURNS