def _MakeProcess(self, node, job_state=None, disable_errexit=False):
"""
Assume we will run the node in another process. Return a process.
"""
if node.tag == command_e.ControlFlow:
# Pipeline or subshells with control flow are invalid, e.g.:
# - break | less
# - continue | less
# - ( return )
# NOTE: This could be done at parse time too.
e_die('Invalid control flow %r in pipeline / subshell / background',
node.token.val, token=node.token)
# NOTE: If ErrExit(), we could be verbose about subprogram errors? This
# only really matters when executing 'exit 42', because the child shell
# inherits errexit and will be verbose. Other notes:
#
# - We might want errors to fit on a single line so they don't get
# interleaved.
# - We could turn the `exit` builtin into a FatalRuntimeError exception and
# get this check for "free".
thunk = process.SubProgramThunk(self, node,
disable_errexit=disable_errexit)
p = process.Process(thunk, job_state=job_state)
return p
def _GetProcessForNode(self, node):
"""
Assume we will run the node in another process. Return a process.
"""
if node.tag == command_e.SimpleCommand:
words = braces.BraceExpandWords(node.words)
argv = self.ev.EvalWordSequence(words)
more_env = self.mem.GetExported()
self._EvalEnv(node.more_env, more_env)
thunk = self._GetThunkForSimpleCommand(argv, more_env)
elif node.tag == command_e.ControlFlow:
# Pipeline or subshells with control flow are invalid, e.g.:
# - break | less
# - continue | less
# - ( return )
# NOTE: This could be done at parse time too.
e_die('Invalid control flow %r in pipeline or subshell', node.token.val,
token=node.token)
else:
thunk = process.SubProgramThunk(self, node)
redirects = self._EvalRedirects(node)
p = process.Process(thunk, fd_state=self.fd_state, redirects=redirects)
return p
def _MakeProcess(self, node, parent_pipeline=None, inherit_errexit=True):
# type: (command_t, process.Pipeline, bool) -> process.Process
"""
Assume we will run the node in another process. Return a process.
"""
UP_node = node
if node.tag_() == command_e.ControlFlow:
node = cast(command__ControlFlow, UP_node)
# Pipeline or subshells with control flow are invalid, e.g.:
# - break | less
# - continue | less
# - ( return )
# NOTE: This could be done at parse time too.
if node.token.id != Id.ControlFlow_Exit:
e_die(
'Invalid control flow %r in pipeline / subshell / background',
node.token.val,
token=node.token)
# NOTE: If ErrExit(), we could be verbose about subprogram errors? This
# only really matters when executing 'exit 42', because the child shell
# inherits errexit and will be verbose. Other notes:
#
# - We might want errors to fit on a single line so they don't get
# interleaved.
# - We could turn the `exit` builtin into a FatalRuntimeError exception and
# get this check for "free".
thunk = process.SubProgramThunk(self.cmd_ev,
node,
inherit_errexit=inherit_errexit)
p = process.Process(thunk, self.job_state)
p.Init_ParentPipeline(parent_pipeline)
return p
def _RunSimpleCommand(self, argv, fork_external, span_id, funcs=True):
"""
Args:
fork_external: for subshell ( ls / ) or ( command ls / )
"""
# This happens when you write "$@" but have no arguments.
if not argv:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = builtin.ResolveSpecial(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv, span_id)
except args.UsageError as e:
ui.usage('osh %r usage error: %s', arg0, e)
status = 2 # consistent error code for usage error
return status
# Builtins like 'true' can be redefined as functions.
if funcs:
func_node = self.funcs.get(arg0)
if func_node is not None:
# NOTE: Functions could call 'exit 42' directly, etc.
status = self._RunFunc(func_node, argv[1:])
return status
builtin_id = builtin.Resolve(arg0)
if builtin_id == builtin_e.COMMAND: # 'command ls' suppresses function lookup
n = len(argv)
if n == 1:
return 0 # 'command', like the 'if not argv' case above
# The 'command' builtin syntax is simple enough that this is 100%
# correct, not a heuristic.
elif n >= 2 and argv[1] != '-v':
return self._RunSimpleCommand(argv[1:], fork_external, span_id,
funcs=False)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv, span_id)
except args.UsageError as e:
ui.usage('osh %r usage error: %s', arg0, e)
status = 2 # consistent error code for usage error
return status
environ = self.mem.GetExported() # Include temporary variables
if fork_external:
thunk = process.ExternalThunk(argv, environ)
p = process.Process(thunk)
status = p.Run(self.waiter)
return status
# NOTE: Never returns!
process.ExecExternalProgram(argv, environ)
def RunSimpleCommand(self, argv, fork_external, span_id, funcs=True):
"""
Args:
fork_external: for subshell ( ls / ) or ( command ls / )
"""
# This happens when you write "$@" but have no arguments.
if not argv:
if self.exec_opts.strict_argv:
e_die("Command evaluated to an empty argv array",
span_id=span_id)
else:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = builtin.ResolveSpecial(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv, fork_external, span_id)
except args.UsageError as e:
ui.usage('osh %r usage error: %s', arg0, e)
status = 2 # consistent error code for usage error
return status
# Builtins like 'true' can be redefined as functions.
if funcs:
func_node = self.funcs.get(arg0)
if func_node is not None:
# NOTE: Functions could call 'exit 42' directly, etc.
status = self._RunFunc(func_node, argv[1:])
return status
builtin_id = builtin.Resolve(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv, fork_external, span_id)
except args.UsageError as e:
ui.usage('osh %r usage error: %s', arg0, e)
status = 2 # consistent error code for usage error
return status
environ = self.mem.GetExported() # Include temporary variables
if fork_external:
thunk = process.ExternalThunk(self.ext_prog, argv, environ)
p = process.Process(thunk)
status = p.Run(self.waiter)
return status
self.ext_prog.Exec(argv, environ) # NEVER RETURNS
def _RunSimpleCommand(self, argv, fork_external):
# This happens when you write "$@" but have no arguments.
if not argv:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = builtin.ResolveSpecial(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv)
except args.UsageError as e:
# TODO: Make this message more consistent?
util.usage(str(e))
status = 2 # consistent error code for usage error
return status
# Builtins like 'true' can be redefined as functions.
func_node = self.funcs.get(arg0)
if func_node is not None:
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.RunFunc(func_node, argv)
return status
builtin_id = builtin.Resolve(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv)
except args.UsageError as e:
# TODO: Make this message more consistent?
util.usage(str(e))
status = 2 # consistent error code for usage error
return status
environ = self.mem.GetExported() # Include temporary variables
if fork_external:
thunk = process.ExternalThunk(argv, environ)
p = process.Process(thunk)
status = p.Run(self.waiter)
return status
# NOTE: Never returns!
process.ExecExternalProgram(argv, environ)
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
"""
Run builtins, functions, external commands
Oil and other languages might have different, simpler rules. No special
builtins, etc.
Oil might have OIL_PATH = @( ... ) or something.
Interpreters might want to define all their own builtins.
Args:
procs: whether to look up procs.
"""
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(cmd_val.arg_spids) else runtime.NO_SPID
# This happens when you write "$@" but have no arguments.
if len(argv) == 0:
if self.exec_opts.strict_argv():
e_die("Command evaluated to an empty argv array", span_id=span_id)
else:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = consts.LookupAssignBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
# command readonly is disallowed, for technical reasons. Could relax it
# later.
self.errfmt.Print_("Can't run assignment builtin recursively",
span_id=span_id)
return 1
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
status = self.RunBuiltin(builtin_id, cmd_val)
# TODO: Enable this and fix spec test failures.
# Also update _SPECIAL_BUILTINS in osh/builtin.py.
#if status != 0:
# e_die('special builtin failed', status=status)
return status
# TODO: if shopt -s namespaces, then look up in current namespace FIRST.
#
# Then fallback on self.procs, which should be renamed self.procs?
#
# honestly there is no real chance of colllision because
# foo-bar() {} can't be accessed anyway
# functions can have hyphens, but variables can't
# Builtins like 'true' can be redefined as functions.
if call_procs:
proc_node = self.procs.get(arg0)
if proc_node is not None:
if (self.exec_opts.strict_errexit() and
self.mutable_opts.ErrExitIsDisabled()):
self.errfmt.Print_('errexit was disabled for this construct',
span_id=self.mutable_opts.ErrExitSpanId())
self.errfmt.StderrLine('')
e_die("Can't run a proc while errexit is disabled. "
"Use 'run' or wrap it in a process with $0 myproc",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(proc_node, argv[1:])
return status
# TODO:
# look up arg0 in global namespace? And see if the type is value.Obj
# And it's a proc?
# isinstance(val.obj, objects.Proc)
UP_val = self.mem.GetVar(arg0)
if mylib.PYTHON: # Not reusing CPython objects
if UP_val.tag_() == value_e.Obj:
val = cast(value__Obj, UP_val)
if isinstance(val.obj, objects.Proc):
status = self.cmd_ev.RunOilProc(val.obj, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
environ = self.mem.GetExported() # Include temporary variables
if cmd_val.block:
e_die('Unexpected block passed to external command %r', arg0,
span_id=cmd_val.block.spids[0])
# Resolve argv[0] BEFORE forking.
argv0_path = self.search_path.CachedLookup(arg0)
if argv0_path is None:
self.errfmt.Print_('%r not found' % arg0, span_id=span_id)
return 127
# Normal case: ls /
if do_fork:
thunk = process.ExternalThunk(self.ext_prog, argv0_path, cmd_val, environ)
p = process.Process(thunk, self.job_state)
status = p.Run(self.waiter)
return status
# Already forked for pipeline: ls / | wc -l
# TODO: count subshell? ( ls / ) vs. ( ls /; ls / )
self.ext_prog.Exec(argv0_path, cmd_val, environ) # NEVER RETURNS
assert False, "This line should never be reached" # makes mypy happy
def _Execute(self, node):
"""
Args:
node: of type AstNode
"""
redirects = self._EvalRedirects(node)
# TODO: Only eval argv[0] once. It can have side effects!
if node.tag == command_e.SimpleCommand:
words = braces.BraceExpandWords(node.words)
argv = self.ev.EvalWordSequence(words)
more_env = self.mem.GetExported()
self._EvalEnv(node.more_env, more_env)
thunk = self._GetThunkForSimpleCommand(argv, more_env)
# Don't waste a process if we'd launch one anyway.
if thunk.IsExternal():
p = process.Process(thunk, fd_state=self.fd_state, redirects=redirects)
status = p.Run()
else: # Internal
#log('ARGV %s', argv)
# NOTE: _EvalRedirects turns LST nodes into core/process.py nodes. And
# then we use polymorphism here. Does it make sense to use functional
# style based on the RedirType? Might be easier to read.
self.fd_state.PushFrame()
for r in redirects:
r.ApplyInParent(self.fd_state)
status = thunk.RunInParent()
restore_fd_state = thunk.ShouldRestoreFdState()
# Special case for exec 1>&2 (with no args): we permanently change the
# fd state. BUT we don't want to restore later.
# TODO: Instead of this, maybe r.ApplyPermaent(self.fd_state)?
if restore_fd_state:
self.fd_state.PopAndRestore()
else:
self.fd_state.PopAndForget()
elif node.tag == command_e.Sentence:
# TODO: Compile this away.
status = self._Execute(node.command)
elif node.tag == command_e.Pipeline:
status = self._RunPipeline(node)
elif node.tag == command_e.Subshell:
# This makes sure we don't waste a process if we'd launch one anyway.
p = self._GetProcessForNode(node.children[0])
status = p.Run()
elif node.tag == command_e.DBracket:
bool_ev = expr_eval.BoolEvaluator(self.mem, self.ev)
ok = bool_ev.Eval(node.expr)
if ok:
status = 0 if bool_ev.Result() else 1
else:
e_die('Error evaluating boolean: %s' % bool_ev.Error())
elif node.tag == command_e.DParen:
arith_ev = expr_eval.ArithEvaluator(self.mem, self.ev)
ok = arith_ev.Eval(node.child)
if ok:
i = arith_ev.Result()
# Negate the value: non-zero in arithmetic is true, which is zero in
# shell land
status = 0 if i != 0 else 1
else:
e_die('Error evaluating (( )): %s' % arith_ev.Error())
elif node.tag == command_e.Assignment:
pairs = []
for pair in node.pairs:
if pair.rhs:
# RHS can be a string or array.
val = self.ev.EvalWordToAny(pair.rhs)
assert isinstance(val, runtime.value), val
else:
# 'local x' is equivalent to local x=""
val = runtime.Str('')
pairs.append((pair.lhs, val))
if node.keyword == Id.Assign_Local:
self.mem.SetLocals(pairs)
else:
# NOTE: could be readonly/export/etc.
self.mem.SetLocalsOrGlobals(pairs)
# TODO: This should be eval of RHS, unlike bash!
status = 0
elif node.tag == command_e.ControlFlow:
if node.arg_word: # Evaluate the argument
_, val = self.ev.EvalWordToString(node.arg_word)
assert val.tag == value_e.Str
arg = int(val.s) # They all take integers
else:
arg = 0 # return 0, break 0 levels, etc.
raise _ControlFlow(node.token, arg)
# The only difference between these two is that CommandList has no
# redirects. We already took care of that above.
elif node.tag in (command_e.CommandList, command_e.BraceGroup):
self.fd_state.PushFrame()
for r in redirects:
r.ApplyInParent(self.fd_state)
status = 0 # for empty list
for child in node.children:
status = self._Execute(child) # last status wins
self.fd_state.PopAndRestore()
elif node.tag == command_e.AndOr:
#print(node.children)
left, right = node.children
status = self._Execute(left)
if node.op_id == Id.Op_DPipe:
if status != 0:
status = self._Execute(right)
elif node.op_id == Id.Op_DAmp:
if status == 0:
status = self._Execute(right)
else:
raise AssertionError
elif node.tag in (command_e.While, command_e.Until):
# TODO: Compile this out?
if node.tag == command_e.While:
_DonePredicate = lambda status: status != 0
else:
_DonePredicate = lambda status: status == 0
while True:
status = self._Execute(node.cond)
done = status != 0
if _DonePredicate(status):
break
try:
status = self._Execute(node.body) # last one wins
except _ControlFlow as e:
if e.IsBreak():
status = 0
break
elif e.IsContinue():
status = 0
continue
else: # return needs to pop up more
raise
elif node.tag == command_e.ForEach:
iter_name = node.iter_name
if node.do_arg_iter:
iter_list = self.mem.GetArgv()
else:
words = braces.BraceExpandWords(node.iter_words)
iter_list = self.ev.EvalWordSequence(words)
# We need word splitting and so forth
# NOTE: This expands globs too. TODO: We should pass in a Globber()
# object.
status = 0 # in case we don't loop
for x in iter_list:
#log('> ForEach setting %r', x)
self.mem.SetLocal(iter_name, runtime.Str(x))
#log('<')
try:
status = self._Execute(node.body) # last one wins
except _ControlFlow as e:
if e.IsBreak():
status = 0
break
elif e.IsContinue():
status = 0
continue
else: # return needs to pop up more
raise
elif node.tag == command_e.ForExpr:
raise NotImplementedError(node.tag)
elif node.tag == command_e.DoGroup:
# Delegate to command list
# TODO: This should be compiled out!
status = self._Execute(node.child)
elif node.tag == command_e.FuncDef:
self.funcs[node.name] = node
status = 0
elif node.tag == command_e.If:
done = False
for arm in node.arms:
status = self._Execute(arm.cond)
if status == 0:
status = self._Execute(arm.action)
done = True
break
# TODO: The compiler should flatten this
if not done and node.else_action is not None:
status = self._Execute(node.else_action)
elif node.tag == command_e.NoOp:
status = 0 # make it true
elif node.tag == command_e.Case:
ok, val = self.ev.EvalWordToString(node.to_match)
assert ok
to_match = val.s
status = 0 # If there are no arms, it should be zero?
done = False
for arm in node.arms:
for pat_word in arm.pat_list:
# NOTE: Is it OK that we're evaluating these as we go?
ok, pat_val = self.ev.EvalWordToString(pat_word, do_fnmatch=True)
assert ok
#log('Matching word %r against pattern %r', to_match, pat_val.s)
if libc.fnmatch(pat_val.s, to_match):
status = self._Execute(arm.action)
done = True # TODO: Parse ;;& and for fallthrough and such?
if done:
break
elif node.tag == command_e.TimeBlock:
# TODO:
# - When do we need RUSAGE_CHILDREN?
# - Respect TIMEFORMAT environment variable.
# "If this variable is not set, Bash acts as if it had the value"
# $'\nreal\t%3lR\nuser\t%3lU\nsys\t%3lS'
# "A trailing newline is added when the format string is displayed."
start_t = time.time() # calls gettimeofday() under the hood
start_u = resource.getrusage(resource.RUSAGE_SELF)
status = self._Execute(node.pipeline)
end_t = time.time()
end_u = resource.getrusage(resource.RUSAGE_SELF)
real = end_t - start_t
user = end_u.ru_utime - start_u.ru_utime
sys_ = end_u.ru_stime - start_u.ru_stime
print('real\t%.3f' % real, file=sys.stderr)
print('user\t%.3f' % user, file=sys.stderr)
print('sys\t%.3f' % sys_, file=sys.stderr)
else:
raise AssertionError(node.tag)
if self.exec_opts.errexit and status != 0:
if node.tag == command_e.SimpleCommand:
# TODO: Add context
e_die('%r command exited with status %d (%s)', argv[0], status, node.words[0])
else:
e_die('%r command exited with status %d', node.__class__.__name__, status)
# TODO: Is this the right place to put it? Does it need a stack for
# function calls?
self.mem.last_status = status
return status
def RunSimpleCommand(self, arg_vec, fork_external, funcs=True):
"""Public interface to run a simple command (excluding assignment)
Args:
fork_external: for subshell ( ls / ) or ( command ls / )
"""
argv = arg_vec.strs
if arg_vec.spids:
span_id = arg_vec.spids[0]
else:
span_id = const.NO_INTEGER
# This happens when you write "$@" but have no arguments.
if not argv:
if self.exec_opts.strict_argv:
e_die("Command evaluated to an empty argv array",
span_id=span_id)
else:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = builtin.ResolveAssign(arg0)
if builtin_id != builtin_e.NONE:
# command readonly is disallowed, for technical reasons. Could relax it
# later.
self.errfmt.Print("Can't run assignment builtin recursively",
span_id=span_id)
return 1
builtin_id = builtin.ResolveSpecial(arg0)
if builtin_id != builtin_e.NONE:
status = self._RunBuiltin(builtin_id, arg_vec, fork_external)
# TODO: Enable this and fix spec test failures.
# Also update _SPECIAL_BUILTINS in osh/builtin.py.
#if status != 0:
# e_die('special builtin failed', status=status)
return status
# Builtins like 'true' can be redefined as functions.
if funcs:
func_node = self.funcs.get(arg0)
if func_node is not None:
# NOTE: Functions could call 'exit 42' directly, etc.
status = self._RunFunc(func_node, argv[1:])
return status
builtin_id = builtin.Resolve(arg0)
if builtin_id != builtin_e.NONE:
return self._RunBuiltin(builtin_id, arg_vec, fork_external)
environ = self.mem.GetExported() # Include temporary variables
# Resolve argv[0] BEFORE forking.
argv0_path = self.search_path.CachedLookup(argv[0])
if argv0_path is None:
self.errfmt.Print('%r not found', argv[0], span_id=span_id)
return 127
if fork_external:
thunk = process.ExternalThunk(self.ext_prog, argv0_path, arg_vec, environ)
p = process.Process(thunk, self.job_state)
status = p.Run(self.waiter)
return status
self.ext_prog.Exec(argv0_path, arg_vec, environ) # NEVER RETURNS
