def testPipeline2(self):
Banner('ls | cut -d . -f 1 | head')
p = process.Pipeline()
p.Add(Process(process.ExternalThunk(['ls'])))
p.Add(Process(process.ExternalThunk(['cut', '-d', '.', '-f', '1'])))
p.Add(Process(process.ExternalThunk(['head'])))
print(p.Run())
ex = InitExecutor()
# Simulating subshell for each command
w1 = ast.CompoundWord()
w1.parts.append(ast.LiteralPart(ast.token(Id.Lit_Chars, 'ls')))
node1 = ast.SimpleCommand()
node1.words = [w1]
w2 = ast.CompoundWord()
w2.parts.append(ast.LiteralPart(ast.token(Id.Lit_Chars, 'head')))
node2 = ast.SimpleCommand()
node2.words = [w2]
w3 = ast.CompoundWord()
w3.parts.append(ast.LiteralPart(ast.token(Id.Lit_Chars, 'sort')))
w4 = ast.CompoundWord()
w4.parts.append(ast.LiteralPart(ast.token(Id.Lit_Chars, '--reverse')))
node3 = ast.SimpleCommand()
node3.words = [w3, w4]
p = process.Pipeline()
p.Add(Process(process.SubProgramThunk(ex, node1)))
p.Add(Process(process.SubProgramThunk(ex, node2)))
p.Add(Process(process.SubProgramThunk(ex, node3)))
print(p.Run())
def testPipeline(self):
print('BEFORE', os.listdir('/dev/fd'))
p = process.Pipeline()
p.Add(Process(process.ExternalThunk(['ls'])))
p.Add(Process(process.ExternalThunk(['cut', '-d', '.', '-f', '2'])))
p.Add(Process(process.ExternalThunk(['sort'])))
p.Add(Process(process.ExternalThunk(['uniq', '-c'])))
p.Run()
print('AFTER', os.listdir('/dev/fd'))
def _GetThunkForSimpleCommand(self, argv, more_env):
"""
Given a node, resolve the first command word, and return a thunk. The
thunk may be run in either the parent shell process or a child process.
Args:
argv: evaluated arguments
more_env: evaluated environment
Returns:
is_external: If the node MUST be run in an external process.
thunk: thunk to run
True, ExternalThunk() instance, or
False, argv if the thing to run isn't representable by an external
command.
argv can be None too.
For deciding whether we need a subshell.
"""
if not argv:
return process.NoOpThunk()
# TODO: respect the special builtin order too
builtin_id = self.builtins.Resolve(argv[0])
if builtin_id != EBuiltin.NONE:
return process.BuiltinThunk(self, builtin_id, argv)
func_node = self.funcs.get(argv[0])
if func_node is not None:
return process.FuncThunk(self, func_node, argv)
return process.ExternalThunk(argv, more_env)
def _RunSimpleCommand(self, argv, fork_external, span_id, funcs=True):
"""
Args:
fork_external: for subshell ( ls / ) or ( command ls / )
"""
# This happens when you write "$@" but have no arguments.
if not argv:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = builtin.ResolveSpecial(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv, span_id)
except args.UsageError as e:
ui.usage('osh %r usage error: %s', arg0, e)
status = 2 # consistent error code for usage error
return status
# Builtins like 'true' can be redefined as functions.
if funcs:
func_node = self.funcs.get(arg0)
if func_node is not None:
# NOTE: Functions could call 'exit 42' directly, etc.
status = self._RunFunc(func_node, argv[1:])
return status
builtin_id = builtin.Resolve(arg0)
if builtin_id == builtin_e.COMMAND: # 'command ls' suppresses function lookup
n = len(argv)
if n == 1:
return 0 # 'command', like the 'if not argv' case above
# The 'command' builtin syntax is simple enough that this is 100%
# correct, not a heuristic.
elif n >= 2 and argv[1] != '-v':
return self._RunSimpleCommand(argv[1:], fork_external, span_id,
funcs=False)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv, span_id)
except args.UsageError as e:
ui.usage('osh %r usage error: %s', arg0, e)
status = 2 # consistent error code for usage error
return status
environ = self.mem.GetExported() # Include temporary variables
if fork_external:
thunk = process.ExternalThunk(argv, environ)
p = process.Process(thunk)
status = p.Run(self.waiter)
return status
# NOTE: Never returns!
process.ExecExternalProgram(argv, environ)
def _Exec(self, argv):
# Either execute command with redirects, or apply redirects in this shell.
# NOTE: Redirects were processed earlier.
argv = argv[1:]
if argv:
thunk = process.ExternalThunk(argv)
thunk.RunInParent() # never returns
else:
return 0
def RunSimpleCommand(self, argv, fork_external, span_id, funcs=True):
"""
Args:
fork_external: for subshell ( ls / ) or ( command ls / )
"""
# This happens when you write "$@" but have no arguments.
if not argv:
if self.exec_opts.strict_argv:
e_die("Command evaluated to an empty argv array",
span_id=span_id)
else:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = builtin.ResolveSpecial(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv, fork_external, span_id)
except args.UsageError as e:
ui.usage('osh %r usage error: %s', arg0, e)
status = 2 # consistent error code for usage error
return status
# Builtins like 'true' can be redefined as functions.
if funcs:
func_node = self.funcs.get(arg0)
if func_node is not None:
# NOTE: Functions could call 'exit 42' directly, etc.
status = self._RunFunc(func_node, argv[1:])
return status
builtin_id = builtin.Resolve(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv, fork_external, span_id)
except args.UsageError as e:
ui.usage('osh %r usage error: %s', arg0, e)
status = 2 # consistent error code for usage error
return status
environ = self.mem.GetExported() # Include temporary variables
if fork_external:
thunk = process.ExternalThunk(self.ext_prog, argv, environ)
p = process.Process(thunk)
status = p.Run(self.waiter)
return status
self.ext_prog.Exec(argv, environ) # NEVER RETURNS
def testFilenameRedirect(self):
print('BEFORE', os.listdir('/dev/fd'))
fd_state = process.FdState()
r = process.DescriptorRedirect(Id.Redir_GreatAnd, 1, 2) # 1>&2
r.ApplyInParent(fd_state)
sys.stdout.write('write stdout to stderr\n')
#os.write(sys.stdout.fileno(), 'write stdout to stderr\n')
sys.stdout.flush() # flush required
fd_state.PopAndRestore()
fd_state.PushFrame()
sys.stdout.write('after restoring stdout\n')
sys.stdout.flush() # flush required
r1 = process.FilenameRedirect(Id.Redir_Great, 1, '_tmp/desc3-out.txt')
r2 = process.FilenameRedirect(Id.Redir_Great, 2, '_tmp/desc3-err.txt')
r1.ApplyInParent(fd_state)
r2.ApplyInParent(fd_state)
sys.stdout.write('stdout to file\n')
sys.stdout.flush() # flush required
sys.stderr.write('stderr to file\n')
sys.stderr.flush() # flush required
fd_state.PopAndRestore()
r1 = process.FilenameRedirect(Id.Redir_Great, 1, '_tmp/ls-out.txt')
r2 = process.FilenameRedirect(Id.Redir_Great, 2, '_tmp/ls-err.txt')
p = Process(
process.ExternalThunk(['ls', '/error', '.']), fd_state=fd_state,
redirects=[r1, r2])
# Bad File Descriptor
ok = fd_state.SaveAndDup(5, 1) # 1>&5
if ok:
sys.stdout.write('write stdout to stderr\n')
sys.stdout.flush() # flush required
fd_state.PopAndRestore()
else:
print('SaveAndDup FAILED')
print('FDs AFTER', os.listdir('/dev/fd'))
def _RunSimpleCommand(self, argv, fork_external):
# This happens when you write "$@" but have no arguments.
if not argv:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = builtin.ResolveSpecial(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv)
except args.UsageError as e:
# TODO: Make this message more consistent?
util.usage(str(e))
status = 2 # consistent error code for usage error
return status
# Builtins like 'true' can be redefined as functions.
func_node = self.funcs.get(arg0)
if func_node is not None:
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.RunFunc(func_node, argv)
return status
builtin_id = builtin.Resolve(arg0)
if builtin_id != builtin_e.NONE:
try:
status = self._RunBuiltin(builtin_id, argv)
except args.UsageError as e:
# TODO: Make this message more consistent?
util.usage(str(e))
status = 2 # consistent error code for usage error
return status
environ = self.mem.GetExported() # Include temporary variables
if fork_external:
thunk = process.ExternalThunk(argv, environ)
p = process.Process(thunk)
status = p.Run(self.waiter)
return status
# NOTE: Never returns!
process.ExecExternalProgram(argv, environ)
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
"""
Run builtins, functions, external commands
Oil and other languages might have different, simpler rules. No special
builtins, etc.
Oil might have OIL_PATH = @( ... ) or something.
Interpreters might want to define all their own builtins.
Args:
procs: whether to look up procs.
"""
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(cmd_val.arg_spids) else runtime.NO_SPID
# This happens when you write "$@" but have no arguments.
if len(argv) == 0:
if self.exec_opts.strict_argv():
e_die("Command evaluated to an empty argv array", span_id=span_id)
else:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = consts.LookupAssignBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
# command readonly is disallowed, for technical reasons. Could relax it
# later.
self.errfmt.Print_("Can't run assignment builtin recursively",
span_id=span_id)
return 1
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
status = self.RunBuiltin(builtin_id, cmd_val)
# TODO: Enable this and fix spec test failures.
# Also update _SPECIAL_BUILTINS in osh/builtin.py.
#if status != 0:
# e_die('special builtin failed', status=status)
return status
# TODO: if shopt -s namespaces, then look up in current namespace FIRST.
#
# Then fallback on self.procs, which should be renamed self.procs?
#
# honestly there is no real chance of colllision because
# foo-bar() {} can't be accessed anyway
# functions can have hyphens, but variables can't
# Builtins like 'true' can be redefined as functions.
if call_procs:
proc_node = self.procs.get(arg0)
if proc_node is not None:
if (self.exec_opts.strict_errexit() and
self.mutable_opts.ErrExitIsDisabled()):
self.errfmt.Print_('errexit was disabled for this construct',
span_id=self.mutable_opts.ErrExitSpanId())
self.errfmt.StderrLine('')
e_die("Can't run a proc while errexit is disabled. "
"Use 'run' or wrap it in a process with $0 myproc",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(proc_node, argv[1:])
return status
# TODO:
# look up arg0 in global namespace? And see if the type is value.Obj
# And it's a proc?
# isinstance(val.obj, objects.Proc)
UP_val = self.mem.GetVar(arg0)
if mylib.PYTHON: # Not reusing CPython objects
if UP_val.tag_() == value_e.Obj:
val = cast(value__Obj, UP_val)
if isinstance(val.obj, objects.Proc):
status = self.cmd_ev.RunOilProc(val.obj, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
environ = self.mem.GetExported() # Include temporary variables
if cmd_val.block:
e_die('Unexpected block passed to external command %r', arg0,
span_id=cmd_val.block.spids[0])
# Resolve argv[0] BEFORE forking.
argv0_path = self.search_path.CachedLookup(arg0)
if argv0_path is None:
self.errfmt.Print_('%r not found' % arg0, span_id=span_id)
return 127
# Normal case: ls /
if do_fork:
thunk = process.ExternalThunk(self.ext_prog, argv0_path, cmd_val, environ)
p = process.Process(thunk, self.job_state)
status = p.Run(self.waiter)
return status
# Already forked for pipeline: ls / | wc -l
# TODO: count subshell? ( ls / ) vs. ( ls /; ls / )
self.ext_prog.Exec(argv0_path, cmd_val, environ) # NEVER RETURNS
assert False, "This line should never be reached" # makes mypy happy
def RunSimpleCommand(self, arg_vec, fork_external, funcs=True):
"""Public interface to run a simple command (excluding assignment)
Args:
fork_external: for subshell ( ls / ) or ( command ls / )
"""
argv = arg_vec.strs
if arg_vec.spids:
span_id = arg_vec.spids[0]
else:
span_id = const.NO_INTEGER
# This happens when you write "$@" but have no arguments.
if not argv:
if self.exec_opts.strict_argv:
e_die("Command evaluated to an empty argv array",
span_id=span_id)
else:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = builtin.ResolveAssign(arg0)
if builtin_id != builtin_e.NONE:
# command readonly is disallowed, for technical reasons. Could relax it
# later.
self.errfmt.Print("Can't run assignment builtin recursively",
span_id=span_id)
return 1
builtin_id = builtin.ResolveSpecial(arg0)
if builtin_id != builtin_e.NONE:
status = self._RunBuiltin(builtin_id, arg_vec, fork_external)
# TODO: Enable this and fix spec test failures.
# Also update _SPECIAL_BUILTINS in osh/builtin.py.
#if status != 0:
# e_die('special builtin failed', status=status)
return status
# Builtins like 'true' can be redefined as functions.
if funcs:
func_node = self.funcs.get(arg0)
if func_node is not None:
# NOTE: Functions could call 'exit 42' directly, etc.
status = self._RunFunc(func_node, argv[1:])
return status
builtin_id = builtin.Resolve(arg0)
if builtin_id != builtin_e.NONE:
return self._RunBuiltin(builtin_id, arg_vec, fork_external)
environ = self.mem.GetExported() # Include temporary variables
# Resolve argv[0] BEFORE forking.
argv0_path = self.search_path.CachedLookup(argv[0])
if argv0_path is None:
self.errfmt.Print('%r not found', argv[0], span_id=span_id)
return 127
if fork_external:
thunk = process.ExternalThunk(self.ext_prog, argv0_path, arg_vec, environ)
p = process.Process(thunk, self.job_state)
status = p.Run(self.waiter)
return status
self.ext_prog.Exec(argv0_path, arg_vec, environ) # NEVER RETURNS
