def RunCommandSub(self, node):
# type: (command_t) -> str
# Hack for weird $(<file) construct
if node.tag_() == command_e.Simple:
simple = cast(command__Simple, node)
# Detect '< file'
if (len(simple.words) == 0 and len(simple.redirects) == 1
and simple.redirects[0].op.id == Id.Redir_Less):
# change it to __cat < file
# note: cmd_eval.py _Dispatch works around lack of spid
tok = Token(Id.Lit_Chars, runtime.NO_SPID, '__cat')
cat_word = compound_word([tok])
# MUTATE the command.Simple node. This will only be done the first
# time in the parent process.
simple.words.append(cat_word)
p = self._MakeProcess(node,
inherit_errexit=self.exec_opts.inherit_errexit())
r, w = posix.pipe()
p.AddStateChange(process.StdoutToPipe(r, w))
_ = p.Start()
#log('Command sub started %d', pid)
chunks = [] # type: List[str]
posix.close(w) # not going to write
while True:
byte_str = posix.read(r, 4096)
if len(byte_str) == 0:
break
chunks.append(byte_str)
posix.close(r)
status = p.Wait(self.waiter)
# OSH has the concept of aborting in the middle of a WORD. We're not
# waiting until the command is over!
if self.exec_opts.more_errexit():
if self.exec_opts.errexit() and status != 0:
raise error.ErrExit('Command sub exited with status %d (%r)',
status, NewStr(command_str(node.tag_())))
else:
# Set a flag so we check errexit at the same time as bash. Example:
#
# a=$(false)
# echo foo # no matter what comes here, the flag is reset
#
# Set ONLY until this command node has finished executing.
# HACK: move this
self.cmd_ev.check_command_sub_status = True
self.mem.SetLastStatus(status)
# Runtime errors test case: # $("echo foo > $@")
# Why rstrip()?
# https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
return ''.join(chunks).rstrip('\n')
def Add(self, p):
"""Append a process to the pipeline."""
if len(self.procs) == 0:
self.procs.append(p)
return
r, w = posix.pipe()
#log('pipe for %s: %d %d', p, r, w)
prev = self.procs[-1]
prev.AddStateChange(StdoutToPipe(r, w)) # applied on Start()
p.AddStateChange(StdinFromPipe(r, w)) # applied on Start()
p.AddPipeToClose(r, w) # MaybeClosePipe() on Start()
self.procs.append(p)
def AddLast(self, thunk):
"""Append the last node to the pipeline.
This is run in the CURRENT process. It is OPTIONAL, because pipelines in
the background are run uniformly.
"""
self.last_thunk = thunk
if len(self.procs) == 0: # No pipe: if ! foo
return
r, w = posix.pipe()
#log('last pipe %d %d', r, w)
prev = self.procs[-1]
prev.AddStateChange(StdoutToPipe(r, w))
self.last_pipe = (r, w) # So we can connect it to last_thunk
def testWrite(self):
if posix_.environ.get('EINTR_TEST'):
signal.signal(signal.SIGTERM, _Handler)
r, w = posix_.pipe()
log('Hanging on write in pid %d', posix_.getpid())
# 1 byte bigger than pipe size
n = posix_.write(w, 'x' * 65537)
log('1: Wrote %d bytes', n)
# write returns early when a signal interrupts it, and we read at least
# one byte! We do NOT get EINTR>
# On the second try, it didn't write anything, and we get EINTR!
log('Second try (pid %d)', posix_.getpid())
n = posix_.write(w, 'x' * 65537)
log('2: Wrote %d bytes', n)
def RunCommandSub(self, node):
p = self._MakeProcess(node,
disable_errexit=not self.exec_opts.strict_errexit)
r, w = posix.pipe()
p.AddStateChange(process.StdoutToPipe(r, w))
pid = p.Start()
#log('Command sub started %d', pid)
chunks = []
posix.close(w) # not going to write
while True:
byte_str = posix.read(r, 4096)
if not byte_str:
break
chunks.append(byte_str)
posix.close(r)
status = p.Wait(self.waiter)
# OSH has the concept of aborting in the middle of a WORD. We're not
# waiting until the command is over!
if self.exec_opts.strict_errexit:
if self.exec_opts.ErrExit() and status != 0:
raise util.ErrExitFailure(
'Command sub exited with status %d (%r)', status,
node.__class__.__name__)
else:
# Set a flag so we check errexit at the same time as bash. Example:
#
# a=$(false)
# echo foo # no matter what comes here, the flag is reset
#
# Set ONLY until this command node has finished executing.
self.check_command_sub_status = True
self.mem.SetLastStatus(status)
# Runtime errors test case: # $("echo foo > $@")
# Why rstrip()?
# https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
return ''.join(chunks).rstrip('\n')
def testPrint(self):
# Conclusion: print CAN raise IOError with EINTR.
if posix_.environ.get('EINTR_TEST'):
signal.signal(signal.SIGTERM, _Handler)
r, w = posix_.pipe()
log('Hanging on write in pid %d', posix_.getpid())
f = posix_.fdopen(w, 'w')
# 1 byte bigger than pipe size
print('x' * 65537, file=f)
log('1: done')
# write returns early when a signal interrupts it, and we read at least
# one byte! We do NOT get EINTR>
# On the second try, it didn't write anything, and we get EINTR!
log('Second try (pid %d)', posix_.getpid())
print('x' * 65537, file=f)
log('2: done')
def _ApplyRedirect(self, r, waiter):
# type: (redirect, Waiter) -> None
arg = r.arg
UP_arg = arg
with tagswitch(arg) as case:
if case(redirect_arg_e.Path):
arg = cast(redirect_arg__Path, UP_arg)
if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
# NOTE: This is different than >| because it respects noclobber, but
# that option is almost never used. See test/wild.sh.
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id == Id.Redir_Clobber: # >|
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id in (Id.Redir_DGreat,
Id.Redir_AndDGreat): # >> &>>
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND
elif r.op_id == Id.Redir_Less: # <
mode = posix.O_RDONLY
elif r.op_id == Id.Redir_LessGreat: # <>
mode = posix.O_CREAT | posix.O_RDWR
else:
raise NotImplementedError(r.op_id)
# NOTE: 0666 is affected by umask, all shells use it.
try:
open_fd = posix.open(arg.filename, mode, 0o666)
except OSError as e:
self.errfmt.Print_("Can't open %r: %s" %
(arg.filename, pyutil.strerror(e)),
span_id=r.op_spid)
raise # redirect failed
new_fd = self._PushDup(open_fd, r.loc)
if new_fd != NO_FD:
posix.close(open_fd)
# Now handle &> and &>> and their variants. These pairs are the same:
#
# stdout_stderr.py &> out-err.txt
# stdout_stderr.py > out-err.txt 2>&1
#
# stdout_stderr.py 3&> out-err.txt
# stdout_stderr.py 3> out-err.txt 2>&3
#
# Ditto for {fd}> and {fd}&>
if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
self._PushDup(new_fd, redir_loc.Fd(2))
elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
arg = cast(redirect_arg__CopyFd, UP_arg)
if r.op_id == Id.Redir_GreatAnd: # 1>&2
self._PushDup(arg.target_fd, r.loc)
elif r.op_id == Id.Redir_LessAnd: # 0<&5
# The only difference between >& and <& is the default file
# descriptor argument.
self._PushDup(arg.target_fd, r.loc)
else:
raise NotImplementedError()
elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
arg = cast(redirect_arg__MoveFd, UP_arg)
new_fd = self._PushDup(arg.target_fd, r.loc)
if new_fd != NO_FD:
posix.close(arg.target_fd)
UP_loc = r.loc
if r.loc.tag_() == redir_loc_e.Fd:
fd = cast(redir_loc__Fd, UP_loc).fd
else:
fd = NO_FD
self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
self._PushCloseFd(r.loc)
elif case(redirect_arg_e.HereDoc):
arg = cast(redirect_arg__HereDoc, UP_arg)
# NOTE: Do these descriptors have to be moved out of the range 0-9?
read_fd, write_fd = posix.pipe()
self._PushDup(read_fd, r.loc) # stdin is now the pipe
# We can't close like we do in the filename case above? The writer can
# get a "broken pipe".
self._PushClose(read_fd)
thunk = _HereDocWriterThunk(write_fd, arg.body)
# TODO: Use PIPE_SIZE to save a process in the case of small here docs,
# which are the common case. (dash does this.)
start_process = True
#start_process = False
if start_process:
here_proc = Process(thunk, self.job_state)
# NOTE: we could close the read pipe here, but it doesn't really
# matter because we control the code.
_ = here_proc.Start()
#log('Started %s as %d', here_proc, pid)
self._PushWait(here_proc, waiter)
# Now that we've started the child, close it in the parent.
posix.close(write_fd)
else:
posix.write(write_fd, arg.body)
posix.close(write_fd)
def RunProcessSub(self, cs_part):
# type: (command_sub) -> str
"""Process sub creates a forks a process connected to a pipe.
The pipe is typically passed to another process via a /dev/fd/$FD path.
Life cycle of a process substitution:
1. Start with this code
diff <(seq 3) <(seq 4)
2. To evaluate the command line, we evaluate every word. The
NormalWordEvaluator this method, RunProcessSub(), which does 3 things:
a. Create a pipe(), getting r and w
b. Starts the seq process, which inherits r and w
It has a StdoutToPipe() redirect, which means that it dup2(w, 1)
and close(r)
c. Close the w FD, because neither the shell or 'diff' will write to it.
However we must retain 'r', because 'diff' hasn't opened /dev/fd yet!
d. We evaluate <(seq 3) to /dev/fd/$r, so "diff" can read from it
3. Now we're done evaluating every word, so we know the command line of
diff, which looks like
diff /dev/fd/64 /dev/fd/65
Those are the FDs for the read ends of the pipes we created.
4. diff inherits a copy of the read end of bot pipes. But it actually
calls open() both files passed as argv. (I think this is fine.)
5. wait() for the diff process.
6. The shell closes both the read ends of both pipes. Neither us or
'diffd' will read again.
7. The shell waits for both 'seq' processes.
Related:
shopt -s process_sub_fail
_process_sub_status
"""
p = self._MakeProcess(cs_part.child)
r, w = posix.pipe()
#log('pipe = %d, %d', r, w)
op_id = cs_part.left_token.id
if op_id == Id.Left_ProcSubIn:
# Example: cat < <(head foo.txt)
#
# The head process should write its stdout to a pipe.
redir = process.StdoutToPipe(r, w) # type: process.ChildStateChange
elif op_id == Id.Left_ProcSubOut:
# Example: head foo.txt > >(tac)
#
# The tac process should read its stdin from a pipe.
# Note: this example sometimes requires you to hit "enter" in bash and
# zsh. WHy?
redir = process.StdinFromPipe(r, w)
else:
raise AssertionError()
p.AddStateChange(redir)
# Fork, letting the child inherit the pipe file descriptors.
pid = p.Start()
ps_frame = self.process_sub_stack[-1]
# Note: bash never waits() on the process, but zsh does. The calling
# program needs to read() before we can wait, e.g.
# diff <(sort left.txt) <(sort right.txt)
ps_frame.to_wait.append(p)
ps_frame.span_ids.append(cs_part.left_token.span_id)
# After forking, close the end of the pipe we're not using.
if op_id == Id.Left_ProcSubIn:
posix.close(w) # cat < <(head foo.txt)
ps_frame.to_close.append(r) # close later
elif op_id == Id.Left_ProcSubOut:
posix.close(r)
#log('Left_ProcSubOut closed %d', r)
ps_frame.to_close.append(w) # close later
else:
raise AssertionError()
# Is /dev Linux-specific?
if op_id == Id.Left_ProcSubIn:
return '/dev/fd/%d' % r
elif op_id == Id.Left_ProcSubOut:
return '/dev/fd/%d' % w
else:
raise AssertionError()
def _ApplyRedirect(self, r, waiter):
# type: (redirect_t, Waiter) -> bool
ok = True
UP_r = r
with tagswitch(r) as case:
if case(redirect_e.Path):
r = cast(redirect__Path, UP_r)
if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
# NOTE: This is different than >| because it respects noclobber, but
# that option is almost never used. See test/wild.sh.
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id == Id.Redir_Clobber: # >|
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id in (Id.Redir_DGreat, Id.Redir_AndDGreat): # >> &>>
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND
elif r.op_id == Id.Redir_Less: # <
mode = posix.O_RDONLY
else:
raise NotImplementedError(r.op_id)
# NOTE: 0666 is affected by umask, all shells use it.
try:
target_fd = posix.open(r.filename, mode, 0o666)
except OSError as e:
self.errfmt.Print(
"Can't open %r: %s", r.filename, posix.strerror(e.errno),
span_id=r.op_spid)
return False
# Apply redirect
if not self._PushDup(target_fd, r.fd):
ok = False
# Now handle the extra redirects for aliases &> and &>>.
#
# We can rewrite
# stdout_stderr.py &> out-err.txt
# as
# stdout_stderr.py > out-err.txt 2>&1
#
# And rewrite
# stdout_stderr.py 3&> out-err.txt
# as
# stdout_stderr.py 3> out-err.txt 2>&3
if ok:
if r.op_id == Id.Redir_AndGreat:
if not self._PushDup(r.fd, 2):
ok = False
elif r.op_id == Id.Redir_AndDGreat:
if not self._PushDup(r.fd, 2):
ok = False
posix.close(target_fd) # We already made a copy of it.
# I don't think we need to close(0) because it will be restored from its
# saved position (10), which closes it.
#self._PushClose(r.fd)
elif case(redirect_e.FileDesc): # e.g. echo hi 1>&2
r = cast(redirect__FileDesc, UP_r)
if r.op_id == Id.Redir_GreatAnd: # 1>&2
if not self._PushDup(r.target_fd, r.fd):
ok = False
elif r.op_id == Id.Redir_LessAnd: # 0<&5
# The only difference between >& and <& is the default file
# descriptor argument.
if not self._PushDup(r.target_fd, r.fd):
ok = False
else:
raise NotImplementedError()
elif case(redirect_e.HereDoc):
r = cast(redirect__HereDoc, UP_r)
# NOTE: Do these descriptors have to be moved out of the range 0-9?
read_fd, write_fd = posix.pipe()
if not self._PushDup(read_fd, r.fd): # stdin is now the pipe
ok = False
# We can't close like we do in the filename case above? The writer can
# get a "broken pipe".
self._PushClose(read_fd)
thunk = _HereDocWriterThunk(write_fd, r.body)
# TODO: Use PIPE_SIZE to save a process in the case of small here docs,
# which are the common case. (dash does this.)
start_process = True
#start_process = False
if start_process:
here_proc = Process(thunk, self.job_state)
# NOTE: we could close the read pipe here, but it doesn't really
# matter because we control the code.
_ = here_proc.Start()
#log('Started %s as %d', here_proc, pid)
self._PushWait(here_proc, waiter)
# Now that we've started the child, close it in the parent.
posix.close(write_fd)
else:
posix.write(write_fd, r.body)
posix.close(write_fd)
return ok
def RunProcessSub(self, node, op_id):
# type: (command_t, Id_t) -> str
"""Process sub creates a forks a process connected to a pipe.
The pipe is typically passed to another process via a /dev/fd/$FD path.
TODO:
sane-proc-sub:
- wait for all the words
Otherwise, set $! (mem.last_bg_pid)
strict-proc-sub:
- Don't allow it anywhere except SimpleCommand, any redirect, or
ShAssignment? And maybe not even assignment?
Should you put return codes in @PROCESS_SUB_STATUS? You need two of them.
"""
p = self._MakeProcess(node)
r, w = posix.pipe()
if op_id == Id.Left_ProcSubIn:
# Example: cat < <(head foo.txt)
#
# The head process should write its stdout to a pipe.
redir = process.StdoutToPipe(r,
w) # type: process.ChildStateChange
elif op_id == Id.Left_ProcSubOut:
# Example: head foo.txt > >(tac)
#
# The tac process should read its stdin from a pipe.
#
# NOTE: This appears to hang in bash? At least when done interactively.
# It doesn't work at all in osh interactively?
redir = process.StdinFromPipe(r, w)
else:
raise AssertionError()
p.AddStateChange(redir)
# Fork, letting the child inherit the pipe file descriptors.
pid = p.Start()
# After forking, close the end of the pipe we're not using.
if op_id == Id.Left_ProcSubIn:
posix.close(w)
elif op_id == Id.Left_ProcSubOut:
posix.close(r)
else:
raise AssertionError()
# NOTE: Like bash, we never actually wait on it!
# TODO: At least set $! ?
# Is /dev Linux-specific?
if op_id == Id.Left_ProcSubIn:
return '/dev/fd/%d' % r
elif op_id == Id.Left_ProcSubOut:
return '/dev/fd/%d' % w
else:
raise AssertionError()
def RunCommandSub(self, cs_part):
# type: (command_sub) -> str
if not self.exec_opts.allow_command_sub():
# TODO:
# - Add spid of $(
# - Better hints. Use 'run' for 'if myfunc', and 2 lines like local x;
# x=$(false) fo assignment builtins.
# - Maybe we should have an error message ID that links somewhere?
e_die(
"Command subs not allowed here because status wouldn't be checked (strict_errexit)."
)
node = cs_part.child
# Hack for weird $(<file) construct
if node.tag_() == command_e.Simple:
simple = cast(command__Simple, node)
# Detect '< file'
if (len(simple.words) == 0 and len(simple.redirects) == 1
and simple.redirects[0].op.id == Id.Redir_Less):
# change it to __cat < file
# note: cmd_eval.py _Dispatch works around lack of spid
tok = Token(Id.Lit_Chars, runtime.NO_SPID, '__cat')
cat_word = compound_word([tok])
# MUTATE the command.Simple node. This will only be done the first
# time in the parent process.
simple.words.append(cat_word)
p = self._MakeProcess(node,
inherit_errexit=self.exec_opts.inherit_errexit())
r, w = posix.pipe()
p.AddStateChange(process.StdoutToPipe(r, w))
_ = p.Start()
#log('Command sub started %d', pid)
chunks = [] # type: List[str]
posix.close(w) # not going to write
while True:
byte_str = posix.read(r, 4096)
if len(byte_str) == 0:
break
chunks.append(byte_str)
posix.close(r)
status = p.Wait(self.waiter)
# OSH has the concept of aborting in the middle of a WORD. We're not
# waiting until the command is over!
if self.exec_opts.command_sub_errexit():
if status != 0:
raise error.ErrExit('Command sub exited with status %d (%s)' %
(status, ui.CommandType(node)),
span_id=cs_part.left_token.span_id,
status=status)
else:
# Set a flag so we check errexit at the same time as bash. Example:
#
# a=$(false)
# echo foo # no matter what comes here, the flag is reset
#
# Set ONLY until this command node has finished executing.
# HACK: move this
self.cmd_ev.check_command_sub_status = True
self.mem.SetLastStatus(status)
# Runtime errors test case: # $("echo foo > $@")
# Why rstrip()?
# https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
return ''.join(chunks).rstrip('\n')
