def _Open(self, path, c_mode, fd_mode):
# type: (str, str, int) -> mylib.LineReader
fd = posix.open(path, fd_mode, 0o666) # may raise OSError
# Immediately move it to a new location
new_fd = fcntl_.fcntl(fd, fcntl_.F_DUPFD, _SHELL_MIN_FD) # type: int
posix.close(fd)
# Return a Python file handle
# Might raise IOError. Python-induced wart: we have to handle BOTH IOError
# and OSError at higher levels.
f = posix.fdopen(new_fd, c_mode)
return f
def MaybeWaitOnProcessSubs(self, frame, compound_st):
# type: (_ProcessSubFrame, CompoundStatus) -> None
# Wait in the same order that they were evaluated. That seems fine.
for fd in frame.to_close:
posix.close(fd)
for i, p in enumerate(frame.to_wait):
#log('waiting for %s', p)
st = p.Wait(self.waiter)
compound_st.codes.append(st)
compound_st.spids.append(frame.span_ids[i])
i += 1
def _Open(self, path, c_mode, fd_mode):
# type: (str, str, int) -> mylib.LineReader
fd = posix.open(path, fd_mode, 0o666) # may raise OSError
# Immediately move it to a new location
new_fd = fcntl.fcntl(fd, fcntl.F_DUPFD, _SHELL_MIN_FD)
posix.close(fd)
# Return a Python file handle
try:
f = posix.fdopen(new_fd, c_mode) # Might raise IOError
except IOError as e:
raise OSError(*e.args) # Consistently raise OSError
return f
def Run(self, waiter, fd_state):
# type: (Waiter, FdState) -> List[int]
"""Run this pipeline synchronously (foreground pipeline).
Returns:
pipe_status (list of integers).
"""
self.Start(waiter)
# Run our portion IN PARALLEL with other processes. This may or may not
# fork:
# ls | wc -l
# echo foo | read line # no need to fork
cmd_ev, node = self.last_thunk
#log('thunk %s', self.last_thunk)
if self.last_pipe is not None:
r, w = self.last_pipe # set in AddLast()
posix.close(w) # we will not write here
fd_state.PushStdinFromPipe(r)
# TODO: determine fork_external here, so we can go BEYOND lastpipe. Not
# only do we run builtins in the same process. External processes will
# exec() rather than fork/exec().
try:
cmd_ev.ExecuteAndCatch(node)
finally:
fd_state.Pop()
# We won't read anymore. If we don't do this, then 'cat' in 'cat
# /dev/urandom | sleep 1' will never get SIGPIPE.
posix.close(r)
else:
if len(self.procs):
cmd_ev.ExecuteAndCatch(
node) # Background pipeline without last_pipe
else:
cmd_ev._Execute(
node) # singleton foreground pipeline, e.g. '! func'
self.pipe_status[-1] = cmd_ev.LastStatus()
#log('pipestatus before all have finished = %s', self.pipe_status)
if len(self.procs):
return self.Wait(waiter)
else:
return self.pipe_status # singleton foreground pipeline, e.g. '! func'
def Run(self, waiter, fd_state):
# type: (Waiter, FdState) -> List[int]
"""Run this pipeline synchronously (foreground pipeline).
Returns:
pipe_status (list of integers).
"""
self.Start(waiter)
# Run our portion IN PARALLEL with other processes. This may or may not
# fork:
# ls | wc -l
# echo foo | read line # no need to fork
ex, node = self.last_thunk
#log('thunk %s', self.last_thunk)
if self.last_pipe is not None:
r, w = self.last_pipe # set in AddLast()
posix.close(w) # we will not write here
fd_state.PushStdinFromPipe(r)
try:
ex.ExecuteAndCatch(node)
finally:
fd_state.Pop()
# We won't read anymore. If we don't do this, then 'cat' in 'cat
# /dev/urandom | sleep 1' will never get SIGPIPE.
posix.close(r)
else:
if self.procs:
ex.ExecuteAndCatch(
node) # Background pipeline without last_pipe
else:
ex._Execute(
node) # singleton foreground pipeline, e.g. '! func'
self.pipe_status[-1] = ex.LastStatus()
#log('pipestatus before all have finished = %s', self.pipe_status)
if self.procs:
return self.Wait(waiter)
else:
return self.pipe_status # singleton foreground pipeline, e.g. '! func'
def _PushDup(self, fd1, fd2):
# type: (int, int) -> bool
"""Save fd2, and dup fd1 onto fd2.
Mutates self.cur_frame.saved.
Returns:
success Bool
"""
new_fd = self._GetFreeDescriptor()
#log('---- _PushDup %s %s', fd1, fd2)
need_restore = True
try:
fcntl.fcntl(fd2, fcntl.F_DUPFD, new_fd)
except IOError as e:
# Example program that causes this error: exec 4>&1. Descriptor 4 isn't
# open.
# This seems to be ignored in dash too in savefd()?
if e.errno == errno.EBADF:
#log('ERROR %s', e)
need_restore = False
else:
raise
else:
posix.close(fd2)
fcntl.fcntl(new_fd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
#log('==== dup %s %s\n' % (fd1, fd2))
try:
posix.dup2(fd1, fd2)
except OSError as e:
# bash/dash give this error too, e.g. for 'echo hi 1>&3'
self.errfmt.Print('%d: %s', fd1, posix.strerror(e.errno))
# Restore and return error
posix.dup2(new_fd, fd2)
posix.close(new_fd)
# Undo it
return False
if need_restore:
self.cur_frame.saved.append((new_fd, fd2))
return True
def RunCommandSub(self, node):
p = self._MakeProcess(node,
disable_errexit=not self.exec_opts.strict_errexit)
r, w = posix.pipe()
p.AddStateChange(process.StdoutToPipe(r, w))
pid = p.Start()
#log('Command sub started %d', pid)
chunks = []
posix.close(w) # not going to write
while True:
byte_str = posix.read(r, 4096)
if not byte_str:
break
chunks.append(byte_str)
posix.close(r)
status = p.Wait(self.waiter)
# OSH has the concept of aborting in the middle of a WORD. We're not
# waiting until the command is over!
if self.exec_opts.strict_errexit:
if self.exec_opts.ErrExit() and status != 0:
raise util.ErrExitFailure(
'Command sub exited with status %d (%r)', status,
node.__class__.__name__)
else:
# Set a flag so we check errexit at the same time as bash. Example:
#
# a=$(false)
# echo foo # no matter what comes here, the flag is reset
#
# Set ONLY until this command node has finished executing.
self.check_command_sub_status = True
self.mem.SetLastStatus(status)
# Runtime errors test case: # $("echo foo > $@")
# Why rstrip()?
# https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
return ''.join(chunks).rstrip('\n')
def MaybeClosePipe(self):
# type: () -> None
if self.close_r != -1:
posix.close(self.close_r)
posix.close(self.close_w)
def Apply(self):
# type: () -> None
posix.dup2(self.w, 1)
posix.close(self.w) # close after dup
posix.close(self.r) # we're writing to the pipe, not reading
def Apply(self):
# type: () -> None
posix.dup2(self.r, 0)
posix.close(self.r) # close after dup
posix.close(self.w) # we're reading from the pipe, not writing
def _ApplyRedirect(self, r, waiter):
# type: (redirect, Waiter) -> None
arg = r.arg
UP_arg = arg
with tagswitch(arg) as case:
if case(redirect_arg_e.Path):
arg = cast(redirect_arg__Path, UP_arg)
if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
# NOTE: This is different than >| because it respects noclobber, but
# that option is almost never used. See test/wild.sh.
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id == Id.Redir_Clobber: # >|
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id in (Id.Redir_DGreat,
Id.Redir_AndDGreat): # >> &>>
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND
elif r.op_id == Id.Redir_Less: # <
mode = posix.O_RDONLY
elif r.op_id == Id.Redir_LessGreat: # <>
mode = posix.O_CREAT | posix.O_RDWR
else:
raise NotImplementedError(r.op_id)
# NOTE: 0666 is affected by umask, all shells use it.
try:
open_fd = posix.open(arg.filename, mode, 0o666)
except OSError as e:
self.errfmt.Print_("Can't open %r: %s" %
(arg.filename, pyutil.strerror(e)),
span_id=r.op_spid)
raise # redirect failed
new_fd = self._PushDup(open_fd, r.loc)
if new_fd != NO_FD:
posix.close(open_fd)
# Now handle &> and &>> and their variants. These pairs are the same:
#
# stdout_stderr.py &> out-err.txt
# stdout_stderr.py > out-err.txt 2>&1
#
# stdout_stderr.py 3&> out-err.txt
# stdout_stderr.py 3> out-err.txt 2>&3
#
# Ditto for {fd}> and {fd}&>
if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
self._PushDup(new_fd, redir_loc.Fd(2))
elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
arg = cast(redirect_arg__CopyFd, UP_arg)
if r.op_id == Id.Redir_GreatAnd: # 1>&2
self._PushDup(arg.target_fd, r.loc)
elif r.op_id == Id.Redir_LessAnd: # 0<&5
# The only difference between >& and <& is the default file
# descriptor argument.
self._PushDup(arg.target_fd, r.loc)
else:
raise NotImplementedError()
elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
arg = cast(redirect_arg__MoveFd, UP_arg)
new_fd = self._PushDup(arg.target_fd, r.loc)
if new_fd != NO_FD:
posix.close(arg.target_fd)
UP_loc = r.loc
if r.loc.tag_() == redir_loc_e.Fd:
fd = cast(redir_loc__Fd, UP_loc).fd
else:
fd = NO_FD
self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
self._PushCloseFd(r.loc)
elif case(redirect_arg_e.HereDoc):
arg = cast(redirect_arg__HereDoc, UP_arg)
# NOTE: Do these descriptors have to be moved out of the range 0-9?
read_fd, write_fd = posix.pipe()
self._PushDup(read_fd, r.loc) # stdin is now the pipe
# We can't close like we do in the filename case above? The writer can
# get a "broken pipe".
self._PushClose(read_fd)
thunk = _HereDocWriterThunk(write_fd, arg.body)
# TODO: Use PIPE_SIZE to save a process in the case of small here docs,
# which are the common case. (dash does this.)
start_process = True
#start_process = False
if start_process:
here_proc = Process(thunk, self.job_state)
# NOTE: we could close the read pipe here, but it doesn't really
# matter because we control the code.
_ = here_proc.Start()
#log('Started %s as %d', here_proc, pid)
self._PushWait(here_proc, waiter)
# Now that we've started the child, close it in the parent.
posix.close(write_fd)
else:
posix.write(write_fd, arg.body)
posix.close(write_fd)
def RunProcessSub(self, cs_part):
# type: (command_sub) -> str
"""Process sub creates a forks a process connected to a pipe.
The pipe is typically passed to another process via a /dev/fd/$FD path.
Life cycle of a process substitution:
1. Start with this code
diff <(seq 3) <(seq 4)
2. To evaluate the command line, we evaluate every word. The
NormalWordEvaluator this method, RunProcessSub(), which does 3 things:
a. Create a pipe(), getting r and w
b. Starts the seq process, which inherits r and w
It has a StdoutToPipe() redirect, which means that it dup2(w, 1)
and close(r)
c. Close the w FD, because neither the shell or 'diff' will write to it.
However we must retain 'r', because 'diff' hasn't opened /dev/fd yet!
d. We evaluate <(seq 3) to /dev/fd/$r, so "diff" can read from it
3. Now we're done evaluating every word, so we know the command line of
diff, which looks like
diff /dev/fd/64 /dev/fd/65
Those are the FDs for the read ends of the pipes we created.
4. diff inherits a copy of the read end of bot pipes. But it actually
calls open() both files passed as argv. (I think this is fine.)
5. wait() for the diff process.
6. The shell closes both the read ends of both pipes. Neither us or
'diffd' will read again.
7. The shell waits for both 'seq' processes.
Related:
shopt -s process_sub_fail
_process_sub_status
"""
p = self._MakeProcess(cs_part.child)
r, w = posix.pipe()
#log('pipe = %d, %d', r, w)
op_id = cs_part.left_token.id
if op_id == Id.Left_ProcSubIn:
# Example: cat < <(head foo.txt)
#
# The head process should write its stdout to a pipe.
redir = process.StdoutToPipe(r, w) # type: process.ChildStateChange
elif op_id == Id.Left_ProcSubOut:
# Example: head foo.txt > >(tac)
#
# The tac process should read its stdin from a pipe.
# Note: this example sometimes requires you to hit "enter" in bash and
# zsh. WHy?
redir = process.StdinFromPipe(r, w)
else:
raise AssertionError()
p.AddStateChange(redir)
# Fork, letting the child inherit the pipe file descriptors.
pid = p.Start()
ps_frame = self.process_sub_stack[-1]
# Note: bash never waits() on the process, but zsh does. The calling
# program needs to read() before we can wait, e.g.
# diff <(sort left.txt) <(sort right.txt)
ps_frame.to_wait.append(p)
ps_frame.span_ids.append(cs_part.left_token.span_id)
# After forking, close the end of the pipe we're not using.
if op_id == Id.Left_ProcSubIn:
posix.close(w) # cat < <(head foo.txt)
ps_frame.to_close.append(r) # close later
elif op_id == Id.Left_ProcSubOut:
posix.close(r)
#log('Left_ProcSubOut closed %d', r)
ps_frame.to_close.append(w) # close later
else:
raise AssertionError()
# Is /dev Linux-specific?
if op_id == Id.Left_ProcSubIn:
return '/dev/fd/%d' % r
elif op_id == Id.Left_ProcSubOut:
return '/dev/fd/%d' % w
else:
raise AssertionError()
def RunCommandSub(self, cs_part):
# type: (command_sub) -> str
if not self.exec_opts.allow_command_sub():
# TODO: Add spid of $(
e_die("Command subs not allowed when errexit disabled (strict_errexit)")
node = cs_part.child
# Hack for weird $(<file) construct
if node.tag_() == command_e.Simple:
simple = cast(command__Simple, node)
# Detect '< file'
if (len(simple.words) == 0 and
len(simple.redirects) == 1 and
simple.redirects[0].op.id == Id.Redir_Less):
# change it to __cat < file
# note: cmd_eval.py _Dispatch works around lack of spid
tok = Token(Id.Lit_Chars, runtime.NO_SPID, '__cat')
cat_word = compound_word([tok])
# MUTATE the command.Simple node. This will only be done the first
# time in the parent process.
simple.words.append(cat_word)
p = self._MakeProcess(node,
inherit_errexit=self.exec_opts.inherit_errexit())
r, w = posix.pipe()
p.AddStateChange(process.StdoutToPipe(r, w))
_ = p.Start()
#log('Command sub started %d', pid)
chunks = [] # type: List[str]
posix.close(w) # not going to write
while True:
byte_str = posix.read(r, 4096)
if len(byte_str) == 0:
break
chunks.append(byte_str)
posix.close(r)
status = p.Wait(self.waiter)
# OSH has the concept of aborting in the middle of a WORD. We're not
# waiting until the command is over!
if self.exec_opts.command_sub_errexit():
if status != 0:
raise error.ErrExit(
'Command sub exited with status %d (%s)' %
(status, ui.CommandType(node)), span_id=cs_part.left_token.span_id,
status=status)
else:
# Set a flag so we check errexit at the same time as bash. Example:
#
# a=$(false)
# echo foo # no matter what comes here, the flag is reset
#
# Set ONLY until this command node has finished executing.
# HACK: move this
self.cmd_ev.check_command_sub_status = True
self.mem.SetLastStatus(status)
# Runtime errors test case: # $("echo foo > $@")
# Why rstrip()?
# https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
return ''.join(chunks).rstrip('\n')
def _ApplyRedirect(self, r, waiter):
# type: (redirect_t, Waiter) -> bool
ok = True
UP_r = r
with tagswitch(r) as case:
if case(redirect_e.Path):
r = cast(redirect__Path, UP_r)
if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
# NOTE: This is different than >| because it respects noclobber, but
# that option is almost never used. See test/wild.sh.
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id == Id.Redir_Clobber: # >|
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id in (Id.Redir_DGreat, Id.Redir_AndDGreat): # >> &>>
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND
elif r.op_id == Id.Redir_Less: # <
mode = posix.O_RDONLY
else:
raise NotImplementedError(r.op_id)
# NOTE: 0666 is affected by umask, all shells use it.
try:
target_fd = posix.open(r.filename, mode, 0o666)
except OSError as e:
self.errfmt.Print(
"Can't open %r: %s", r.filename, posix.strerror(e.errno),
span_id=r.op_spid)
return False
# Apply redirect
if not self._PushDup(target_fd, r.fd):
ok = False
# Now handle the extra redirects for aliases &> and &>>.
#
# We can rewrite
# stdout_stderr.py &> out-err.txt
# as
# stdout_stderr.py > out-err.txt 2>&1
#
# And rewrite
# stdout_stderr.py 3&> out-err.txt
# as
# stdout_stderr.py 3> out-err.txt 2>&3
if ok:
if r.op_id == Id.Redir_AndGreat:
if not self._PushDup(r.fd, 2):
ok = False
elif r.op_id == Id.Redir_AndDGreat:
if not self._PushDup(r.fd, 2):
ok = False
posix.close(target_fd) # We already made a copy of it.
# I don't think we need to close(0) because it will be restored from its
# saved position (10), which closes it.
#self._PushClose(r.fd)
elif case(redirect_e.FileDesc): # e.g. echo hi 1>&2
r = cast(redirect__FileDesc, UP_r)
if r.op_id == Id.Redir_GreatAnd: # 1>&2
if not self._PushDup(r.target_fd, r.fd):
ok = False
elif r.op_id == Id.Redir_LessAnd: # 0<&5
# The only difference between >& and <& is the default file
# descriptor argument.
if not self._PushDup(r.target_fd, r.fd):
ok = False
else:
raise NotImplementedError()
elif case(redirect_e.HereDoc):
r = cast(redirect__HereDoc, UP_r)
# NOTE: Do these descriptors have to be moved out of the range 0-9?
read_fd, write_fd = posix.pipe()
if not self._PushDup(read_fd, r.fd): # stdin is now the pipe
ok = False
# We can't close like we do in the filename case above? The writer can
# get a "broken pipe".
self._PushClose(read_fd)
thunk = _HereDocWriterThunk(write_fd, r.body)
# TODO: Use PIPE_SIZE to save a process in the case of small here docs,
# which are the common case. (dash does this.)
start_process = True
#start_process = False
if start_process:
here_proc = Process(thunk, self.job_state)
# NOTE: we could close the read pipe here, but it doesn't really
# matter because we control the code.
_ = here_proc.Start()
#log('Started %s as %d', here_proc, pid)
self._PushWait(here_proc, waiter)
# Now that we've started the child, close it in the parent.
posix.close(write_fd)
else:
posix.write(write_fd, r.body)
posix.close(write_fd)
return ok
def MaybeClosePipe(self):
if self.close_r != -1:
posix.close(self.close_r)
posix.close(self.close_w)
def RunProcessSub(self, node, op_id):
# type: (command_t, Id_t) -> str
"""Process sub creates a forks a process connected to a pipe.
The pipe is typically passed to another process via a /dev/fd/$FD path.
TODO:
sane-proc-sub:
- wait for all the words
Otherwise, set $! (mem.last_bg_pid)
strict-proc-sub:
- Don't allow it anywhere except SimpleCommand, any redirect, or
ShAssignment? And maybe not even assignment?
Should you put return codes in @PROCESS_SUB_STATUS? You need two of them.
"""
p = self._MakeProcess(node)
r, w = posix.pipe()
if op_id == Id.Left_ProcSubIn:
# Example: cat < <(head foo.txt)
#
# The head process should write its stdout to a pipe.
redir = process.StdoutToPipe(r,
w) # type: process.ChildStateChange
elif op_id == Id.Left_ProcSubOut:
# Example: head foo.txt > >(tac)
#
# The tac process should read its stdin from a pipe.
#
# NOTE: This appears to hang in bash? At least when done interactively.
# It doesn't work at all in osh interactively?
redir = process.StdinFromPipe(r, w)
else:
raise AssertionError()
p.AddStateChange(redir)
# Fork, letting the child inherit the pipe file descriptors.
pid = p.Start()
# After forking, close the end of the pipe we're not using.
if op_id == Id.Left_ProcSubIn:
posix.close(w)
elif op_id == Id.Left_ProcSubOut:
posix.close(r)
else:
raise AssertionError()
# NOTE: Like bash, we never actually wait on it!
# TODO: At least set $! ?
# Is /dev Linux-specific?
if op_id == Id.Left_ProcSubIn:
return '/dev/fd/%d' % r
elif op_id == Id.Left_ProcSubOut:
return '/dev/fd/%d' % w
else:
raise AssertionError()