def RunCommandSub(self, node): # type: (command_t) -> str # Hack for weird $(<file) construct if node.tag_() == command_e.Simple: simple = cast(command__Simple, node) # Detect '< file' if (len(simple.words) == 0 and len(simple.redirects) == 1 and simple.redirects[0].op.id == Id.Redir_Less): # change it to __cat < file # note: cmd_eval.py _Dispatch works around lack of spid tok = Token(Id.Lit_Chars, runtime.NO_SPID, '__cat') cat_word = compound_word([tok]) # MUTATE the command.Simple node. This will only be done the first # time in the parent process. simple.words.append(cat_word) p = self._MakeProcess(node, inherit_errexit=self.exec_opts.inherit_errexit()) r, w = posix.pipe() p.AddStateChange(process.StdoutToPipe(r, w)) _ = p.Start() #log('Command sub started %d', pid) chunks = [] # type: List[str] posix.close(w) # not going to write while True: byte_str = posix.read(r, 4096) if len(byte_str) == 0: break chunks.append(byte_str) posix.close(r) status = p.Wait(self.waiter) # OSH has the concept of aborting in the middle of a WORD. We're not # waiting until the command is over! if self.exec_opts.more_errexit(): if self.exec_opts.errexit() and status != 0: raise error.ErrExit('Command sub exited with status %d (%r)', status, NewStr(command_str(node.tag_()))) else: # Set a flag so we check errexit at the same time as bash. Example: # # a=$(false) # echo foo # no matter what comes here, the flag is reset # # Set ONLY until this command node has finished executing. # HACK: move this self.cmd_ev.check_command_sub_status = True self.mem.SetLastStatus(status) # Runtime errors test case: # $("echo foo > $@") # Why rstrip()? # https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char return ''.join(chunks).rstrip('\n')
def Add(self, p): """Append a process to the pipeline.""" if len(self.procs) == 0: self.procs.append(p) return r, w = posix.pipe() #log('pipe for %s: %d %d', p, r, w) prev = self.procs[-1] prev.AddStateChange(StdoutToPipe(r, w)) # applied on Start() p.AddStateChange(StdinFromPipe(r, w)) # applied on Start() p.AddPipeToClose(r, w) # MaybeClosePipe() on Start() self.procs.append(p)
def AddLast(self, thunk): """Append the last node to the pipeline. This is run in the CURRENT process. It is OPTIONAL, because pipelines in the background are run uniformly. """ self.last_thunk = thunk if len(self.procs) == 0: # No pipe: if ! foo return r, w = posix.pipe() #log('last pipe %d %d', r, w) prev = self.procs[-1] prev.AddStateChange(StdoutToPipe(r, w)) self.last_pipe = (r, w) # So we can connect it to last_thunk
def testWrite(self): if posix_.environ.get('EINTR_TEST'): signal.signal(signal.SIGTERM, _Handler) r, w = posix_.pipe() log('Hanging on write in pid %d', posix_.getpid()) # 1 byte bigger than pipe size n = posix_.write(w, 'x' * 65537) log('1: Wrote %d bytes', n) # write returns early when a signal interrupts it, and we read at least # one byte! We do NOT get EINTR> # On the second try, it didn't write anything, and we get EINTR! log('Second try (pid %d)', posix_.getpid()) n = posix_.write(w, 'x' * 65537) log('2: Wrote %d bytes', n)
def RunCommandSub(self, node): p = self._MakeProcess(node, disable_errexit=not self.exec_opts.strict_errexit) r, w = posix.pipe() p.AddStateChange(process.StdoutToPipe(r, w)) pid = p.Start() #log('Command sub started %d', pid) chunks = [] posix.close(w) # not going to write while True: byte_str = posix.read(r, 4096) if not byte_str: break chunks.append(byte_str) posix.close(r) status = p.Wait(self.waiter) # OSH has the concept of aborting in the middle of a WORD. We're not # waiting until the command is over! if self.exec_opts.strict_errexit: if self.exec_opts.ErrExit() and status != 0: raise util.ErrExitFailure( 'Command sub exited with status %d (%r)', status, node.__class__.__name__) else: # Set a flag so we check errexit at the same time as bash. Example: # # a=$(false) # echo foo # no matter what comes here, the flag is reset # # Set ONLY until this command node has finished executing. self.check_command_sub_status = True self.mem.SetLastStatus(status) # Runtime errors test case: # $("echo foo > $@") # Why rstrip()? # https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char return ''.join(chunks).rstrip('\n')
def testPrint(self): # Conclusion: print CAN raise IOError with EINTR. if posix_.environ.get('EINTR_TEST'): signal.signal(signal.SIGTERM, _Handler) r, w = posix_.pipe() log('Hanging on write in pid %d', posix_.getpid()) f = posix_.fdopen(w, 'w') # 1 byte bigger than pipe size print('x' * 65537, file=f) log('1: done') # write returns early when a signal interrupts it, and we read at least # one byte! We do NOT get EINTR> # On the second try, it didn't write anything, and we get EINTR! log('Second try (pid %d)', posix_.getpid()) print('x' * 65537, file=f) log('2: done')
def _ApplyRedirect(self, r, waiter): # type: (redirect, Waiter) -> None arg = r.arg UP_arg = arg with tagswitch(arg) as case: if case(redirect_arg_e.Path): arg = cast(redirect_arg__Path, UP_arg) if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &> # NOTE: This is different than >| because it respects noclobber, but # that option is almost never used. See test/wild.sh. mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC elif r.op_id == Id.Redir_Clobber: # >| mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC elif r.op_id in (Id.Redir_DGreat, Id.Redir_AndDGreat): # >> &>> mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND elif r.op_id == Id.Redir_Less: # < mode = posix.O_RDONLY elif r.op_id == Id.Redir_LessGreat: # <> mode = posix.O_CREAT | posix.O_RDWR else: raise NotImplementedError(r.op_id) # NOTE: 0666 is affected by umask, all shells use it. try: open_fd = posix.open(arg.filename, mode, 0o666) except OSError as e: self.errfmt.Print_("Can't open %r: %s" % (arg.filename, pyutil.strerror(e)), span_id=r.op_spid) raise # redirect failed new_fd = self._PushDup(open_fd, r.loc) if new_fd != NO_FD: posix.close(open_fd) # Now handle &> and &>> and their variants. These pairs are the same: # # stdout_stderr.py &> out-err.txt # stdout_stderr.py > out-err.txt 2>&1 # # stdout_stderr.py 3&> out-err.txt # stdout_stderr.py 3> out-err.txt 2>&3 # # Ditto for {fd}> and {fd}&> if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat): self._PushDup(new_fd, redir_loc.Fd(2)) elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2 arg = cast(redirect_arg__CopyFd, UP_arg) if r.op_id == Id.Redir_GreatAnd: # 1>&2 self._PushDup(arg.target_fd, r.loc) elif r.op_id == Id.Redir_LessAnd: # 0<&5 # The only difference between >& and <& is the default file # descriptor argument. self._PushDup(arg.target_fd, r.loc) else: raise NotImplementedError() elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6- arg = cast(redirect_arg__MoveFd, UP_arg) new_fd = self._PushDup(arg.target_fd, r.loc) if new_fd != NO_FD: posix.close(arg.target_fd) UP_loc = r.loc if r.loc.tag_() == redir_loc_e.Fd: fd = cast(redir_loc__Fd, UP_loc).fd else: fd = NO_FD self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False)) elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&- self._PushCloseFd(r.loc) elif case(redirect_arg_e.HereDoc): arg = cast(redirect_arg__HereDoc, UP_arg) # NOTE: Do these descriptors have to be moved out of the range 0-9? read_fd, write_fd = posix.pipe() self._PushDup(read_fd, r.loc) # stdin is now the pipe # We can't close like we do in the filename case above? The writer can # get a "broken pipe". self._PushClose(read_fd) thunk = _HereDocWriterThunk(write_fd, arg.body) # TODO: Use PIPE_SIZE to save a process in the case of small here docs, # which are the common case. (dash does this.) start_process = True #start_process = False if start_process: here_proc = Process(thunk, self.job_state) # NOTE: we could close the read pipe here, but it doesn't really # matter because we control the code. _ = here_proc.Start() #log('Started %s as %d', here_proc, pid) self._PushWait(here_proc, waiter) # Now that we've started the child, close it in the parent. posix.close(write_fd) else: posix.write(write_fd, arg.body) posix.close(write_fd)
def RunProcessSub(self, cs_part): # type: (command_sub) -> str """Process sub creates a forks a process connected to a pipe. The pipe is typically passed to another process via a /dev/fd/$FD path. Life cycle of a process substitution: 1. Start with this code diff <(seq 3) <(seq 4) 2. To evaluate the command line, we evaluate every word. The NormalWordEvaluator this method, RunProcessSub(), which does 3 things: a. Create a pipe(), getting r and w b. Starts the seq process, which inherits r and w It has a StdoutToPipe() redirect, which means that it dup2(w, 1) and close(r) c. Close the w FD, because neither the shell or 'diff' will write to it. However we must retain 'r', because 'diff' hasn't opened /dev/fd yet! d. We evaluate <(seq 3) to /dev/fd/$r, so "diff" can read from it 3. Now we're done evaluating every word, so we know the command line of diff, which looks like diff /dev/fd/64 /dev/fd/65 Those are the FDs for the read ends of the pipes we created. 4. diff inherits a copy of the read end of bot pipes. But it actually calls open() both files passed as argv. (I think this is fine.) 5. wait() for the diff process. 6. The shell closes both the read ends of both pipes. Neither us or 'diffd' will read again. 7. The shell waits for both 'seq' processes. Related: shopt -s process_sub_fail _process_sub_status """ p = self._MakeProcess(cs_part.child) r, w = posix.pipe() #log('pipe = %d, %d', r, w) op_id = cs_part.left_token.id if op_id == Id.Left_ProcSubIn: # Example: cat < <(head foo.txt) # # The head process should write its stdout to a pipe. redir = process.StdoutToPipe(r, w) # type: process.ChildStateChange elif op_id == Id.Left_ProcSubOut: # Example: head foo.txt > >(tac) # # The tac process should read its stdin from a pipe. # Note: this example sometimes requires you to hit "enter" in bash and # zsh. WHy? redir = process.StdinFromPipe(r, w) else: raise AssertionError() p.AddStateChange(redir) # Fork, letting the child inherit the pipe file descriptors. pid = p.Start() ps_frame = self.process_sub_stack[-1] # Note: bash never waits() on the process, but zsh does. The calling # program needs to read() before we can wait, e.g. # diff <(sort left.txt) <(sort right.txt) ps_frame.to_wait.append(p) ps_frame.span_ids.append(cs_part.left_token.span_id) # After forking, close the end of the pipe we're not using. if op_id == Id.Left_ProcSubIn: posix.close(w) # cat < <(head foo.txt) ps_frame.to_close.append(r) # close later elif op_id == Id.Left_ProcSubOut: posix.close(r) #log('Left_ProcSubOut closed %d', r) ps_frame.to_close.append(w) # close later else: raise AssertionError() # Is /dev Linux-specific? if op_id == Id.Left_ProcSubIn: return '/dev/fd/%d' % r elif op_id == Id.Left_ProcSubOut: return '/dev/fd/%d' % w else: raise AssertionError()
def _ApplyRedirect(self, r, waiter): # type: (redirect_t, Waiter) -> bool ok = True UP_r = r with tagswitch(r) as case: if case(redirect_e.Path): r = cast(redirect__Path, UP_r) if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &> # NOTE: This is different than >| because it respects noclobber, but # that option is almost never used. See test/wild.sh. mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC elif r.op_id == Id.Redir_Clobber: # >| mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC elif r.op_id in (Id.Redir_DGreat, Id.Redir_AndDGreat): # >> &>> mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND elif r.op_id == Id.Redir_Less: # < mode = posix.O_RDONLY else: raise NotImplementedError(r.op_id) # NOTE: 0666 is affected by umask, all shells use it. try: target_fd = posix.open(r.filename, mode, 0o666) except OSError as e: self.errfmt.Print( "Can't open %r: %s", r.filename, posix.strerror(e.errno), span_id=r.op_spid) return False # Apply redirect if not self._PushDup(target_fd, r.fd): ok = False # Now handle the extra redirects for aliases &> and &>>. # # We can rewrite # stdout_stderr.py &> out-err.txt # as # stdout_stderr.py > out-err.txt 2>&1 # # And rewrite # stdout_stderr.py 3&> out-err.txt # as # stdout_stderr.py 3> out-err.txt 2>&3 if ok: if r.op_id == Id.Redir_AndGreat: if not self._PushDup(r.fd, 2): ok = False elif r.op_id == Id.Redir_AndDGreat: if not self._PushDup(r.fd, 2): ok = False posix.close(target_fd) # We already made a copy of it. # I don't think we need to close(0) because it will be restored from its # saved position (10), which closes it. #self._PushClose(r.fd) elif case(redirect_e.FileDesc): # e.g. echo hi 1>&2 r = cast(redirect__FileDesc, UP_r) if r.op_id == Id.Redir_GreatAnd: # 1>&2 if not self._PushDup(r.target_fd, r.fd): ok = False elif r.op_id == Id.Redir_LessAnd: # 0<&5 # The only difference between >& and <& is the default file # descriptor argument. if not self._PushDup(r.target_fd, r.fd): ok = False else: raise NotImplementedError() elif case(redirect_e.HereDoc): r = cast(redirect__HereDoc, UP_r) # NOTE: Do these descriptors have to be moved out of the range 0-9? read_fd, write_fd = posix.pipe() if not self._PushDup(read_fd, r.fd): # stdin is now the pipe ok = False # We can't close like we do in the filename case above? The writer can # get a "broken pipe". self._PushClose(read_fd) thunk = _HereDocWriterThunk(write_fd, r.body) # TODO: Use PIPE_SIZE to save a process in the case of small here docs, # which are the common case. (dash does this.) start_process = True #start_process = False if start_process: here_proc = Process(thunk, self.job_state) # NOTE: we could close the read pipe here, but it doesn't really # matter because we control the code. _ = here_proc.Start() #log('Started %s as %d', here_proc, pid) self._PushWait(here_proc, waiter) # Now that we've started the child, close it in the parent. posix.close(write_fd) else: posix.write(write_fd, r.body) posix.close(write_fd) return ok
def RunProcessSub(self, node, op_id): # type: (command_t, Id_t) -> str """Process sub creates a forks a process connected to a pipe. The pipe is typically passed to another process via a /dev/fd/$FD path. TODO: sane-proc-sub: - wait for all the words Otherwise, set $! (mem.last_bg_pid) strict-proc-sub: - Don't allow it anywhere except SimpleCommand, any redirect, or ShAssignment? And maybe not even assignment? Should you put return codes in @PROCESS_SUB_STATUS? You need two of them. """ p = self._MakeProcess(node) r, w = posix.pipe() if op_id == Id.Left_ProcSubIn: # Example: cat < <(head foo.txt) # # The head process should write its stdout to a pipe. redir = process.StdoutToPipe(r, w) # type: process.ChildStateChange elif op_id == Id.Left_ProcSubOut: # Example: head foo.txt > >(tac) # # The tac process should read its stdin from a pipe. # # NOTE: This appears to hang in bash? At least when done interactively. # It doesn't work at all in osh interactively? redir = process.StdinFromPipe(r, w) else: raise AssertionError() p.AddStateChange(redir) # Fork, letting the child inherit the pipe file descriptors. pid = p.Start() # After forking, close the end of the pipe we're not using. if op_id == Id.Left_ProcSubIn: posix.close(w) elif op_id == Id.Left_ProcSubOut: posix.close(r) else: raise AssertionError() # NOTE: Like bash, we never actually wait on it! # TODO: At least set $! ? # Is /dev Linux-specific? if op_id == Id.Left_ProcSubIn: return '/dev/fd/%d' % r elif op_id == Id.Left_ProcSubOut: return '/dev/fd/%d' % w else: raise AssertionError()
def RunCommandSub(self, cs_part): # type: (command_sub) -> str if not self.exec_opts.allow_command_sub(): # TODO: # - Add spid of $( # - Better hints. Use 'run' for 'if myfunc', and 2 lines like local x; # x=$(false) fo assignment builtins. # - Maybe we should have an error message ID that links somewhere? e_die( "Command subs not allowed here because status wouldn't be checked (strict_errexit)." ) node = cs_part.child # Hack for weird $(<file) construct if node.tag_() == command_e.Simple: simple = cast(command__Simple, node) # Detect '< file' if (len(simple.words) == 0 and len(simple.redirects) == 1 and simple.redirects[0].op.id == Id.Redir_Less): # change it to __cat < file # note: cmd_eval.py _Dispatch works around lack of spid tok = Token(Id.Lit_Chars, runtime.NO_SPID, '__cat') cat_word = compound_word([tok]) # MUTATE the command.Simple node. This will only be done the first # time in the parent process. simple.words.append(cat_word) p = self._MakeProcess(node, inherit_errexit=self.exec_opts.inherit_errexit()) r, w = posix.pipe() p.AddStateChange(process.StdoutToPipe(r, w)) _ = p.Start() #log('Command sub started %d', pid) chunks = [] # type: List[str] posix.close(w) # not going to write while True: byte_str = posix.read(r, 4096) if len(byte_str) == 0: break chunks.append(byte_str) posix.close(r) status = p.Wait(self.waiter) # OSH has the concept of aborting in the middle of a WORD. We're not # waiting until the command is over! if self.exec_opts.command_sub_errexit(): if status != 0: raise error.ErrExit('Command sub exited with status %d (%s)' % (status, ui.CommandType(node)), span_id=cs_part.left_token.span_id, status=status) else: # Set a flag so we check errexit at the same time as bash. Example: # # a=$(false) # echo foo # no matter what comes here, the flag is reset # # Set ONLY until this command node has finished executing. # HACK: move this self.cmd_ev.check_command_sub_status = True self.mem.SetLastStatus(status) # Runtime errors test case: # $("echo foo > $@") # Why rstrip()? # https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char return ''.join(chunks).rstrip('\n')