Beispiel #1
0
    def _Open(self, path, c_mode, fd_mode):
        # type: (str, str, int) -> mylib.LineReader
        fd = posix.open(path, fd_mode, 0o666)  # may raise OSError

        # Immediately move it to a new location
        new_fd = fcntl_.fcntl(fd, fcntl_.F_DUPFD, _SHELL_MIN_FD)  # type: int
        posix.close(fd)

        # Return a Python file handle
        # Might raise IOError.  Python-induced wart: we have to handle BOTH IOError
        # and OSError at higher levels.
        f = posix.fdopen(new_fd, c_mode)
        return f
Beispiel #2
0
    def MaybeWaitOnProcessSubs(self, frame, compound_st):
        # type: (_ProcessSubFrame, CompoundStatus) -> None

        # Wait in the same order that they were evaluated.  That seems fine.
        for fd in frame.to_close:
            posix.close(fd)

        for i, p in enumerate(frame.to_wait):
            #log('waiting for %s', p)
            st = p.Wait(self.waiter)
            compound_st.codes.append(st)
            compound_st.spids.append(frame.span_ids[i])
            i += 1
Beispiel #3
0
    def _Open(self, path, c_mode, fd_mode):
        # type: (str, str, int) -> mylib.LineReader
        fd = posix.open(path, fd_mode, 0o666)  # may raise OSError

        # Immediately move it to a new location
        new_fd = fcntl.fcntl(fd, fcntl.F_DUPFD, _SHELL_MIN_FD)
        posix.close(fd)

        # Return a Python file handle
        try:
            f = posix.fdopen(new_fd, c_mode)  # Might raise IOError
        except IOError as e:
            raise OSError(*e.args)  # Consistently raise OSError
        return f
Beispiel #4
0
    def Run(self, waiter, fd_state):
        # type: (Waiter, FdState) -> List[int]
        """Run this pipeline synchronously (foreground pipeline).

    Returns:
      pipe_status (list of integers).
    """
        self.Start(waiter)

        # Run our portion IN PARALLEL with other processes.  This may or may not
        # fork:
        # ls | wc -l
        # echo foo | read line  # no need to fork

        cmd_ev, node = self.last_thunk

        #log('thunk %s', self.last_thunk)
        if self.last_pipe is not None:
            r, w = self.last_pipe  # set in AddLast()
            posix.close(w)  # we will not write here
            fd_state.PushStdinFromPipe(r)

            # TODO: determine fork_external here, so we can go BEYOND lastpipe.  Not
            # only do we run builtins in the same process.  External processes will
            # exec() rather than fork/exec().

            try:
                cmd_ev.ExecuteAndCatch(node)
            finally:
                fd_state.Pop()
            # We won't read anymore.  If we don't do this, then 'cat' in 'cat
            # /dev/urandom | sleep 1' will never get SIGPIPE.
            posix.close(r)

        else:
            if len(self.procs):
                cmd_ev.ExecuteAndCatch(
                    node)  # Background pipeline without last_pipe
            else:
                cmd_ev._Execute(
                    node)  # singleton foreground pipeline, e.g. '! func'

        self.pipe_status[-1] = cmd_ev.LastStatus()
        #log('pipestatus before all have finished = %s', self.pipe_status)

        if len(self.procs):
            return self.Wait(waiter)
        else:
            return self.pipe_status  # singleton foreground pipeline, e.g. '! func'
Beispiel #5
0
    def Run(self, waiter, fd_state):
        # type: (Waiter, FdState) -> List[int]
        """Run this pipeline synchronously (foreground pipeline).

    Returns:
      pipe_status (list of integers).
    """
        self.Start(waiter)

        # Run our portion IN PARALLEL with other processes.  This may or may not
        # fork:
        # ls | wc -l
        # echo foo | read line  # no need to fork

        ex, node = self.last_thunk

        #log('thunk %s', self.last_thunk)
        if self.last_pipe is not None:
            r, w = self.last_pipe  # set in AddLast()
            posix.close(w)  # we will not write here
            fd_state.PushStdinFromPipe(r)
            try:
                ex.ExecuteAndCatch(node)
            finally:
                fd_state.Pop()
            # We won't read anymore.  If we don't do this, then 'cat' in 'cat
            # /dev/urandom | sleep 1' will never get SIGPIPE.
            posix.close(r)

        else:
            if self.procs:
                ex.ExecuteAndCatch(
                    node)  # Background pipeline without last_pipe
            else:
                ex._Execute(
                    node)  # singleton foreground pipeline, e.g. '! func'

        self.pipe_status[-1] = ex.LastStatus()
        #log('pipestatus before all have finished = %s', self.pipe_status)

        if self.procs:
            return self.Wait(waiter)
        else:
            return self.pipe_status  # singleton foreground pipeline, e.g. '! func'
Beispiel #6
0
  def _PushDup(self, fd1, fd2):
    # type: (int, int) -> bool
    """Save fd2, and dup fd1 onto fd2.

    Mutates self.cur_frame.saved.

    Returns:
      success Bool
    """
    new_fd = self._GetFreeDescriptor()
    #log('---- _PushDup %s %s', fd1, fd2)
    need_restore = True
    try:
      fcntl.fcntl(fd2, fcntl.F_DUPFD, new_fd)
    except IOError as e:
      # Example program that causes this error: exec 4>&1.  Descriptor 4 isn't
      # open.
      # This seems to be ignored in dash too in savefd()?
      if e.errno == errno.EBADF:
        #log('ERROR %s', e)
        need_restore = False
      else:
        raise
    else:
      posix.close(fd2)
      fcntl.fcntl(new_fd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)

    #log('==== dup %s %s\n' % (fd1, fd2))
    try:
      posix.dup2(fd1, fd2)
    except OSError as e:
      # bash/dash give this error too, e.g. for 'echo hi 1>&3'
      self.errfmt.Print('%d: %s', fd1, posix.strerror(e.errno))

      # Restore and return error
      posix.dup2(new_fd, fd2)
      posix.close(new_fd)
      # Undo it
      return False

    if need_restore:
      self.cur_frame.saved.append((new_fd, fd2))
    return True
Beispiel #7
0
  def RunCommandSub(self, node):
    p = self._MakeProcess(node,
                          disable_errexit=not self.exec_opts.strict_errexit)

    r, w = posix.pipe()
    p.AddStateChange(process.StdoutToPipe(r, w))
    pid = p.Start()
    #log('Command sub started %d', pid)

    chunks = []
    posix.close(w)  # not going to write
    while True:
      byte_str = posix.read(r, 4096)
      if not byte_str:
        break
      chunks.append(byte_str)
    posix.close(r)

    status = p.Wait(self.waiter)

    # OSH has the concept of aborting in the middle of a WORD.  We're not
    # waiting until the command is over!
    if self.exec_opts.strict_errexit:
      if self.exec_opts.ErrExit() and status != 0:
        raise util.ErrExitFailure(
            'Command sub exited with status %d (%r)', status,
            node.__class__.__name__)
    else:
      # Set a flag so we check errexit at the same time as bash.  Example:
      #
      # a=$(false)
      # echo foo  # no matter what comes here, the flag is reset
      #
      # Set ONLY until this command node has finished executing.
      self.check_command_sub_status = True
      self.mem.SetLastStatus(status)

    # Runtime errors test case: # $("echo foo > $@")
    # Why rstrip()?
    # https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
    return ''.join(chunks).rstrip('\n')
Beispiel #8
0
 def MaybeClosePipe(self):
     # type: () -> None
     if self.close_r != -1:
         posix.close(self.close_r)
         posix.close(self.close_w)
Beispiel #9
0
    def Apply(self):
        # type: () -> None
        posix.dup2(self.w, 1)
        posix.close(self.w)  # close after dup

        posix.close(self.r)  # we're writing to the pipe, not reading
Beispiel #10
0
    def Apply(self):
        # type: () -> None
        posix.dup2(self.r, 0)
        posix.close(self.r)  # close after dup

        posix.close(self.w)  # we're reading from the pipe, not writing
Beispiel #11
0
    def _ApplyRedirect(self, r, waiter):
        # type: (redirect, Waiter) -> None
        arg = r.arg
        UP_arg = arg
        with tagswitch(arg) as case:

            if case(redirect_arg_e.Path):
                arg = cast(redirect_arg__Path, UP_arg)

                if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat):  # >   &>
                    # NOTE: This is different than >| because it respects noclobber, but
                    # that option is almost never used.  See test/wild.sh.
                    mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
                elif r.op_id == Id.Redir_Clobber:  # >|
                    mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
                elif r.op_id in (Id.Redir_DGreat,
                                 Id.Redir_AndDGreat):  # >>   &>>
                    mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND
                elif r.op_id == Id.Redir_Less:  # <
                    mode = posix.O_RDONLY
                elif r.op_id == Id.Redir_LessGreat:  # <>
                    mode = posix.O_CREAT | posix.O_RDWR
                else:
                    raise NotImplementedError(r.op_id)

                # NOTE: 0666 is affected by umask, all shells use it.
                try:
                    open_fd = posix.open(arg.filename, mode, 0o666)
                except OSError as e:
                    self.errfmt.Print_("Can't open %r: %s" %
                                       (arg.filename, pyutil.strerror(e)),
                                       span_id=r.op_spid)
                    raise  # redirect failed

                new_fd = self._PushDup(open_fd, r.loc)
                if new_fd != NO_FD:
                    posix.close(open_fd)

                # Now handle &> and &>> and their variants.  These pairs are the same:
                #
                #   stdout_stderr.py &> out-err.txt
                #   stdout_stderr.py > out-err.txt 2>&1
                #
                #   stdout_stderr.py 3&> out-err.txt
                #   stdout_stderr.py 3> out-err.txt 2>&3
                #
                # Ditto for {fd}> and {fd}&>

                if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
                    self._PushDup(new_fd, redir_loc.Fd(2))

            elif case(redirect_arg_e.CopyFd):  # e.g. echo hi 1>&2
                arg = cast(redirect_arg__CopyFd, UP_arg)

                if r.op_id == Id.Redir_GreatAnd:  # 1>&2
                    self._PushDup(arg.target_fd, r.loc)

                elif r.op_id == Id.Redir_LessAnd:  # 0<&5
                    # The only difference between >& and <& is the default file
                    # descriptor argument.
                    self._PushDup(arg.target_fd, r.loc)

                else:
                    raise NotImplementedError()

            elif case(redirect_arg_e.MoveFd):  # e.g. echo hi 5>&6-
                arg = cast(redirect_arg__MoveFd, UP_arg)
                new_fd = self._PushDup(arg.target_fd, r.loc)
                if new_fd != NO_FD:
                    posix.close(arg.target_fd)

                    UP_loc = r.loc
                    if r.loc.tag_() == redir_loc_e.Fd:
                        fd = cast(redir_loc__Fd, UP_loc).fd
                    else:
                        fd = NO_FD

                    self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))

            elif case(redirect_arg_e.CloseFd):  # e.g. echo hi 5>&-
                self._PushCloseFd(r.loc)

            elif case(redirect_arg_e.HereDoc):
                arg = cast(redirect_arg__HereDoc, UP_arg)

                # NOTE: Do these descriptors have to be moved out of the range 0-9?
                read_fd, write_fd = posix.pipe()

                self._PushDup(read_fd, r.loc)  # stdin is now the pipe

                # We can't close like we do in the filename case above?  The writer can
                # get a "broken pipe".
                self._PushClose(read_fd)

                thunk = _HereDocWriterThunk(write_fd, arg.body)

                # TODO: Use PIPE_SIZE to save a process in the case of small here docs,
                # which are the common case.  (dash does this.)
                start_process = True
                #start_process = False

                if start_process:
                    here_proc = Process(thunk, self.job_state)

                    # NOTE: we could close the read pipe here, but it doesn't really
                    # matter because we control the code.
                    _ = here_proc.Start()
                    #log('Started %s as %d', here_proc, pid)
                    self._PushWait(here_proc, waiter)

                    # Now that we've started the child, close it in the parent.
                    posix.close(write_fd)

                else:
                    posix.write(write_fd, arg.body)
                    posix.close(write_fd)
Beispiel #12
0
  def RunProcessSub(self, cs_part):
    # type: (command_sub) -> str
    """Process sub creates a forks a process connected to a pipe.

    The pipe is typically passed to another process via a /dev/fd/$FD path.

    Life cycle of a process substitution:

    1. Start with this code

      diff <(seq 3) <(seq 4)

    2. To evaluate the command line, we evaluate every word.  The
    NormalWordEvaluator this method, RunProcessSub(), which does 3 things:

      a. Create a pipe(), getting r and w
      b. Starts the seq process, which inherits r and w
         It has a StdoutToPipe() redirect, which means that it dup2(w, 1)
         and close(r)
      c. Close the w FD, because neither the shell or 'diff' will write to it.
         However we must retain 'r', because 'diff' hasn't opened /dev/fd yet!
      d. We evaluate <(seq 3) to /dev/fd/$r, so "diff" can read from it

    3. Now we're done evaluating every word, so we know the command line of
       diff, which looks like

      diff /dev/fd/64 /dev/fd/65

    Those are the FDs for the read ends of the pipes we created.

    4. diff inherits a copy of the read end of bot pipes.  But it actually
    calls open() both files passed as argv.  (I think this is fine.)

    5. wait() for the diff process.

    6. The shell closes both the read ends of both pipes.  Neither us or
    'diffd' will read again.

    7. The shell waits for both 'seq' processes.

    Related:
      shopt -s process_sub_fail
      _process_sub_status
    """
    p = self._MakeProcess(cs_part.child)

    r, w = posix.pipe()
    #log('pipe = %d, %d', r, w)

    op_id = cs_part.left_token.id
    if op_id == Id.Left_ProcSubIn:
      # Example: cat < <(head foo.txt)
      #
      # The head process should write its stdout to a pipe.
      redir = process.StdoutToPipe(r, w)  # type: process.ChildStateChange

    elif op_id == Id.Left_ProcSubOut:
      # Example: head foo.txt > >(tac)
      #
      # The tac process should read its stdin from a pipe.

      # Note: this example sometimes requires you to hit "enter" in bash and
      # zsh.  WHy?
      redir = process.StdinFromPipe(r, w)

    else:
      raise AssertionError()

    p.AddStateChange(redir)

    # Fork, letting the child inherit the pipe file descriptors.
    pid = p.Start()

    ps_frame = self.process_sub_stack[-1]

    # Note: bash never waits() on the process, but zsh does.  The calling
    # program needs to read() before we can wait, e.g.
    #   diff <(sort left.txt) <(sort right.txt)
    ps_frame.to_wait.append(p)
    ps_frame.span_ids.append(cs_part.left_token.span_id)

    # After forking, close the end of the pipe we're not using.
    if op_id == Id.Left_ProcSubIn:
      posix.close(w)  # cat < <(head foo.txt)
      ps_frame.to_close.append(r)  # close later
    elif op_id == Id.Left_ProcSubOut:
      posix.close(r)
      #log('Left_ProcSubOut closed %d', r)
      ps_frame.to_close.append(w)  # close later
    else:
      raise AssertionError()

    # Is /dev Linux-specific?
    if op_id == Id.Left_ProcSubIn:
      return '/dev/fd/%d' % r

    elif op_id == Id.Left_ProcSubOut:
      return '/dev/fd/%d' % w

    else:
      raise AssertionError()
Beispiel #13
0
  def RunCommandSub(self, cs_part):
    # type: (command_sub) -> str

    if not self.exec_opts.allow_command_sub():
      # TODO: Add spid of $(
      e_die("Command subs not allowed when errexit disabled (strict_errexit)")

    node = cs_part.child

    # Hack for weird $(<file) construct
    if node.tag_() == command_e.Simple:
      simple = cast(command__Simple, node)
      # Detect '< file'
      if (len(simple.words) == 0 and
          len(simple.redirects) == 1 and
          simple.redirects[0].op.id == Id.Redir_Less):
        # change it to __cat < file
        # note: cmd_eval.py _Dispatch works around lack of spid
        tok = Token(Id.Lit_Chars, runtime.NO_SPID, '__cat')
        cat_word = compound_word([tok])
        # MUTATE the command.Simple node.  This will only be done the first
        # time in the parent process.
        simple.words.append(cat_word)

    p = self._MakeProcess(node,
                          inherit_errexit=self.exec_opts.inherit_errexit())

    r, w = posix.pipe()
    p.AddStateChange(process.StdoutToPipe(r, w))
    _ = p.Start()
    #log('Command sub started %d', pid)

    chunks = []  # type: List[str]
    posix.close(w)  # not going to write
    while True:
      byte_str = posix.read(r, 4096)
      if len(byte_str) == 0:
        break
      chunks.append(byte_str)
    posix.close(r)

    status = p.Wait(self.waiter)

    # OSH has the concept of aborting in the middle of a WORD.  We're not
    # waiting until the command is over!
    if self.exec_opts.command_sub_errexit():
      if status != 0:
        raise error.ErrExit(
            'Command sub exited with status %d (%s)' %
            (status, ui.CommandType(node)), span_id=cs_part.left_token.span_id,
            status=status)

    else:
      # Set a flag so we check errexit at the same time as bash.  Example:
      #
      # a=$(false)
      # echo foo  # no matter what comes here, the flag is reset
      #
      # Set ONLY until this command node has finished executing.

      # HACK: move this
      self.cmd_ev.check_command_sub_status = True
      self.mem.SetLastStatus(status)

    # Runtime errors test case: # $("echo foo > $@")
    # Why rstrip()?
    # https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
    return ''.join(chunks).rstrip('\n')
Beispiel #14
0
  def _ApplyRedirect(self, r, waiter):
    # type: (redirect_t, Waiter) -> bool
    ok = True

    UP_r = r
    with tagswitch(r) as case:

      if case(redirect_e.Path):
        r = cast(redirect__Path, UP_r)

        if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat):  # >   &>
          # NOTE: This is different than >| because it respects noclobber, but
          # that option is almost never used.  See test/wild.sh.
          mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
        elif r.op_id == Id.Redir_Clobber:  # >|
          mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
        elif r.op_id in (Id.Redir_DGreat, Id.Redir_AndDGreat):  # >>   &>>
          mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND
        elif r.op_id == Id.Redir_Less:  # <
          mode = posix.O_RDONLY
        else:
          raise NotImplementedError(r.op_id)

        # NOTE: 0666 is affected by umask, all shells use it.
        try:
          target_fd = posix.open(r.filename, mode, 0o666)
        except OSError as e:
          self.errfmt.Print(
              "Can't open %r: %s", r.filename, posix.strerror(e.errno),
              span_id=r.op_spid)
          return False

        # Apply redirect
        if not self._PushDup(target_fd, r.fd):
          ok = False

        # Now handle the extra redirects for aliases &> and &>>.
        #
        # We can rewrite
        #   stdout_stderr.py &> out-err.txt
        # as
        #   stdout_stderr.py > out-err.txt 2>&1
        #
        # And rewrite
        #   stdout_stderr.py 3&> out-err.txt
        # as
        #   stdout_stderr.py 3> out-err.txt 2>&3
        if ok:
          if r.op_id == Id.Redir_AndGreat:
            if not self._PushDup(r.fd, 2):
              ok = False
          elif r.op_id == Id.Redir_AndDGreat:
            if not self._PushDup(r.fd, 2):
              ok = False

        posix.close(target_fd)  # We already made a copy of it.
        # I don't think we need to close(0) because it will be restored from its
        # saved position (10), which closes it.
        #self._PushClose(r.fd)

      elif case(redirect_e.FileDesc):  # e.g. echo hi 1>&2
        r = cast(redirect__FileDesc, UP_r)

        if r.op_id == Id.Redir_GreatAnd:  # 1>&2
          if not self._PushDup(r.target_fd, r.fd):
            ok = False
        elif r.op_id == Id.Redir_LessAnd:  # 0<&5
          # The only difference between >& and <& is the default file
          # descriptor argument.
          if not self._PushDup(r.target_fd, r.fd):
            ok = False
        else:
          raise NotImplementedError()

      elif case(redirect_e.HereDoc):
        r = cast(redirect__HereDoc, UP_r)

        # NOTE: Do these descriptors have to be moved out of the range 0-9?
        read_fd, write_fd = posix.pipe()

        if not self._PushDup(read_fd, r.fd):  # stdin is now the pipe
          ok = False

        # We can't close like we do in the filename case above?  The writer can
        # get a "broken pipe".
        self._PushClose(read_fd)

        thunk = _HereDocWriterThunk(write_fd, r.body)

        # TODO: Use PIPE_SIZE to save a process in the case of small here docs,
        # which are the common case.  (dash does this.)
        start_process = True
        #start_process = False

        if start_process:
          here_proc = Process(thunk, self.job_state)

          # NOTE: we could close the read pipe here, but it doesn't really
          # matter because we control the code.
          _ = here_proc.Start()
          #log('Started %s as %d', here_proc, pid)
          self._PushWait(here_proc, waiter)

          # Now that we've started the child, close it in the parent.
          posix.close(write_fd)

        else:
          posix.write(write_fd, r.body)
          posix.close(write_fd)

    return ok
Beispiel #15
0
 def MaybeClosePipe(self):
     if self.close_r != -1:
         posix.close(self.close_r)
         posix.close(self.close_w)
Beispiel #16
0
    def RunProcessSub(self, node, op_id):
        # type: (command_t, Id_t) -> str
        """Process sub creates a forks a process connected to a pipe.

    The pipe is typically passed to another process via a /dev/fd/$FD path.

    TODO:

    sane-proc-sub:
    - wait for all the words

    Otherwise, set $!  (mem.last_bg_pid)

    strict-proc-sub:
    - Don't allow it anywhere except SimpleCommand, any redirect, or
    ShAssignment?  And maybe not even assignment?

    Should you put return codes in @PROCESS_SUB_STATUS?  You need two of them.
    """
        p = self._MakeProcess(node)

        r, w = posix.pipe()

        if op_id == Id.Left_ProcSubIn:
            # Example: cat < <(head foo.txt)
            #
            # The head process should write its stdout to a pipe.
            redir = process.StdoutToPipe(r,
                                         w)  # type: process.ChildStateChange

        elif op_id == Id.Left_ProcSubOut:
            # Example: head foo.txt > >(tac)
            #
            # The tac process should read its stdin from a pipe.
            #
            # NOTE: This appears to hang in bash?  At least when done interactively.
            # It doesn't work at all in osh interactively?
            redir = process.StdinFromPipe(r, w)

        else:
            raise AssertionError()

        p.AddStateChange(redir)

        # Fork, letting the child inherit the pipe file descriptors.
        pid = p.Start()

        # After forking, close the end of the pipe we're not using.
        if op_id == Id.Left_ProcSubIn:
            posix.close(w)
        elif op_id == Id.Left_ProcSubOut:
            posix.close(r)
        else:
            raise AssertionError()

        # NOTE: Like bash, we never actually wait on it!
        # TODO: At least set $! ?

        # Is /dev Linux-specific?
        if op_id == Id.Left_ProcSubIn:
            return '/dev/fd/%d' % r

        elif op_id == Id.Left_ProcSubOut:
            return '/dev/fd/%d' % w

        else:
            raise AssertionError()